diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000000..5eead66489830d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +# Binary data types +*.aif binary +*.aifc binary +*.aiff binary +*.au binary +*.bmp binary +*.exe binary +*.icns binary +*.gif binary +*.ico binary +*.jpg binary +*.pck binary +*.png binary +*.psd binary +*.tar binary +*.wav binary +*.whl binary +*.zip binary + +# Specific binary files +Lib/test/sndhdrdata/sndhdr.* binary + +# Text files that should not be subject to eol conversion +Lib/test/cjkencodings/* -text +Lib/test/decimaltestdata/*.decTest -text +Lib/test/test_email/data/*.txt -text +Lib/test/xmltestdata/* -text +Lib/test/coding20731.py -text + +# Special files in third party code +Modules/zlib/zlib.map -text + +# CRLF files +*.bat text eol=crlf +*.ps1 text eol=crlf +*.sln text eol=crlf +*.vcxproj* text eol=crlf +*.props text eol=crlf +*.proj text eol=crlf +PCbuild/readme.txt text eol=crlf +PC/readme.txt text eol=crlf diff --git a/.github/appveyor.yml b/.github/appveyor.yml new file mode 100644 index 00000000000000..96674ce84d8cdd --- /dev/null +++ b/.github/appveyor.yml @@ -0,0 +1,36 @@ +version: 3.6.1+.{build} +clone_depth: 5 +branches: + only: + - master + - /\d\.\d/ + - buildbot-custom +cache: + - externals -> PCbuild\* +build_script: +- cmd: PCbuild\build.bat -e +- cmd: PCbuild\win32\python.exe -m test.pythoninfo +test_script: +- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 +environment: + HOST_PYTHON: C:\Python36\python.exe +image: +- Visual Studio 2015 +- Visual Studio 2017 + +# Only trigger AppVeyor if actual code or its configuration changes +only_commits: + files: + - .github/appveyor.yml + - .gitattributes + - Grammar/ + - Include/ + - Lib/ + - Modules/ + - Objects/ + - PC/ + - PCBuild/ + - Parser/ + - Programs/ + - Python/ + - Tools/ diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 00000000000000..dc21321d0baaf0 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,36 @@ +codecov: + strict_yaml_branch: master + notify: + require_ci_to_pass: true +comment: off +ignore: + - "Doc/**/*" + - "Misc/*" + - "Mac/**/*" + - "PC/**/*" + - "PCbuild/**/*" + - "Tools/**/*" + - "Grammar/*" +coverage: + precision: 2 + range: + - 70.0 + - 100.0 + round: down + status: + changes: off + project: off + patch: + default: + target: 100% + only_pulls: true + threshold: 0.05 +parsers: + gcov: + branch_detection: + conditional: true + loop: true + macro: false + method: false + javascript: + enable_partials: false diff --git a/.gitignore b/.gitignore index ed4ebfbbd9de74..81f31989c55a70 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ .gdb_history Doc/build/ Doc/venv/ +Doc/.venv/ +Doc/env/ +Doc/.env/ +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* @@ -51,10 +55,12 @@ PCbuild/*.suo PCbuild/*.*sdf PCbuild/*-pgi PCbuild/*-pgo +PCbuild/*.VC.db +PCbuild/*.VC.opendb PCbuild/.vs/ PCbuild/amd64/ PCbuild/obj/ -PCBuild/win32/ +PCbuild/win32/ .purify Parser/pgen __pycache__ diff --git a/.hgtouch b/.hgtouch deleted file mode 100644 index b9be0f11fdb829..00000000000000 --- a/.hgtouch +++ /dev/null @@ -1,17 +0,0 @@ -# -*- Makefile -*- -# Define dependencies of generated files that are checked into hg. -# The syntax of this file uses make rule dependencies, without actions - -Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c - -Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py - -Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py -Python/Python-ast.c: Include/Python-ast.h - -Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py - -Objects/typeslots.inc: Include/typeslots.h Objects/typeslots.py - -Include/graminit.h: Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c -Python/graminit.c: Include/graminit.h Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..031ba792cbbe64 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,105 @@ +language: c +dist: trusty +sudo: false +group: beta + +# To cache doc-building dependencies and C compiler output. +cache: + - pip + - ccache + +branches: + only: + - master + - /^\d\.\d$/ + +matrix: + fast_finish: true + allow_failures: + - env: OPTIONAL=true + include: + - os: linux + language: c + compiler: clang + # gcc also works, but to keep the # of concurrent builds down, we use one C + # compiler here and the other to run the coverage build. Clang is preferred + # in this instance for its better error messages. + env: TESTING=cpython + - os: linux + language: python + python: 3.6 + env: TESTING=docs + before_script: + - cd Doc + # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. + # (Updating the version is fine as long as no warnings are raised by doing so.) + - python -m pip install sphinx~=1.6.1 blurb + script: + - make check suspicious html SPHINXOPTS="-q -W -j4" + - os: linux + language: c + compiler: gcc + env: OPTIONAL=true + before_script: + - | + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure + make -s -j4 + # Need a venv that can parse covered code. + ./python -m venv venv + ./venv/bin/python -m pip install -U coverage + ./venv/bin/python -m test.pythoninfo + script: + # Skip tests that re-run the entire test suite. + - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - source ./venv/bin/activate + - bash <(curl -s https://codecov.io/bash) + +# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. +before_script: + - | + set -e + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + exit + fi + ./configure --with-pydebug + make -j4 + make -j4 regen-all clinic + changes=`git status --porcelain` + if ! test -z "$changes" + then + echo "Generated files not up to date" + echo "$changes" + exit 1 + fi + make pythoninfo + +script: + # Using the built Python as patchcheck.py is built around the idea of using + # a checkout-build of CPython to know things like what base branch the changes + # should be compared against. + # Only run on Linux as the check only needs to be run once. + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4 -uall,-cpu" + +notifications: + email: false + irc: + channels: + # This is set to a secure variable to prevent forks from notifying the + # IRC channel whenever they fail a build. This can be removed when travis + # implements https://github.com/travis-ci/travis-ci/issues/1094. + # The actual value here is: irc.freenode.net#python-dev + - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" + on_success: change + on_failure: always + skip_join: true diff --git a/Doc/Makefile b/Doc/Makefile index 91f937f985831a..307d1e0e7de10b 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -4,13 +4,15 @@ # # You can set these variables from the command line. -PYTHON = python -SPHINXBUILD = sphinx-build +PYTHON = python3 +VENVDIR = ./venv +SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build +BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text changes linkcheck \ @@ -38,6 +40,20 @@ help: @echo " serve to serve the documentation on the localhost (8000)" build: + -mkdir -p build +# Look first for a Misc/NEWS file (building from a source release tarball +# or old repo) and use that, otherwise look for a Misc/NEWS.d directory +# (building from a newer repo) and use blurb to generate the NEWS file. + @if [ -f ../Misc/NEWS ] ; then \ + echo "Using existing Misc/NEWS file"; \ + cp ../Misc/NEWS build/NEWS; \ + elif [ -d ../Misc/NEWS.d ]; then \ + echo "Building NEWS from Misc/NEWS.d with blurb"; \ + $(BLURB) merge -f build/NEWS; \ + else \ + echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + exit 1; \ + fi $(SPHINXBUILD) $(ALLSPHINXOPTS) @echo @@ -103,11 +119,12 @@ htmlview: html $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" clean: - -rm -rf build/* venv/* + -rm -rf build/* $(VENVDIR)/* venv: - $(PYTHON) -m venv venv - ./venv/bin/python3 -m pip install -U Sphinx + $(PYTHON) -m venv $(VENVDIR) + $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb + @echo "The venv has been created in the $(VENVDIR) directory" dist: rm -rf dist @@ -153,21 +170,26 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools -i venv + $(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst serve: ../Tools/scripts/serve.py build/html # Targets for daily automated doc build +# By default, Sphinx only rebuilds pages where the page content has changed. +# This means it doesn't always pick up changes to preferred link targets, etc +# To ensure such changes are picked up, we build the published docs with +# `-E` (to ignore the cached environment) and `-a` (to ignore already existing +# output files) # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/README.txt b/Doc/README.rst similarity index 62% rename from Doc/README.txt rename to Doc/README.rst index 4f8e9f8f1417fb..a29d1f3a708a43 100644 --- a/Doc/README.txt +++ b/Doc/README.rst @@ -2,49 +2,58 @@ Python Documentation README ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, prebuilt versions are -available at . +documentation. You don't need to build them yourself, `prebuilt versions are +available `_. Documentation on authoring Python documentation, including information about -both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +both style and markup, is available in the "`Documenting Python +`_" chapter of the +developers guide. Building the docs ================= -You need to have Sphinx installed; it is the toolset -used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +The documentation is built with several tools which are not included in this +tree but are maintained separately and are available from +`PyPI `_. + +* `Sphinx `_ +* `blurb `_ + +The easiest way to install these tools is to create a virtual environment and +install the tools into there. Using make ---------- -A Makefile has been prepared so that on Unix, provided you have installed -Sphinx, you can just run :: +To get started on UNIX, you can create a virtual environment with the command :: - make html + make venv -to build the HTML output files. - -On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. +That will install all the tools necessary to build the documentation. Assuming +the virtual environment was created in the ``env`` directory (the default; +configurable with the VENVDIR variable), you can run the following command to +build the HTML output files:: -To use a Python interpreter that's not called ``python``, use the standard -way to set Makefile variables, using e.g. :: + make html - make html PYTHON=python3 +By default, if the virtual environment is not created, the Makefile will +look for instances of sphinxbuild and blurb installed on your process PATH +(configurable with the SPHINXBUILD and BLURB variables). -On Windows, set the PYTHON environment variable instead. - -To use a specific sphinx-build (something other than ``sphinx-build``), set -the SPHINXBUILD variable. +On Windows, we try to emulate the Makefile as closely as possible with a +``make.bat`` file. If you need to specify the Python interpreter to use, +set the PYTHON environment variable instead. Available make targets are: * "clean", which removes all build files. +* "venv", which creates a virtual environment with all necessary tools + installed. + * "html", which builds standalone HTML files for offline viewing. * "htmlview", which re-uses the "html" builder, but then opens the main page @@ -95,7 +104,7 @@ Available make targets are: Without make ------------ -Install the Sphinx package and its dependencies from PyPI. +First, install the tool dependencies from PyPI. Then, from the ``Doc`` directory, run :: @@ -108,11 +117,10 @@ see the make targets above). Contributing ============ -Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +Bugs in the content should be reported to the +`Python bug tracker `_. -Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +Bugs in the toolset should be reported to the tools themselves. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 1b0a5a9a9377cb..bc1d10f379cb35 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -88,5 +88,5 @@ the `core-mentorship mailing list`_ is a friendly place to get answers to any and all questions pertaining to the process of fixing issues in Python. .. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity -.. _Python Developer's Guide: https://docs.python.org/devguide/ +.. _Python Developer's Guide: https://devguide.python.org/ .. _core-mentorship mailing list: https://mail.python.org/mailman/listinfo/core-mentorship/ diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85cfd11754..2bc1bd876a2fe2 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -291,16 +291,11 @@ an error value). is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. - Warning categories must be subclasses of :c:data:`Warning`; the default warning - category is :c:data:`RuntimeWarning`. The standard Python warning categories are - available as global variables whose names are ``PyExc_`` followed by the Python - exception name. These have the type :c:type:`PyObject\*`; they are all class - objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`, - :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`, - :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and - :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of - :c:data:`PyExc_Exception`; the other warning categories are subclasses of - :c:data:`PyExc_Warning`. + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. For information about warning control, see the documentation for the :mod:`warnings` module and the :option:`-W` option in the command line @@ -750,6 +745,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +809,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +817,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +853,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +879,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +921,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +946,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +957,60 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f592cb65c3e3a9..f50680b3d29558 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -85,13 +85,12 @@ All integers are implemented as "long" integer objects of arbitrary size. Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, the radix will be - determined based on the leading characters of *str*: if *str* starts with - ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or - ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``, - radix 2 will be used; otherwise radix 10 will be used. If *base* is not - ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are - ignored. If there are no digits, :exc:`ValueError` will be raised. + representation of the number. If *base* is ``0``, *str* is interpreted using + the :ref:`integers` definition; in this case, leading zeros in a + non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, + it must be between ``2`` and ``36``, inclusive. Leading spaces and single + underscores after a base specifier and between digits are ignored. If there + are no digits, :exc:`ValueError` will be raised. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f4688d1b78..c6d1d02a2fa510 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff545275fb3b8..873fb2ac1d3cad 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7724350d3c5f95..d3125b86f0729a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -50,7 +50,7 @@ Module Objects .. c:function:: PyObject* PyModule_New(const char *name) - Similar to :c:func:`PyImport_NewObject`, but the name is a UTF-8 encoded + Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded string instead of a Unicode object. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index a825164918f446..8b695e065aeffd 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -56,3 +56,14 @@ Slice Objects .. versionchanged:: 3.2 The parameter type for the *slice* parameter was ``PySliceObject*`` before. + + +Ellipsis Object +--------------- + + +.. c:var:: PyObject *Py_Ellipsis + + The Python ``Ellipsis`` object. This object has no methods. It needs to be + treated just like any other object with respect to reference counts. Like + :c:data:`Py_None` it is a singleton object. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f48119391f2bbd..675f6f26921c55 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. @@ -294,3 +294,43 @@ definition with the same method name. read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies :c:macro:`READONLY`. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` members can be deleted. (They are set to *NULL*). + + +.. c:type:: PyGetSetDef + + Structure to define property-like access for a type. See also description of + the :c:member:`PyTypeObject.tp_getset` slot. + + +-------------+------------------+-----------------------------------+ + | Field | C Type | Meaning | + +=============+==================+===================================+ + | name | char \* | attribute name | + +-------------+------------------+-----------------------------------+ + | get | getter | C Function to get the attribute | + +-------------+------------------+-----------------------------------+ + | set | setter | optional C function to set or | + | | | delete the attribute, if omitted | + | | | the attribute is readonly | + +-------------+------------------+-----------------------------------+ + | doc | char \* | optional docstring | + +-------------+------------------+-----------------------------------+ + | closure | void \* | optional function pointer, | + | | | providing additional data for | + | | | getter and setter | + +-------------+------------------+-----------------------------------+ + + The ``get`` function takes one :c:type:`PyObject\*` parameter (the + instance) and a function pointer (the associated ``closure``):: + + typedef PyObject *(*getter)(PyObject *, void *); + + It should return a new reference on success or *NULL* with a set exception + on failure. + + ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and + the value to be set) and a function pointer (the associated ``closure``):: + + typedef int (*setter)(PyObject *, PyObject *, void *); + + In case the attribute should be deleted the second parameter is *NULL*. + Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ac6fd0b53fcd6d..0b4577f5b950a9 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -719,21 +719,6 @@ type objects) *must* have the :attr:`ob_size` field. This field is not inherited by subtypes (computed attributes are inherited through a different mechanism). - .. XXX belongs elsewhere - - Docs for PyGetSetDef:: - - typedef PyObject *(*getter)(PyObject *, void *); - typedef int (*setter)(PyObject *, PyObject *, void *); - - typedef struct PyGetSetDef { - char *name; /* attribute name */ - getter get; /* C function to get the attribute */ - setter set; /* C function to set or delete the attribute */ - char *doc; /* optional doc string */ - void *closure; /* optional additional data for getter and setter */ - } PyGetSetDef; - .. c:member:: PyTypeObject* PyTypeObject.tp_base diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada7be7ee4..6e91576ee8f004 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Doc/conf.py b/Doc/conf.py index b1bb6208bb4b8e..d4ee50de7db97b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,7 @@ needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] +exclude_patterns = ['venv/*', 'README.rst'] # Options for HTML output @@ -88,11 +88,28 @@ # Options for LaTeX output # ------------------------ +# Get LaTeX to handle Unicode correctly +latex_elements = { + 'inputenc': r'\usepackage[utf8x]{inputenc}', + 'utf8extra': '', + 'fontenc': r'\usepackage[T1,T2A]{fontenc}', +} + +# Additional stuff for the LaTeX preamble. +latex_elements['preamble'] = r''' +\authoraddress{ + \sphinxstrong{Python Software Foundation}\\ + Email: \sphinxemail{docs@python.org} +} +\let\Verbatim=\OriginalVerbatim +\let\endVerbatim=\endOriginalVerbatim +''' + # The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +latex_elements['papersize'] = 'a4' # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_elements['pointsize'] = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -125,22 +142,9 @@ for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst') -# Additional stuff for the LaTeX preamble. -latex_preamble = r''' -\authoraddress{ - \strong{Python Software Foundation}\\ - Email: \email{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] -# Get LaTeX to handle Unicode correctly -latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} - # Options for Epub output # ----------------------- diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 22d7705846ea93..2b5400cd9fb036 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2016 Python Software Foundation. All rights reserved. +Copyright © 2001-2017 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9cdb29a97..4e2761d8a7d046 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 4ac820a432b9b0..93584627a291f7 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -333,12 +333,12 @@ function. The method table must be referenced in the module definition structure:: static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods + PyModuleDef_HEAD_INIT, + "spam", /* name of module */ + spam_doc, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + SpamMethods }; This structure, in turn, must be passed to the interpreter in the module's diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index b8ce4377877e70..abd5da9db65454 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -124,7 +124,7 @@ our objects and in some error messages, for example:: >>> "" + noddy.new_noddy() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: cannot add type "noddy.Noddy" to string Note that the name is a dotted name that includes both the module name and the @@ -728,8 +728,9 @@ functions. With :c:func:`Py_VISIT`, :c:func:`Noddy_traverse` can be simplified: uniformity across these boring implementations. We also need to provide a method for clearing any subobjects that can -participate in cycles. We implement the method and reimplement the deallocator -to use it:: +participate in cycles. + +:: static int Noddy_clear(Noddy *self) @@ -747,13 +748,6 @@ to use it:: return 0; } - static void - Noddy_dealloc(Noddy* self) - { - Noddy_clear(self); - Py_TYPE(self)->tp_free((PyObject*)self); - } - Notice the use of a temporary variable in :c:func:`Noddy_clear`. We use the temporary variable so that we can set each member to *NULL* before decrementing its reference count. We do this because, as was discussed earlier, if the @@ -776,6 +770,23 @@ be simplified:: return 0; } +Note that :c:func:`Noddy_dealloc` may call arbitrary functions through +``__del__`` method or weakref callback. It means circular GC can be +triggered inside the function. Since GC assumes reference count is not zero, +we need to untrack the object from GC by calling :c:func:`PyObject_GC_UnTrack` +before clearing members. Here is reimplemented deallocator which uses +:c:func:`PyObject_GC_UnTrack` and :c:func:`Noddy_clear`. + +:: + + static void + Noddy_dealloc(Noddy* self) + { + PyObject_GC_UnTrack(self); + Noddy_clear(self); + Py_TYPE(self)->tp_free((PyObject*)self); + } + Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index f1e33afdabf8a5..d4a97fd81af9e2 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -159,7 +159,7 @@ How do I obtain a copy of the Python source? The latest Python source distribution is always available from python.org, at https://www.python.org/downloads/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +at https://github.com/python/cpython/. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and @@ -167,7 +167,7 @@ several useful pieces of freely distributable software. The source will compile and run out of the box on most UNIX platforms. Consult the `Getting Started section of the Python Developer's Guide -`__ for more +`__ for more information on getting the source code and compiling it. @@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce newsgroups and on the Python home page at https://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Git. See +`The Python Developer's Guide `_ for details. How do I submit bug reports and patches for Python? @@ -239,7 +239,7 @@ report bugs to Python, you can obtain your Roundup password through Roundup's `password reset procedure `_. For more information on how Python is developed, consult `the Python Developer's -Guide `_. +Guide `_. Are there any published articles about Python that I can reference? diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 477d8c633438d1..38e1796267ff3a 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -94,15 +94,6 @@ Python bindings for `the FLTK toolkit `_, a simple yet powerful and mature cross-platform windowing system, are available from `the PyFLTK project `_. - -FOX ----- - -A wrapper for `the FOX toolkit `_ called `FXpy -`_ is available. FOX supports both Unix variants -and Windows. - - OpenGL ------ diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9c5e20dcadf5a0..7476ce11f4f141 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1638,7 +1638,7 @@ collected. Despite the cycle collector, it's still a good idea to define an explicit ``close()`` method on objects to be called whenever you're done with them. The -``close()`` method can then remove attributes that refer to subobjecs. Don't +``close()`` method can then remove attributes that refer to subobjects. Don't call :meth:`__del__` directly -- :meth:`__del__` should call ``close()`` and ``close()`` should make sure that it can be called more than once for the same object. diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d7253436beaf61..6ac83e45d2e815 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -300,9 +300,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools --> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent size" to 4, and select the "Insert spaces" radio button. -If you suspect mixed tabs and spaces are causing problems in leading whitespace, -run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to -check a directory tree in batch mode. +Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs +and spaces are causing problems in leading whitespace. +You may also run the :mod:`tabnanny` module to check a directory tree +in batch mode. How do I check for a keypress without blocking? diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d83b311fa..b947520b96b665 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -155,7 +159,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to @@ -316,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending @@ -458,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -526,7 +535,10 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`__iter__` or :meth:`__getitem__` method. Iterables can be + with an :meth:`__iter__` method or with a :meth:`__getitem__` method + that implements :term:`Sequence` semantics. + + Iterables can be used in a :keyword:`for` loop and in many other places where a sequence is needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed as an argument to the built-in function :func:`iter`, it returns an @@ -966,6 +978,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 7e161a59add8ae..9d770f5232b440 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -221,7 +221,7 @@ before proceeding. Introducing Optional arguments ============================== -So far we, have been playing with positional arguments. Let us +So far we have been playing with positional arguments. Let us have a look on how to add optional ones:: import argparse diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index eaab20ad975548..083a299cb32a56 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1407,8 +1407,8 @@ Let's start with defining some terminology: ``two-pass`` A buffer like ``buffer``. However, a two-pass buffer can only - be written once, and it prints out all text sent to it during - all of processing, even from Clinic blocks *after* the + be dumped once, and it prints out all text sent to it during + all processing, even from Clinic blocks *after* the dumping point. ``suppress`` The text is suppressed—thrown away. @@ -1471,7 +1471,7 @@ preset configurations, as follows: The default filename is ``"{dirname}/clinic/{basename}.h"``. ``buffer`` - Save up all most of the output from Clinic, to be written into + Save up most of the output from Clinic, to be written into your file near the end. For Python files implementing modules or builtin types, it's recommended that you dump the buffer just above the static structures for your module or diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 188a5cf2231b87..1d3bfb87dc0474 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -65,7 +65,7 @@ and full support for mouse and keyboard input. The Python curses module ------------------------ -Thy Python module is a fairly simple wrapper over the C functions provided by +The Python module is a fairly simple wrapper over the C functions provided by curses; if you're already familiar with curses programming in C, it's really easy to transfer that knowledge to Python. The biggest difference is that the Python interface makes things simpler by merging different C functions such as @@ -538,7 +538,7 @@ the Python interface. Often this isn't because they're difficult to implement, but because no one has needed them yet. Also, Python doesn't yet support the menu library associated with ncurses. Patches adding support for these would be welcome; see -`the Python Developer's Guide `_ to +`the Python Developer's Guide `_ to learn more about submitting patches to Python. * `Writing Programs with NCURSES `_: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index c2bf473e1ff9ea..5e85a9aa6594e4 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -180,7 +180,7 @@ descriptor is useful for monitoring just a few chosen attributes:: The protocol is simple and offers exciting possibilities. Several use cases are so common that they have been packaged into individual function calls. -Properties, bound and unbound methods, static methods, and class methods are all +Properties, bound methods, static methods, and class methods are all based on the descriptor protocol. @@ -252,10 +252,10 @@ to wrap access to the value attribute in a property data descriptor:: class Cell(object): . . . - def getvalue(self, obj): - "Recalculate cell before returning value" + def getvalue(self): + "Recalculate the cell before returning value" self.recalc() - return obj._value + return self._value value = property(getvalue) @@ -266,23 +266,24 @@ Python's object oriented features are built upon a function based environment. Using non-data descriptors, the two are merged seamlessly. Class dictionaries store methods as functions. In a class definition, methods -are written using :keyword:`def` and :keyword:`lambda`, the usual tools for -creating functions. The only difference from regular functions is that the +are written using :keyword:`def` or :keyword:`lambda`, the usual tools for +creating functions. Methods only differ from regular functions in that the first argument is reserved for the object instance. By Python convention, the instance reference is called *self* but may be called *this* or any other variable name. To support method calls, functions include the :meth:`__get__` method for binding methods during attribute access. This means that all functions are -non-data descriptors which return bound or unbound methods depending whether -they are invoked from an object or a class. In pure python, it works like -this:: +non-data descriptors which return bound methods when they are invoked from an +object. In pure python, it works like this:: class Function(object): . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" - return types.MethodType(self, obj, objtype) + if obj is None: + return self + return types.MethodType(self, obj) Running the interpreter shows how the function descriptor works in practice:: @@ -291,25 +292,34 @@ Running the interpreter shows how the function descriptor works in practice:: ... return x ... >>> d = D() - >>> D.__dict__['f'] # Stored internally as a function - - >>> D.f # Get from a class becomes an unbound method - - >>> d.f # Get from an instance becomes a bound method + + # Access through the class dictionary does not invoke __get__. + # It just returns the underlying function object. + >>> D.__dict__['f'] + + + # Dotted access from a class calls __get__() which just returns + # the underlying function unchanged. + >>> D.f + + + # The function has a __qualname__ attribute to support introspection + >>> D.f.__qualname__ + 'D.f' + + # Dotted access from an instance calls __get__() which returns the + # function wrapped in a bound method object + >>> d.f > -The output suggests that bound and unbound methods are two different types. -While they could have been implemented that way, the actual C implementation of -:c:type:`PyMethod_Type` in :source:`Objects/classobject.c` is a single object -with two different representations depending on whether the :attr:`im_self` -field is set or is *NULL* (the C equivalent of ``None``). - -Likewise, the effects of calling a method object depend on the :attr:`im_self` -field. If set (meaning bound), the original function (stored in the -:attr:`im_func` field) is called as expected with the first argument set to the -instance. If unbound, all of the arguments are passed unchanged to the original -function. The actual C implementation of :func:`instancemethod_call()` is only -slightly more complex in that it includes some type checking. + # Internally, the bound method stores the underlying function, + # the bound instance, and the class of the bound instance. + >>> d.f.__func__ + + >>> d.f.__self__ + <__main__.D object at 0x1012e1f98> + >>> d.f.__class__ + Static Methods and Class Methods diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 8ae9679894a578..40601812a77cb5 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -210,7 +210,7 @@ You can experiment with the iteration interface manually: 3 >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in StopIteration >>> @@ -474,7 +474,7 @@ Here's a sample usage of the ``generate_ints()`` generator: 2 >>> next(gen) Traceback (most recent call last): - File "stdin", line 1, in ? + File "stdin", line 1, in File "stdin", line 2, in generate_ints StopIteration @@ -577,7 +577,7 @@ And here's an example of changing the counter: 9 >>> next(it) #doctest: +SKIP Traceback (most recent call last): - File "t.py", line 15, in ? + File "t.py", line 15, in it.next() StopIteration @@ -653,8 +653,9 @@ This can also be written as a list comprehension: [0, 2, 4, 6, 8] -:func:`enumerate(iter) ` counts off the elements in the iterable, -returning 2-tuples containing the count and each element. :: +:func:`enumerate(iter, start=0) ` counts off the elements in the +iterable returning 2-tuples containing the count (from *start*) and +each element. :: >>> for item in enumerate(['subject', 'verb', 'object']): ... print(item) @@ -747,14 +748,16 @@ The module's functions fall into a few broad classes: Creating new iterators ---------------------- -:func:`itertools.count(n) ` returns an infinite stream of -integers, increasing by 1 each time. You can optionally supply the starting -number, which defaults to 0:: +:func:`itertools.count(start, step) ` returns an infinite +stream of evenly spaced values. You can optionally supply the starting number, +which defaults to 0, and the interval between numbers, which defaults to 1:: itertools.count() => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... itertools.count(10) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... + itertools.count(10, 5) => + 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... :func:`itertools.cycle(iter) ` saves a copy of the contents of a provided iterable and returns a new iterator that returns its elements from @@ -1060,10 +1063,10 @@ write the obvious :keyword:`for` loop:: for i in [1,2,3]: product *= i -A related function is `itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of +returning only the final result, :func:`accumulate` returns an iterator that +also yields each partial result:: itertools.accumulate([1,2,3,4,5]) => 1, 3, 6, 10, 15 @@ -1235,6 +1238,8 @@ Python documentation Documentation for the :mod:`itertools` module. +Documentation for the :mod:`functools` module. + Documentation for the :mod:`operator` module. :pep:`289`: "Generator Expressions" diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index bb79bb1748fb9c..44f38e0341c16d 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1258,8 +1258,8 @@ socket is created separately and passed to the handler (as its 'queue'):: class ZeroMQSocketHandler(QueueHandler): def enqueue(self, record): - data = json.dumps(record.__dict__) - self.queue.send(data) + self.queue.send_json(record.__dict__) + handler = ZeroMQSocketHandler(sock) @@ -1272,11 +1272,10 @@ data needed by the handler to create the socket:: self.ctx = ctx or zmq.Context() socket = zmq.Socket(self.ctx, socktype) socket.bind(uri) - QueueHandler.__init__(self, socket) + super().__init__(socket) def enqueue(self, record): - data = json.dumps(record.__dict__) - self.queue.send(data) + self.queue.send_json(record.__dict__) def close(self): self.queue.close() @@ -1292,12 +1291,13 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: def __init__(self, uri, *handlers, **kwargs): self.ctx = kwargs.get('ctx') or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) - socket.setsockopt(zmq.SUBSCRIBE, '') # subscribe to everything + socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything socket.connect(uri) + super().__init__(socket, *handlers, **kwargs) def dequeue(self): - msg = self.queue.recv() - return logging.makeLogRecord(json.loads(msg)) + msg = self.queue.recv_json() + return logging.makeLogRecord(msg) .. seealso:: @@ -1683,7 +1683,7 @@ Implementing structured logging ------------------------------- Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be cirumstances where you want to output +readily machine-parseable, there might be circumstances where you want to output messages in a structured format which *is* capable of being parsed by a program (without needing complex regular expressions to parse the log message). This is straightforward to achieve using the logging package. There are a number of diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index a48ae1f5faba7e..9649b9c609c255 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -43,9 +43,9 @@ hold values ranging from 0 to 255. ASCII codes only went up to 127, so some machines assigned values between 128 and 255 to accented characters. Different machines had different codes, however, which led to problems exchanging files. Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Standards Organization, -and some were *de facto* conventions that were invented by one company or -another and managed to catch on. +Some were true standards, defined by the International Organization for +Standardization, and some were *de facto* conventions that were invented by one +company or another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 18b5c6556be1c9..8d383e03ee8a28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -34,8 +34,8 @@ handling common situations - like basic authentication, cookies, proxies and so on. These are provided by objects called handlers and openers. urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ":" in URL - for example "ftp" is the URL scheme of -"ftp://python.org/") using their associated network protocols (e.g. FTP, HTTP). +before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of +``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). This tutorial focuses on the most common case, HTTP. For straightforward situations *urlopen* is very easy to use. But as soon as you @@ -511,10 +511,10 @@ than the URL you pass to .add_password() will also match. :: ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) -e.g. "http://example.com/" *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. "example.com" or "example.com:8080" +e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, +optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` (the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example "joe:password@example.com" is +NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is not correct. diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py index 2e142b1e3b71e4..df7ca6f3faa332 100644 --- a/Doc/includes/email-alternative.py +++ b/Doc/includes/email-alternative.py @@ -32,7 +32,7 @@

Salut!

Cela ressemble à un excellent - recipie déjeuner.

diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c index eb9622a87d99cc..08ba4c3d91a030 100644 --- a/Doc/includes/noddy4.c +++ b/Doc/includes/noddy4.c @@ -46,6 +46,7 @@ Noddy_clear(Noddy *self) static void Noddy_dealloc(Noddy* self) { + PyObject_GC_UnTrack(self); Noddy_clear(self); Py_TYPE(self)->tp_free((PyObject*)self); } diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py index b853d23b170985..a38a39de3e7c86 100644 --- a/Doc/includes/setup.py +++ b/Doc/includes/setup.py @@ -5,4 +5,5 @@ Extension("noddy2", ["noddy2.c"]), Extension("noddy3", ["noddy3.c"]), Extension("noddy4", ["noddy4.c"]), + Extension("shoddy", ["shoddy.c"]), ]) diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 07a272124ceaba..0c6d412b3c4cda 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -31,7 +31,7 @@ Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) static PyTypeObject ShoddyType = { - PyObject_HEAD_INIT(NULL) + PyVarObject_HEAD_INIT(NULL, 0) "shoddy.Shoddy", /* tp_name */ sizeof(Shoddy), /* tp_basicsize */ 0, /* tp_itemsize */ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index b22465df2910eb..09bb8251c35d58 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -211,6 +211,17 @@ On such systems, it is often better to use a virtual environment or a per-user installation when installing packages with ``pip``. +Pip not installed +----------------- + +It is possible that ``pip`` does not get installed by default. One potential fix is:: + + python -m ensurepip --default-pip + +There are also additional resources for `installing pip. +`__ + + Installing binary extensions ---------------------------- diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index ace1bfaf8cb9d6..4c9a528d42e703 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -199,13 +199,6 @@ and off individually. They are described here in more detail. because the :class:`memoryview` API is similar but not exactly the same as that of :class:`buffer`. -.. 2to3fixer:: callable - - Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding - an import to :mod:`collections` if needed. Note ``callable(x)`` has returned - in Python 3.2, so if you do not intend to support Python 3.1, you can disable - this fixer. - .. 2to3fixer:: dict Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 966003bd45ada1..6001db32df49c3 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -24,7 +24,33 @@ a class or instance provides a particular interface, for example, is it hashable or a mapping. -This module provides the following classes: +This module provides the metaclass :class:`ABCMeta` for defining ABCs and +a helper class :class:`ABC` to alternatively define ABCs through inheritance: + +.. class:: ABC + + A helper class that has :class:`ABCMeta` as its metaclass. With this class, + an abstract base class can be created by simply deriving from :class:`ABC` + avoiding sometimes confusing metaclass usage, for example:: + + from abc import ABC + + class MyABC(ABC): + pass + + Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore + inheriting from :class:`ABC` requires the usual precautions regarding + metaclass usage, as multiple inheritance may lead to metaclass conflicts. + One may also define an abstract base class by passing the metaclass + keyword and using :class:`ABCMeta` directly, for example:: + + from abc import ABCMeta + + class MyABC(metaclass=ABCMeta): + pass + + .. versionadded:: 3.4 + .. class:: ABCMeta @@ -46,15 +72,15 @@ This module provides the following classes: Register *subclass* as a "virtual subclass" of this ABC. For example:: - from abc import ABCMeta + from abc import ABC - class MyABC(metaclass=ABCMeta): - pass + class MyABC(ABC): + pass - MyABC.register(tuple) + MyABC.register(tuple) - assert issubclass(tuple, MyABC) - assert isinstance((), MyABC) + assert issubclass(tuple, MyABC) + assert isinstance((), MyABC) .. versionchanged:: 3.3 Returns the registered subclass, to allow usage as a class decorator. @@ -95,7 +121,7 @@ This module provides the following classes: def get_iterator(self): return iter(self) - class MyIterable(metaclass=ABCMeta): + class MyIterable(ABC): @abstractmethod def __iter__(self): @@ -132,17 +158,6 @@ This module provides the following classes: available as a method of ``Foo``, so it is provided separately. -.. class:: ABC - - A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC`, - avoiding sometimes confusing metaclass usage. - - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding metaclass - usage, as multiple inheritance may lead to metaclass conflicts. - - .. versionadded:: 3.4 The :mod:`abc` module also provides the following decorators: @@ -168,7 +183,7 @@ The :mod:`abc` module also provides the following decorators: descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: - class C(metaclass=ABCMeta): + class C(ABC): @abstractmethod def my_abstract_method(self, ...): ... @@ -230,7 +245,7 @@ The :mod:`abc` module also provides the following decorators: is now correctly identified as abstract when applied to an abstract method:: - class C(metaclass=ABCMeta): + class C(ABC): @classmethod @abstractmethod def my_abstract_classmethod(cls, ...): @@ -251,7 +266,7 @@ The :mod:`abc` module also provides the following decorators: is now correctly identified as abstract when applied to an abstract method:: - class C(metaclass=ABCMeta): + class C(ABC): @staticmethod @abstractmethod def my_abstract_staticmethod(...): @@ -278,7 +293,7 @@ The :mod:`abc` module also provides the following decorators: is now correctly identified as abstract when applied to an abstract method:: - class C(metaclass=ABCMeta): + class C(ABC): @property @abstractmethod def my_abstract_property(self): @@ -288,7 +303,7 @@ The :mod:`abc` module also provides the following decorators: read-write abstract property by appropriately marking one or more of the underlying methods as abstract:: - class C(metaclass=ABCMeta): + class C(ABC): @property def x(self): ... diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index 23a96207d08389..970a7aeb98a713 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -18,12 +18,6 @@ AIFF is Audio Interchange File Format, a format for storing digital audio samples in a file. AIFF-C is a newer version of the format that includes the ability to compress the audio data. -.. note:: - - Some operations may only work under IRIX; these will raise :exc:`ImportError` - when attempting to import the :mod:`cl` module, which is only available on - IRIX. - Audio files have a number of parameters that describe the audio data. The sampling rate or frame rate is the number of times per second the sound is sampled. The number of channels indicate if the audio is mono, stereo, or diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 45303048174dfa..27c7ba55820486 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -426,7 +426,9 @@ should not be line-wrapped:: -h, --help show this help message and exit :class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. +including argument descriptions. However, multiple new lines are replaced with +one. If you wish to preserve multiple blank lines, add spaces between the +newlines. :class:`ArgumentDefaultsHelpFormatter` automatically adds information about default values to each of the argument help messages:: @@ -898,6 +900,8 @@ values are: usage: PROG [-h] foo [foo ...] PROG: error: too few arguments +.. _`argparse.REMAINDER`: + * ``argparse.REMAINDER``. All the remaining command-line arguments are gathered into a list. This is commonly useful for command line utilities that dispatch to other command line utilities:: @@ -1324,8 +1328,11 @@ The parse_args() method created and how they are assigned. See the documentation for :meth:`add_argument` for details. - By default, the argument strings are taken from :data:`sys.argv`, and a new empty - :class:`Namespace` object is created for the attributes. + * args_ - List of strings to parse. The default is taken from + :data:`sys.argv`. + + * namespace_ - An object to take the attributes. The default is a new empty + :class:`Namespace` object. Option value syntax @@ -1467,6 +1474,7 @@ unambiguous (the prefix matches a unique option):: An error is produced for arguments that could produce more than one options. This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. +.. _args: Beyond ``sys.argv`` ^^^^^^^^^^^^^^^^^^^ @@ -1488,6 +1496,7 @@ interactive prompt:: >>> parser.parse_args(['1', '2', '3', '4', '--sum']) Namespace(accumulate=, integers=[1, 2, 3, 4]) +.. _namespace: The Namespace object ^^^^^^^^^^^^^^^^^^^^ @@ -2008,7 +2017,12 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`: * Replace ``(options, args) = parser.parse_args()`` with ``args = parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in :mod:`argparse` context is called ``args``. + called ``options``, now in the :mod:`argparse` context is called ``args``. + +* Replace :meth:`optparse.OptionParser.disable_interspersed_args` + by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or + use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument + strings in a separate list. * Replace callback actions and the ``callback_*`` keyword arguments with ``type`` or ``action`` arguments. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index cc4a14b2016226..1838eb95a7d522 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -52,7 +52,7 @@ Cancellation ------------ Cancellation of tasks is not common in classic programming. In asynchronous -programming, not only it is something common, but you have to prepare your +programming, not only is it something common, but you have to prepare your code to handle it. Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel` diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index fa6a29604ca2ba..83bbb70b037928 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -5,6 +5,8 @@ Base Event Loop =============== +**Source code:** :source:`Lib/asyncio/events.py` + The event loop is the central execution device provided by :mod:`asyncio`. It provides multiple facilities, including: diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst index 1dc18fce79d021..d74fcb1e07f249 100644 --- a/Doc/library/asyncio-eventloops.rst +++ b/Doc/library/asyncio-eventloops.rst @@ -3,6 +3,8 @@ Event loops =========== +**Source code:** :source:`Lib/asyncio/events.py` + Event loop functions -------------------- diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3fbf51058673d9..cd84ae76b5d86e 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1,8 +1,12 @@ .. currentmodule:: asyncio -++++++++++++++++++++++++++++++++++++++++++++++ -Transports and protocols (callback based API) -++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++ +Transports and protocols (callback based API) ++++++++++++++++++++++++++++++++++++++++++++++ + +**Source code:** :source:`Lib/asyncio/transports.py` + +**Source code:** :source:`Lib/asyncio/protocols.py` .. _asyncio-transport: @@ -36,7 +40,7 @@ BaseTransport Base class for transports. - .. method:: close(self) + .. method:: close() Close the transport. If the transport has a buffer for outgoing data, buffered data will be flushed asynchronously. No more data @@ -44,7 +48,7 @@ BaseTransport protocol's :meth:`connection_lost` method will be called with :const:`None` as its argument. - .. method:: is_closing(self) + .. method:: is_closing() Return ``True`` if the transport is closing or is closed. @@ -163,11 +167,17 @@ WriteTransport Set the *high*- and *low*-water limits for write flow control. - These two values control when call the protocol's + These two values (measured in number of + bytes) control when the protocol's :meth:`pause_writing` and :meth:`resume_writing` methods are called. If specified, the low-water limit must be less than or equal to the high-water limit. Neither *high* nor *low* can be negative. + :meth:`pause_writing` is called when the buffer size becomes greater + than or equal to the *high* value. If writing has been paused, + :meth:`resume_writing` is called when the buffer size becomes less + than or equal to the *low* value. + The defaults are implementation-specific. If only the high-water limit is given, the low-water limit defaults to an implementation-specific value less than or equal to the @@ -251,7 +261,7 @@ BaseSubprocessTransport if it hasn't returned, similarly to the :attr:`subprocess.Popen.returncode` attribute. - .. method:: kill(self) + .. method:: kill() Kill the subprocess, as in :meth:`subprocess.Popen.kill`. @@ -384,7 +394,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index f11c09ac29055e..ea78755008244d 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -3,6 +3,8 @@ Queues ====== +**Source code:** :source:`Lib/asyncio/queues.py` + Queues: * :class:`Queue` diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 6177b4bb0f8b14..491afdd610ca08 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -6,6 +6,8 @@ Streams (coroutine based API) +++++++++++++++++++++++++++++ +**Source code:** :source:`Lib/asyncio/streams.py` + Stream functions ================ diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74c6dee12..1c1d0be918da74 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -5,6 +5,8 @@ Subprocess ========== +**Source code:** :source:`Lib/asyncio/subprocess.py` + Windows event loop ------------------ @@ -80,7 +82,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +125,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 09093521524a89..14e3defbf4118a 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -4,6 +4,8 @@ Synchronization primitives ========================== +**Source code:** :source:`Lib/asyncio/locks.py` + Locks: * :class:`Lock` diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 558d17c0969793..5298c11058c4b1 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -3,6 +3,10 @@ Tasks and coroutines ==================== +**Source code:** :source:`Lib/asyncio/tasks.py` + +**Source code:** :source:`Lib/asyncio/coroutines.py` + .. _coroutine: Coroutines @@ -540,6 +544,11 @@ Task functions .. deprecated:: 3.4.4 +.. function:: wrap_future(future, \*, loop=None) + + Wrap a :class:`concurrent.futures.Future` object in a :class:`Future` + object. + .. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) Return a future aggregating results from the given coroutine objects or diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 080d9d77ec984e..ceecf17cba23e3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -237,14 +237,18 @@ The legacy interface: .. function:: decodebytes(s) - decodestring(s) Decode the :term:`bytes-like object` *s*, which must contain one or more lines of base64 encoded data, and return the decoded :class:`bytes`. - ``decodestring`` is a deprecated alias. .. versionadded:: 3.1 +.. function:: decodestring(s) + + Deprecated alias of :func:`decodebytes`. + + .. deprecated:: 3.1 + .. function:: encode(input, output) @@ -257,14 +261,19 @@ The legacy interface: .. function:: encodebytes(s) - encodestring(s) Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary data, and return :class:`bytes` containing the base64-encoded data, with newlines (``b'\n'``) inserted after every 76 bytes of output, and ensuring that there is a trailing newline, as per :rfc:`2045` (MIME). - ``encodestring`` is a deprecated alias. + .. versionadded:: 3.1 + +.. function:: encodestring(s) + + Deprecated alias of :func:`encodebytes`. + + .. deprecated:: 3.1 An example usage of the module: diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23b2f9787..2966e0dbfbcfe8 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index f40cfdfd592163..3b4a8ff440e7ae 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -266,10 +266,10 @@ immediate playback:: 'Draw circle with given radius an options extent and steps: CIRCLE 50' circle(*parse(arg)) def do_position(self, arg): - 'Print the current turle position: POSITION' + 'Print the current turtle position: POSITION' print('Current position is %d %d\n' % position()) def do_heading(self, arg): - 'Print the current turle heading in degrees: HEADING' + 'Print the current turtle heading in degrees: HEADING' print('Current heading is %d\n' % (heading(),)) def do_color(self, arg): 'Set the color: COLOR BLUE' diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 0ecd89ba75b617..b9c1868d5956d1 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -554,19 +554,19 @@ define in order to be compatible with the Python codec registry. if necessary, to reset the encoder and to get the output. -.. method:: IncrementalEncoder.getstate() + .. method:: getstate() - Return the current state of the encoder which must be an integer. The - implementation should make sure that ``0`` is the most common state. (States - that are more complicated than integers can be converted into an integer by - marshaling/pickling the state and encoding the bytes of the resulting string - into an integer). + Return the current state of the encoder which must be an integer. The + implementation should make sure that ``0`` is the most common + state. (States that are more complicated than integers can be converted + into an integer by marshaling/pickling the state and encoding the bytes + of the resulting string into an integer). -.. method:: IncrementalEncoder.setstate(state) + .. method:: setstate(state) - Set the state of the encoder to *state*. *state* must be an encoder state - returned by :meth:`getstate`. + Set the state of the encoder to *state*. *state* must be an encoder state + returned by :meth:`getstate`. .. _incremental-decoder-objects: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 58b03b9bd76804..60154532094423 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -107,7 +107,12 @@ ABC Inherits from Abstract Methods Mixin .. class:: Iterable ABC for classes that provide the :meth:`__iter__` method. - See also the definition of :term:`iterable`. + + Checking ``isinstance(obj, Iterable)`` detects classes that are registered + as :class:`Iterable` or that have an :meth:`__iter__` method, but it does + not detect classes that iterate with the :meth:`__getitem__` method. + The only reliable way to determine whether an object is :term:`iterable` + is to call ``iter(obj)``. .. class:: Collection diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 8e2eb4d9b0382f..d6d2056dfc496c 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -771,9 +771,9 @@ they add the ability to access fields by name instead of position index. helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` method which lists the tuple contents in a ``name=value`` format. - The *field_names* are a single string with each fieldname separated by whitespace - and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* - can be a sequence of strings such as ``['x', 'y']``. + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. Any valid Python identifier may be used for a fieldname except for names starting with an underscore. Valid identifiers consist of letters, digits, @@ -866,7 +866,7 @@ field names, the method and attribute names start with an underscore. .. versionchanged:: 3.1 Returns an :class:`OrderedDict` instead of a regular :class:`dict`. -.. method:: somenamedtuple._replace(kwargs) +.. method:: somenamedtuple._replace(**kwargs) Return a new instance of the named tuple replacing specified fields with new values:: diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e09562dc9046c7..04c2a820921759 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also @@ -983,13 +983,16 @@ ConfigParser Objects .. method:: read(filenames, encoding=None) Attempt to read and parse a list of filenames, returning a list of - filenames which were successfully parsed. If *filenames* is a string, it - is treated as a single filename. If a file named in *filenames* cannot - be opened, that file will be ignored. This is designed so that you can - specify a list of potential configuration file locations (for example, - the current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the list will be - read. If none of the named files exist, the :class:`ConfigParser` + filenames which were successfully parsed. + + If *filenames* is a string or :term:`path-like object`, it is treated as + a single filename. If a file named in *filenames* cannot be opened, that + file will be ignored. This is designed so that you can specify a list of + potential configuration file locations (for example, the current + directory, the user's home directory, and some system-wide directory), + and all existing configuration files in the list will be read. + + If none of the named files exist, the :class:`ConfigParser` instance will contain an empty dataset. An application which requires initial values to be loaded from a file should load the required file or files using :meth:`read_file` before calling :meth:`read` for any @@ -1006,6 +1009,9 @@ ConfigParser Objects The *encoding* parameter. Previously, all files were read using the default encoding for :func:`open`. + .. versionadded:: 3.6.1 + The *filenames* parameter accepts a :term:`path-like object`. + .. method:: read_file(f, source=None) diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742cee55bd55..469a3eed606ff0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d469bc05..2041d9175ea587 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 52a8a310ec9d62..43714f7479283d 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -401,8 +401,10 @@ Reader objects (:class:`DictReader` instances and objects returned by the .. method:: csvreader.__next__() - Return the next row of the reader's iterable object as a list, parsed according - to the current dialect. Usually you should call this as ``next(reader)``. + Return the next row of the reader's iterable object as a list (if the object + was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` + instance), parsed according to the current dialect. Usually you should call + this as ``next(reader)``. Reader objects have the following public attributes: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3840935ce04eb6..cdcbefa4e8084a 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -97,7 +97,7 @@ Functions are accessed as attributes of dll objects:: <_FuncPtr object at 0x...> >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 239, in __getattr__ func = _StdcallFuncPtr(name, self) AttributeError: function 'MyOwnFunction' not found @@ -135,7 +135,7 @@ functions can be accessed by indexing the dll object with the ordinal number:: <_FuncPtr object at 0x...> >>> cdll.kernel32[0] # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 310, in __getitem__ func = _StdcallFuncPtr(name, self) AttributeError: function ordinal 0 not found @@ -161,33 +161,25 @@ as the NULL pointer):: 0x1d000000 >>> -:mod:`ctypes` tries to protect you from calling functions with the wrong number -of arguments or the wrong calling convention. Unfortunately this only works on -Windows. It does this by examining the stack after the function returns, so -although an error is raised the function *has* been called:: +.. note:: - >>> windll.kernel32.GetModuleHandleA() # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> windll.kernel32.GetModuleHandleA(0, 0) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> + :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if + it detects that an invalid number of arguments were passed. This behavior + should not be relied upon. It is deprecated in 3.6.2, and will be removed + in 3.7. -The same exception is raised when you call an ``stdcall`` function with the +:exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with not enough arguments (4 bytes missing) >>> >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with too many arguments (4 bytes in excess) >>> @@ -200,7 +192,7 @@ argument values:: >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in OSError: exception: access violation reading 0x00000020 >>> @@ -284,7 +276,7 @@ the correct type and value:: >>> c_int() c_long(0) >>> c_wchar_p("Hello, World") - c_wchar_p('Hello, World') + c_wchar_p(140018365411392) >>> c_ushort(-3) c_ushort(65533) >>> @@ -309,11 +301,15 @@ bytes objects are immutable):: >>> s = "Hello, World" >>> c_s = c_wchar_p(s) >>> print(c_s) - c_wchar_p('Hello, World') + c_wchar_p(139966785747344) + >>> print(c_s.value) + Hello World >>> c_s.value = "Hi, there" - >>> print(c_s) - c_wchar_p('Hi, there') - >>> print(s) # first object is unchanged + >>> print(c_s) # the memory location has changed + c_wchar_p(139966783348904) + >>> print(c_s.value) + Hi, there + >>> print(s) # first object is unchanged Hello, World >>> @@ -369,7 +365,7 @@ from within *IDLE* or *PythonWin*:: 19 >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 >>> @@ -432,7 +428,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: wrong type >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 @@ -482,7 +478,7 @@ single character Python bytes object into a C char:: 'def' >>> strchr(b"abcdef", b"def") Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: one character string expected >>> print(strchr(b"abcdef", b"x")) None @@ -508,7 +504,7 @@ useful to check for error return values and automatically raise an exception:: 486539264 >>> GetModuleHandle("something silly") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 3, in ValidHandle OSError: [Errno 126] The specified module could not be found. >>> @@ -579,7 +575,7 @@ Here is a simple example of a POINT structure, which contains two integers named 0 5 >>> POINT(1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: too many initializers >>> @@ -782,7 +778,7 @@ new type:: >>> PI(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: expected c_long instead of int >>> PI(c_int(42)) @@ -858,7 +854,7 @@ but not instances of other types:: >>> bar.values = (c_byte * 4)() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance >>> @@ -909,7 +905,7 @@ work:: ... ("next", POINTER(cell))] ... Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 2, in cell NameError: name 'cell' is not defined >>> diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index b6ac2517335cf5..04b2a260af0946 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -176,14 +176,12 @@ C library: Checks for a non-ASCII character (ordinal values 0x80 and above). -These functions accept either integers or strings; when the argument is a +These functions accept either integers or single-character strings; when the argument is a string, it is first converted using the built-in function :func:`ord`. -Note that all these functions check ordinal bit values derived from the first +Note that all these functions check ordinal bit values derived from the character of the string you pass in; they do not actually know anything about -the host machine's character encoding. For functions that know about the -character encoding (and handle internationalization properly) see the -:mod:`string` module. +the host machine's character encoding. The following two functions take either a single-character string or integer byte value; they return a value of the same type. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index c99c37a5651bad..d770c03c8375f4 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -74,7 +74,7 @@ Panel objects have the following methods: .. method:: Panel.hidden() - Returns true if the panel is hidden (not visible), false otherwise. + Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. .. method:: Panel.hide() diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eafaea9b5c..3697407f5c4995 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -19,6 +19,14 @@ for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. +.. note:: + + Whenever the documentation mentions a *character* it can be specified + as an integer, a one-character Unicode string or a one-byte byte string. + + Whenever the documentation mentions a *character string* it can be specified + as a Unicode string or a byte string. + .. note:: Since version 5.4, the ncurses library decides how to interpret non-ASCII data @@ -104,8 +112,8 @@ The module :mod:`curses` defines the following functions: .. function:: color_content(color_number) Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. A 3-tuple is - returned, containing the R,G,B values for the given color, which will be between + *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, + containing the R,G,B values for the given color, which will be between ``0`` (no component) and ``1000`` (maximum amount of component). @@ -119,9 +127,9 @@ The module :mod:`curses` defines the following functions: .. function:: curs_set(visibility) - Set the cursor state. *visibility* can be set to 0, 1, or 2, for invisible, - normal, or very visible. If the terminal supports the visibility requested, the - previous cursor state is returned; otherwise, an exception is raised. On many + Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, + normal, or very visible. If the terminal supports the visibility requested, return the + previous cursor state; otherwise raise an exception. On many terminals, the "visible" mode is an underline cursor and the "very visible" mode is a block cursor. @@ -154,12 +162,12 @@ The module :mod:`curses` defines the following functions: representing the desired next state. The :func:`doupdate` ground updates the physical screen to match the virtual screen. - The virtual screen may be updated by a :meth:`noutrefresh` call after write - operations such as :meth:`addstr` have been performed on a window. The normal - :meth:`refresh` call is simply :meth:`noutrefresh` followed by :func:`doupdate`; + The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write + operations such as :meth:`~window.addstr` have been performed on a window. The normal + :meth:`~window.refresh` call is simply :meth:`!noutrefresh` followed by :func:`!doupdate`; if you have to update multiple windows, you can speed performance and perhaps - reduce screen flicker by issuing :meth:`noutrefresh` calls on all windows, - followed by a single :func:`doupdate`. + reduce screen flicker by issuing :meth:`!noutrefresh` calls on all windows, + followed by a single :func:`!doupdate`. .. function:: echo() @@ -175,7 +183,7 @@ The module :mod:`curses` defines the following functions: .. function:: erasechar() - Return the user's current erase character. Under Unix operating systems this + Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this is a property of the controlling tty of the curses program, and is not set by the curses library itself. @@ -183,9 +191,9 @@ The module :mod:`curses` defines the following functions: .. function:: filter() The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to 1; the - capabilities clear, cup, cud, cud1, cuu1, cuu, vpa are disabled; and the home - string is set to the value of cr. The effect is that the cursor is confined to + called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the + capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` + string is set to the value of ``cr``. The effect is that the cursor is confined to the current line, and so are screen updates. This may be used for enabling character-at-a-time line editing without touching the rest of the screen. @@ -205,7 +213,7 @@ The module :mod:`curses` defines the following functions: .. function:: getmouse() - After :meth:`getch` returns :const:`KEY_MOUSE` to signal a mouse event, this + After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this method should be call to retrieve the queued mouse event, represented as a 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is @@ -219,8 +227,8 @@ The module :mod:`curses` defines the following functions: .. function:: getsyx() - Return the current coordinates of the virtual screen cursor in y and x. If - leaveok is currently true, then -1,-1 is returned. + Return the current coordinates of the virtual screen cursor as a tuple + ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. .. function:: getwin(file) @@ -260,7 +268,7 @@ The module :mod:`curses` defines the following functions: Used for half-delay mode, which is similar to cbreak mode in that characters typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, an exception is raised if nothing has + blocking for *tenths* tenths of seconds, raise an exception if nothing has been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use :func:`nocbreak` to leave half-delay mode. @@ -273,7 +281,7 @@ The module :mod:`curses` defines the following functions: :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and ``1000``. When :func:`init_color` is used, all occurrences of that color on the screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``1``. + most terminals; it is active only if :func:`can_change_color` returns ``True``. .. function:: init_pair(pair_number, fg, bg) @@ -290,8 +298,8 @@ The module :mod:`curses` defines the following functions: .. function:: initscr() - Initialize the library. Return a :class:`WindowObject` which represents the - whole screen. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. .. note:: @@ -313,32 +321,32 @@ The module :mod:`curses` defines the following functions: .. function:: keyname(k) - Return the name of the key numbered *k*. The name of a key generating printable + Return the name of the key numbered *k* as a bytes object. The name of a key generating printable ASCII character is the key's character. The name of a control-key combination - is a two-character string consisting of a caret followed by the corresponding + is a two-byte bytes object consisting of a caret (``b'^'``) followed by the corresponding printable ASCII character. The name of an alt-key combination (128--255) is a - string consisting of the prefix 'M-' followed by the name of the corresponding + bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. .. function:: killchar() - Return the user's current line kill character. Under Unix operating systems + Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems this is a property of the controlling tty of the curses program, and is not set by the curses library itself. .. function:: longname() - Return a string containing the terminfo long name field describing the current + Return a bytes object containing the terminfo long name field describing the current terminal. The maximum length of a verbose description is 128 characters. It is defined only after the call to :func:`initscr`. -.. function:: meta(yes) +.. function:: meta(flag) - If *yes* is 1, allow 8-bit characters to be input. If *yes* is 0, allow only - 7-bit chars. + If *flag* is ``True``, allow 8-bit characters to be input. If + *flag* is ``False``, allow only 7-bit chars. .. function:: mouseinterval(interval) @@ -352,7 +360,7 @@ The module :mod:`curses` defines the following functions: Set the mouse events to be reported, and return a tuple ``(availmask, oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns 0. *oldmask* is the previous value of + reported; on complete failure it returns ``0``. *oldmask* is the previous value of the given window's mouse event mask. If this function is never called, no mouse events are ever reported. @@ -365,13 +373,13 @@ The module :mod:`curses` defines the following functions: .. function:: newpad(nlines, ncols) Create and return a pointer to a new pad data structure with the given number - of lines and columns. A pad is returned as a window object. + of lines and columns. Return a pad as a window object. A pad is like a window, except that it is not restricted by the screen size, and is not necessarily associated with a particular part of the screen. Pads can be used when a large window is needed, and only a part of the window will be on the screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`refresh` and :meth:`noutrefresh` + echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` methods of a pad require 6 arguments to specify the part of the pad to be displayed and the location on the screen to be used for the display. The arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* @@ -383,8 +391,8 @@ The module :mod:`curses` defines the following functions: .. function:: newwin(nlines, ncols) newwin(nlines, ncols, begin_y, begin_x) - Return a new window, whose left-upper corner is at ``(begin_y, begin_x)``, and - whose height/width is *nlines*/*ncols*. + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. By default, the window will extend from the specified position to the lower right corner of the screen. @@ -419,9 +427,9 @@ The module :mod:`curses` defines the following functions: .. function:: noqiflush() - When the :func:`noqiflush` routine is used, normal flush of input and output queues - associated with the INTR, QUIT and SUSP characters will not be done. You may - want to call :func:`noqiflush` in a signal handler if you want output to + When the :func:`!noqiflush` routine is used, normal flush of input and output queues + associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may + want to call :func:`!noqiflush` in a signal handler if you want output to continue as though the interrupt had not occurred, after the handler exits. @@ -442,14 +450,14 @@ The module :mod:`curses` defines the following functions: :func:`color_pair` is the counterpart to this function. -.. function:: putp(string) +.. function:: putp(str) Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified terminfo capability for the current terminal. Note that the output of :func:`putp` always goes to standard output. -.. function:: qiflush( [flag] ) +.. function:: qiflush([flag]) If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If *flag* is ``True``, or no argument is provided, the queues will be flushed when @@ -486,7 +494,7 @@ The module :mod:`curses` defines the following functions: Backend function used by :func:`resizeterm`, performing most of the work; when resizing the windows, :func:`resize_term` blank-fills the areas that are extended. The calling application should fill in these areas with - appropriate data. The :func:`resize_term` function attempts to resize all + appropriate data. The :func:`!resize_term` function attempts to resize all windows. However, due to the calling convention of pads, it is not possible to resize these without additional interaction with the application. @@ -506,16 +514,17 @@ The module :mod:`curses` defines the following functions: .. function:: setsyx(y, x) - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both -1, then - leaveok is set. + Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then + :meth:`leaveok ` is set ``True``. -.. function:: setupterm([termstr, fd]) +.. function:: setupterm(term=None, fd=-1) - Initialize the terminal. *termstr* is a string giving the terminal name; if - omitted, the value of the :envvar:`TERM` environment variable will be used. *fd* is the + Initialize the terminal. *term* is a string giving + the terminal name, or ``None``; if omitted or ``None``, the value of the + :envvar:`TERM` environment variable will be used. *fd* is the file descriptor to which any initialization sequences will be sent; if not - supplied, the file descriptor for ``sys.stdout`` will be used. + supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. .. function:: start_color() @@ -540,13 +549,14 @@ The module :mod:`curses` defines the following functions: .. function:: termname() - Return the value of the environment variable :envvar:`TERM`, truncated to 14 characters. + Return the value of the environment variable :envvar:`TERM`, as a bytes object, + truncated to 14 characters. .. function:: tigetflag(capname) Return the value of the Boolean capability corresponding to the terminfo - capability name *capname*. The value ``-1`` is returned if *capname* is not a + capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a Boolean capability, or ``0`` if it is canceled or absent from the terminal description. @@ -554,7 +564,7 @@ The module :mod:`curses` defines the following functions: .. function:: tigetnum(capname) Return the value of the numeric capability corresponding to the terminfo - capability name *capname*. The value ``-2`` is returned if *capname* is not a + capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a numeric capability, or ``-1`` if it is canceled or absent from the terminal description. @@ -562,13 +572,14 @@ The module :mod:`curses` defines the following functions: .. function:: tigetstr(capname) Return the value of the string capability corresponding to the terminfo - capability name *capname*. ``None`` is returned if *capname* is not a string - capability, or is canceled or absent from the terminal description. + capability name *capname* as a bytes object. Return ``None`` if *capname* + is not a terminfo "string capability", or is canceled or absent from the + terminal description. .. function:: tparm(str[, ...]) - Instantiate the string *str* with the supplied parameters, where *str* should + Instantiate the bytes object *str* with the supplied parameters, where *str* should be a parameterized string obtained from the terminfo database. E.g. ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact result depending on terminal type. @@ -588,18 +599,18 @@ The module :mod:`curses` defines the following functions: .. function:: unctrl(ch) - Return a string which is a printable representation of the character *ch*. - Control characters are displayed as a caret followed by the character, for - example as ``^C``. Printing characters are left as they are. + Return a bytes object which is a printable representation of the character *ch*. + Control characters are represented as a caret followed by the character, for + example as ``b'^C'``. Printing characters are left as they are. .. function:: ungetch(ch) - Push *ch* so the next :meth:`getch` will return it. + Push *ch* so the next :meth:`~window.getch` will return it. .. note:: - Only one *ch* can be pushed before :meth:`getch` is called. + Only one *ch* can be pushed before :meth:`!getch` is called. .. function:: update_lines_cols() @@ -611,11 +622,11 @@ The module :mod:`curses` defines the following functions: .. function:: unget_wch(ch) - Push *ch* so the next :meth:`get_wch` will return it. + Push *ch* so the next :meth:`~window.get_wch` will return it. .. note:: - Only one *ch* can be pushed before :meth:`get_wch` is called. + Only one *ch* can be pushed before :meth:`!get_wch` is called. .. versionadded:: 3.3 @@ -640,7 +651,7 @@ The module :mod:`curses` defines the following functions: Allow use of default values for colors on terminals supporting this feature. Use this to support transparency in your application. The default color is assigned - to the color number -1. After calling this function, ``init_pair(x, + to the color number ``-1``. After calling this function, ``init_pair(x, curses.COLOR_RED, -1)`` initializes, for instance, color pair *x* to a red foreground color on the default background. @@ -652,7 +663,7 @@ The module :mod:`curses` defines the following functions: this function will restore the terminal to a sane state before re-raising the exception and generating a traceback. The callable object *func* is then passed the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`wrapper`. Before calling *func*, :func:`wrapper` turns on + passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on cbreak mode, turns off echo, enables the terminal keypad, and initializes colors if the terminal has color support. On exit (whether normally or by exception) it restores cooked mode, turns on echo, and disables the terminal keypad. @@ -670,13 +681,6 @@ the following methods and attributes: .. method:: window.addch(ch[, attr]) window.addch(y, x, ch[, attr]) - .. note:: - - A *character* means a C character (an ASCII code), rather than a Python - character (a string of length 1). (This note is true whenever the - documentation mentions a character.) The built-in :func:`ord` is handy for - conveying strings to codes. - Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any character previously painter at that location. By default, the character position and attributes are the current settings for the window object. @@ -685,15 +689,16 @@ the following methods and attributes: .. method:: window.addnstr(str, n[, attr]) window.addnstr(y, x, str, n[, attr]) - Paint at most *n* characters of the string *str* at ``(y, x)`` with attributes + Paint at most *n* characters of the character string *str* at + ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. .. method:: window.addstr(str[, attr]) window.addstr(y, x, str[, attr]) - Paint the string *str* at ``(y, x)`` with attributes *attr*, overwriting - anything previously on the display. + Paint the character string *str* at ``(y, x)`` with attributes + *attr*, overwriting anything previously on the display. .. method:: window.attroff(attr) @@ -710,8 +715,8 @@ the following methods and attributes: .. method:: window.attrset(attr) - Set the "background" set of attributes to *attr*. This set is initially 0 (no - attributes). + Set the "background" set of attributes to *attr*. This set is initially + ``0`` (no attributes). .. method:: window.bkgd(ch[, attr]) @@ -741,8 +746,7 @@ the following methods and attributes: Draw a border around the edges of the window. Each parameter specifies the character to use for a specific part of the border; see the table below for more - details. The characters can be specified as integers or as one-character - strings. + details. .. note:: @@ -783,7 +787,7 @@ the following methods and attributes: window.chgat(y, x, num, attr) Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If no value of *num* is given or *num* = -1, + position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, the attribute will be set on all the characters to the end of the line. This function does not move the cursor. The changed line will be touched using the :meth:`touchline` method so that the contents will be redisplayed by the next @@ -796,9 +800,9 @@ the following methods and attributes: call to :meth:`refresh`. -.. method:: window.clearok(yes) +.. method:: window.clearok(flag) - If *yes* is 1, the next call to :meth:`refresh` will clear the window + If *flag* is ``True``, the next call to :meth:`refresh` will clear the window completely. @@ -880,15 +884,16 @@ the following methods and attributes: .. method:: window.getch([y, x]) Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on return numbers higher than 256. In - no-delay mode, -1 is returned if there is no input, else :func:`getch` waits - until a key is pressed. + range: function keys, keypad keys and so on are represented by numbers higher + than 255. In no-delay mode, return ``-1`` if there is no input, otherwise + wait until a key is pressed. .. method:: window.get_wch([y, x]) Get a wide character. Return a character for most keys, or an integer for function keys, keypad keys, and other special keys. + In no-delay mode, raise an exception if there is no input. .. versionadded:: 3.3 @@ -897,7 +902,7 @@ the following methods and attributes: Get a character, returning a string instead of an integer, as :meth:`getch` does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, an exception is raised if + string containing the key name. In no-delay mode, raise an exception if there is no input. @@ -909,13 +914,16 @@ the following methods and attributes: .. method:: window.getparyx() Return the beginning coordinates of this window relative to its parent window - into two integer variables y and x. Return ``-1, -1`` if this window has no + as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no parent. -.. method:: window.getstr([y, x]) +.. method:: window.getstr() + window.getstr(n) + window.getstr(y, x) + window.getstr(y, x, n) - Read a string from the user, with primitive line editing capacity. + Read a bytes object from the user, with primitive line editing capacity. .. method:: window.getyx() @@ -939,9 +947,9 @@ the following methods and attributes: insert/delete is enabled by default. -.. method:: window.idlok(yes) +.. method:: window.idlok(flag) - If called with *yes* equal to 1, :mod:`curses` will try and use hardware line + If *flag* is ``True``, :mod:`curses` will try and use hardware line editing facilities. Otherwise, line insertion/deletion are disabled. @@ -1003,7 +1011,7 @@ the following methods and attributes: .. method:: window.instr([n]) window.instr(y, x[, n]) - Return a string of characters, extracted from the window starting at the + Return a bytes object of characters, extracted from the window starting at the current cursor position, or at *y*, *x* if specified. Attributes are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). @@ -1022,20 +1030,20 @@ the following methods and attributes: :meth:`refresh`; otherwise return ``False``. -.. method:: window.keypad(yes) +.. method:: window.keypad(flag) - If *yes* is 1, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *yes* is 0, escape sequences will be + If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) + will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. -.. method:: window.leaveok(yes) +.. method:: window.leaveok(flag) - If *yes* is 1, cursor is left where it is on update, instead of being at "cursor + If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor position." This reduces cursor movement where possible. If possible the cursor will be made invisible. - If *yes* is 0, cursor will always be at "cursor position" after an update. + If *flag* is ``False``, cursor will always be at "cursor position" after an update. .. method:: window.move(new_y, new_x) @@ -1055,16 +1063,16 @@ the following methods and attributes: Move the window so its upper-left corner is at ``(new_y, new_x)``. -.. method:: window.nodelay(yes) +.. method:: window.nodelay(flag) - If *yes* is ``1``, :meth:`getch` will be non-blocking. + If *flag* is ``True``, :meth:`getch` will be non-blocking. -.. method:: window.notimeout(yes) +.. method:: window.notimeout(flag) - If *yes* is ``1``, escape sequences will not be timed out. + If *flag* is ``True``, escape sequences will not be timed out. - If *yes* is ``0``, after a few milliseconds, an escape sequence will not be + If *flag* is ``False``, after a few milliseconds, an escape sequence will not be interpreted, and will be left in the input stream as is. @@ -1153,8 +1161,8 @@ the following methods and attributes: Control what happens when the cursor of a window is moved off the edge of the window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is false, the - cursor is left on the bottom line. If *flag* is true, the window is scrolled up + line, or typing the last character of the last line. If *flag* is ``False``, the + cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up one line. Note that in order to get the physical scrolling effect on the terminal, it is also necessary to call :meth:`idlok`. @@ -1202,7 +1210,7 @@ the following methods and attributes: .. method:: window.syncok(flag) - If called with *flag* set to ``True``, then :meth:`syncup` is called automatically + If *flag* is ``True``, then :meth:`syncup` is called automatically whenever there is a change in the window. @@ -1216,9 +1224,9 @@ the following methods and attributes: Set blocking or non-blocking read behavior for the window. If *delay* is negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and -1 will be returned by - :meth:`getch` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return -1 if there is + *delay* is zero, then non-blocking read is used, and :meth:`getch` will + return ``-1`` if no input is waiting. If *delay* is positive, then + :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is still no input at the end of that time. @@ -1226,7 +1234,7 @@ the following methods and attributes: Pretend *count* lines have been changed, starting with line *start*. If *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ =1) or unchanged (*changed*\ =0). + having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). .. method:: window.touchwin() @@ -1268,30 +1276,66 @@ The :mod:`curses` module defines the following data members: .. data:: version - A string representing the current version of the module. Also available as + A bytes object representing the current version of the module. Also available as :const:`__version__`. -Several constants are available to specify character cell attributes: +Some constants are available to specify character cell attributes. +The exact constants available are system dependent. +------------------+-------------------------------+ | Attribute | Meaning | +==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode. | +| ``A_ALTCHARSET`` | Alternate character set mode | ++------------------+-------------------------------+ +| ``A_BLINK`` | Blink mode | +------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode. | +| ``A_BOLD`` | Bold mode | +------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode. | +| ``A_DIM`` | Dim mode | +------------------+-------------------------------+ -| ``A_DIM`` | Dim mode. | +| ``A_INVIS`` | Invisible or blank mode | +------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute. | +| ``A_NORMAL`` | Normal attribute | ++------------------+-------------------------------+ +| ``A_PROTECT`` | Protected mode | +------------------+-------------------------------+ | ``A_REVERSE`` | Reverse background and | -| | foreground colors. | +| | foreground colors | ++------------------+-------------------------------+ +| ``A_STANDOUT`` | Standout mode | ++------------------+-------------------------------+ +| ``A_UNDERLINE`` | Underline mode | ++------------------+-------------------------------+ +| ``A_HORIZONTAL`` | Horizontal highlight | ++------------------+-------------------------------+ +| ``A_LEFT`` | Left highlight | ++------------------+-------------------------------+ +| ``A_LOW`` | Low highlight | ++------------------+-------------------------------+ +| ``A_RIGHT`` | Right highlight | ++------------------+-------------------------------+ +| ``A_TOP`` | Top highlight | ++------------------+-------------------------------+ +| ``A_VERTICAL`` | Vertical highlight | ++------------------+-------------------------------+ +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | ++------------------+-------------------------------+ + +Several constants are available to extract corresponding attributes returned +by some methods. + ++------------------+-------------------------------+ +| Bit-mask | Meaning | ++==================+===============================+ +| ``A_ATTRIBUTES`` | Bit-mask to extract | +| | attributes | +------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode. | +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | +------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode. | +| ``A_COLOR`` | Bit-mask to extract | +| | color-pair field information | +------------------+-------------------------------+ Keys are referred to by integer constants with names starting with ``KEY_``. @@ -1443,7 +1487,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ @@ -1679,10 +1723,10 @@ You can instantiate a :class:`Textbox` object as follows: .. class:: Textbox(win) Return a textbox widget object. The *win* argument should be a curses - :class:`WindowObject` in which the textbox is to be contained. The edit cursor - of the textbox is initially located at the upper left hand corner of the - containing window, with coordinates ``(0, 0)``. The instance's - :attr:`stripspaces` flag is initially on. + :ref:`window ` object in which the textbox is to + be contained. The edit cursor of the textbox is initially located at the + upper left hand corner of the containing window, with coordinates ``(0, 0)``. + The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c9318557f5c826..98fe86eba8ac77 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1849,11 +1849,11 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `datetuil.tz `_ + `dateutil.tz `_ The standard library has :class:`timezone` class for handling arbitrary fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. - *datetuil.tz* library brings the *IANA timezone database* (also known as the + *dateutil.tz* library brings the *IANA timezone database* (also known as the Olson database) to Python and its usage is recommended. `IANA timezone database `_ diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a15690ba489a66..c795782034a464 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -772,8 +781,13 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_MAP (count) - Pushes a new dictionary object onto the stack. The dictionary is pre-sized - to hold *count* entries. + Pushes a new dictionary object onto the stack. Pops ``2 * count`` items + so that the dictionary holds *count* entries: + ``{..., TOS3: TOS2, TOS1: TOS}``. + + .. versionchanged:: 3.5 + The dictionary is created from stack items instead of creating an + empty dictionary pre-sized to hold *count* items. .. opcode:: BUILD_CONST_KEY_MAP (count) @@ -793,6 +807,63 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: BUILD_TUPLE_UNPACK (count) + + Pops *count* iterables from the stack, joins them in a single tuple, + and pushes the result. Implements iterable unpacking in tuple + displays ``(*x, *y, *z)``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + +.. opcode:: BUILD_LIST_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list + instead of tuple. Implements iterable unpacking in list + displays ``[*x, *y, *z]``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_SET_UNPACK (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set + instead of tuple. Implements iterable unpacking in set + displays ``{*x, *y, *z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK (count) + + Pops *count* mappings from the stack, merges them into a single dictionary, + and pushes the result. Implements dictionary unpacking in dictionary + displays ``{**x, **y, **z}``. + + .. versionadded:: 3.5 + + +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_MAP_UNPACK`, + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. + + .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. + + .. opcode:: LOAD_ATTR (namei) Replaces TOS with ``getattr(TOS, co_names[namei])``. @@ -947,14 +1018,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -987,28 +1089,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops @@ -1034,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 15b12f7aa786ea..587a0a09a94791 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -408,7 +408,7 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: list.remove(x): x not in list That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): @@ -432,7 +432,7 @@ multi-line detail:: >>> raise ValueError('multi\n line\ndetail') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: multi line detail @@ -591,7 +591,7 @@ doctest decides whether actual output matches an example's expected output: >>> (1, 2)[3] = 'moo' Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: object doesn't support item assignment passes under Python 2.3 and later Python versions with the flag specified, diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 2c65079ebdc3f1..7e11782face074 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -33,11 +33,11 @@ having a MIME type such as :mimetype:`multipart/\*` or The conceptual model provided by a :class:`Message` object is that of an ordered dictionary of headers with additional methods for accessing both specialized information from the headers, for accessing the payload, for -generating a serialized version of the mssage, and for recursively walking over -the object tree. Note that duplicate headers are supported but special methods -must be used to access them. +generating a serialized version of the message, and for recursively walking +over the object tree. Note that duplicate headers are supported but special +methods must be used to access them. -The :class:`Message` psuedo-dictionary is indexed by the header names, which +The :class:`Message` pseudo-dictionary is indexed by the header names, which must be ASCII values. The values of the dictionary are strings that are supposed to contain only ASCII characters; there is some special handling for non-ASCII input, but it doesn't always produce the correct results. Headers @@ -67,7 +67,7 @@ Here are the methods of the :class:`Message` class: Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compabitility reasons, + *unixfrom* defaults to ``False``. For backward compatibility reasons, *maxheaderlen* defaults to ``0``, so if you want a different value you must override it explicitly (the value specified for *max_line_length* in the policy will be ignored by this method). The *policy* argument may be @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: This is a legacy method. On the :class:`~email.emailmessage.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the - realted ``make`` and ``add`` methods. + related ``make`` and ``add`` methods. .. method:: get_payload(i=None, decode=False) diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 57743d5a29f1dd..f56836ae2d2781 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -157,7 +157,7 @@ Currently the email package provides only one concrete content manager, MIME charset name, use the standard charset instead. If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to + encoding, and set the :mailheader:`Content-Transfer-Encoding` header to that value. Possible values for *cte* are ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be encoded in the specified encoding (for example, specifying a *cte* of @@ -203,5 +203,5 @@ Currently the email package provides only one concrete content manager, .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module ` diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 2d0d1923cd25c4..5838767b18f742 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -102,9 +102,9 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. return false even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - enocded bytes, the padding was not correct. Enough padding is added to + encoded bytes, the padding was not correct. Enough padding is added to perform the decode, but the resulting decoded bytes may be invalid. * :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - enocded bytes, characters outside the base64 alphebet were encountered. + encoded bytes, characters outside the base64 alphabet were encountered. The characters are ignored, but the resulting decoded bytes may be invalid. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index ab0fbc29d1ec85..1e64e1066c7da5 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -88,8 +88,8 @@ over channels that are not "8 bit clean". If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, transform parts with non-ASCII - :mailheader:`Cotnent-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatibile + :mailheader:`Content-Transfer-Encoding` + (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus rendering them RFC-compliant. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 2c830cfd81fb47..ce283c6b596cf2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -451,5 +451,5 @@ construct structured values to assign to specific headers. .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional module ` diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 32852e706986b4..261d0d62cfe618 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -52,7 +52,7 @@ message objects. .. class:: EmailMessage(policy=default) - If *policy* is specified use the rules it specifies to udpate and serialize + If *policy* is specified use the rules it specifies to update and serialize the representation of the message. If *policy* is not set, use the :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses @@ -63,7 +63,7 @@ message objects. Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compabitility + string. *unixfrom* defaults to ``False``. For backward compatibility with the base :class:`~email.message.Message` class *maxheaderlen* is accepted, but defaults to ``None``, which means that by default the line length is controlled by the @@ -213,7 +213,7 @@ message objects. del msg['subject'] msg['subject'] = 'Python roolz!' - If the :mod:`policy` defines certain haders to be unique (as the standard + If the :mod:`policy` defines certain headers to be unique (as the standard policies do), this method may raise a :exc:`ValueError` when an attempt is made to assign a value to such a header when one already exists. This behavior is intentional for consistency's sake, but do not depend on it @@ -364,7 +364,7 @@ message objects. *header* specifies an alternative header to :mailheader:`Content-Type`. If the value contains non-ASCII characters, the charset and language may - be explicity specified using the optional *charset* and *language* + be explicitly specified using the optional *charset* and *language* parameters. Optional *language* specifies the :rfc:`2231` language, defaulting to the empty string. Both *charset* and *language* should be strings. The default is to use the ``utf8`` *charset* and ``None`` for @@ -558,7 +558,7 @@ message objects. the part a candidate match if the value of the header is ``inline``. If none of the candidates matches any of the preferences in - *preferneclist*, return ``None``. + *preferencelist*, return ``None``. Notes: (1) For most applications the only *preferencelist* combinations that really make sense are ``('plain',)``, ``('html', 'plain')``, and the @@ -746,6 +746,6 @@ message objects. .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d9dae9f0b993c4..f37f6aa28dec7d 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -242,7 +242,7 @@ Here are the classes: Unless the *_charset* argument is explicitly set to ``None``, the MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Endcoding` + with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` header. This means that a subsequent ``set_payload`` call will not result in an encoded payload, even if a charset is passed in the ``set_payload`` command. You can "reset" this behavior by deleting the diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index c323ebc6401b99..dea409d223da4c 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -139,7 +139,7 @@ message body, instead setting the payload to the raw body. .. class:: BytesParser(_class=None, *, policy=policy.compat32) Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and sematnics as the *_factory* + arguments have the same meaning and semantics as the *_factory* and *policy* arguments of :class:`BytesFeedParser`. Note: **The policy keyword should always be specified**; The default will diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8a418778b8e0fc..8e7076259810f5 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -276,7 +276,7 @@ added matters. To illustrate:: Called when a header is added to an :class:`~email.message.EmailMessage` or :class:`~email.message.Message` object. If the returned value is not ``0`` or ``None``, and there are already a number of headers with the - name *name* greather than or equal to the value returned, a + name *name* greater than or equal to the value returned, a :exc:`ValueError` is raised. Because the default behavior of ``Message.__setitem__`` is to append the @@ -533,7 +533,7 @@ more closely to the RFCs relevant to their domains. The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP trasmission if the + words in the headers. Should only be used for SMTP transmission if the sender or recipient addresses have non-ASCII characters (the :meth:`smtplib.SMTP.send_message` method handles this automatically). @@ -647,5 +647,5 @@ The header objects and their attributes are described in .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional feature `. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f63326d1a2..652339e57f31ab 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -78,6 +78,9 @@ options: Providing both of the script selection options will trigger an exception. +.. versionchanged:: 3.6.3 + The exit status is non-zero if the command fails. + Module API ---------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5cd6472f3e2f45..6548adf789da17 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -24,8 +24,8 @@ Module Contents --------------- This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, and -:class:`IntFlags`. It also defines one decorator, :func:`unique`, and one +sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and +:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one helper, :class:`auto`. .. class:: Enum diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a428f5165fc8d2..a6b20a5ac95b1b 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -243,7 +243,7 @@ The following exceptions are the exceptions that are usually raised. .. note:: - It should not be used to indicate that an operater or method is not + It should not be used to indicate that an operator or method is not meant to be supported at all -- in that case either leave the operator / method undefined or, if a subclass, set it to :data:`None`. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index c03a9d31123739..634c26e95702e9 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -43,9 +43,8 @@ patterns. .. function:: fnmatch(filename, pattern) Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. If the operating system is case-insensitive, - then both parameters will be normalized to all lower- or upper-case before - the comparison is performed. :func:`fnmatchcase` can be used to perform a + :const:`True` or :const:`False`. Both parameters are case-normalized + using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a case-sensitive comparison, regardless of whether that's standard for the operating system. @@ -63,7 +62,8 @@ patterns. .. function:: fnmatchcase(filename, pattern) Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive. + :const:`False`; the comparison is case-sensitive and does not apply + :func:`os.path.normcase`. .. function:: filter(names, pattern) diff --git a/Doc/library/fpectl.rst b/Doc/library/fpectl.rst index e4b528cf0b0b6f..96607165ba4e3e 100644 --- a/Doc/library/fpectl.rst +++ b/Doc/library/fpectl.rst @@ -89,7 +89,7 @@ The following example demonstrates how to start up and test operation of the >>> import math >>> math.exp(1000) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in FloatingPointError: in math_1 diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcfef2d98e..7291dfe84811c6 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037bd9eef19..bd4c94fcae19af 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -16,8 +16,8 @@ are always available. They are listed here in alphabetical order. :func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` :func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ :func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -:func:`bytearray` :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -:func:`bytes` :func:`float` :func:`iter` :func:`print` |func-tuple|_ +|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` +|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ :func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` :func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` :func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` @@ -37,7 +37,8 @@ are always available. They are listed here in alphabetical order. .. |func-str| replace:: ``str()`` .. |func-tuple| replace:: ``tuple()`` .. |func-range| replace:: ``range()`` - +.. |func-bytearray| replace:: ``bytearray()`` +.. |func-bytes| replace:: ``bytes()`` .. function:: abs(x) @@ -80,9 +81,24 @@ are always available. They are listed here in alphabetical order. .. function:: bin(x) - Convert an integer number to a binary string. The result is a valid Python - expression. If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. + Convert an integer number to a binary string prefixed with "0b". The result + is a valid Python expression. If *x* is not a Python :class:`int` object, it + has to define an :meth:`__index__` method that returns an integer. Some + examples: + + >>> bin(3) + '0b11' + >>> bin(-10) + '-0b1010' + + If prefix "0b" is desired or not, you can use either of the following ways. + + >>> format(14, '#b'), format(14, 'b') + ('0b1110', '1110') + >>> f'{14:#b}', f'{14:b}' + ('0b1110', '1110') + + See also :func:`format` for more information. .. class:: bool([x]) @@ -99,6 +115,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray([source[, encoding[, errors]]]) + :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable sequence of integers in the range 0 <= x < 256. It has most of the usual @@ -128,6 +145,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes([source[, encoding[, errors]]]) + :noindex: Return a new "bytes" object, which is an immutable sequence of integers in the range ``0 <= x < 256``. :class:`bytes` is an immutable version of @@ -610,7 +628,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. @@ -632,16 +650,26 @@ are always available. They are listed here in alphabetical order. .. function:: hex(x) - Convert an integer number to a lowercase hexadecimal string - prefixed with "0x", for example: + Convert an integer number to a lowercase hexadecimal string prefixed with + "0x". If x is not a Python :class:`int` object, it has to define an + __index__() method that returns an integer. Some examples: >>> hex(255) '0xff' >>> hex(-42) '-0x2a' - If x is not a Python :class:`int` object, it has to define an __index__() - method that returns an integer. + If you want to convert an integer number to an uppercase or lower hexadecimal + string with prefix or not, you can use either of the following ways: + + >>> '%#x' % 255, '%x' % 255, '%X' % 255 + ('0xff', 'ff', 'FF') + >>> format(255, '#x'), format(255, 'x'), format(255, 'X') + ('0xff', 'ff', 'FF') + >>> f'{255:#x}', f'{255:x}', f'{255:X}' + ('0xff', 'ff', 'FF') + + See also :func:`format` for more information. See also :func:`int` for converting a hexadecimal string to an integer using a base of 16. @@ -875,10 +903,27 @@ are always available. They are listed here in alphabetical order. .. function:: oct(x) - Convert an integer number to an octal string. The result is a valid Python - expression. If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. + Convert an integer number to an octal string prefixed with "0o". The result + is a valid Python expression. If *x* is not a Python :class:`int` object, it + has to define an :meth:`__index__` method that returns an integer. For + example: + + >>> oct(8) + '0o10' + >>> oct(-56) + '-0o70' + If you want to convert an integer number to octal string either with prefix + "0o" or not, you can use either of the following ways. + + >>> '%#o' % 10, '%o' % 10 + ('0o12', '12') + >>> format(10, '#o'), format(10, 'o') + ('0o12', '12') + >>> f'{10:#o}', f'{10:o}' + ('0o12', '12') + + See also :func:`format` for more information. .. index:: single: file object; open() built-in function @@ -1072,7 +1117,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 @@ -1125,7 +1170,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and @@ -1254,17 +1299,21 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, @@ -1312,7 +1361,7 @@ are always available. They are listed here in alphabetical order. :func:`itertools.islice` for an alternate version that returns an iterator. -.. function:: sorted(iterable[, key][, reverse]) +.. function:: sorted(iterable, *, key=None, reverse=False) Return a new sorted list from the items in *iterable*. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 3a87bf591776c4..407853c2d7efad 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -48,9 +48,10 @@ class-based API instead. .. function:: bind_textdomain_codeset(domain, codeset=None) - Bind the *domain* to *codeset*, changing the encoding of strings returned by the - :func:`gettext` family of functions. If *codeset* is omitted, then the current - binding is returned. + Bind the *domain* to *codeset*, changing the encoding of byte strings + returned by the :func:`lgettext`, :func:`ldgettext`, :func:`lngettext` + and :func:`ldngettext` functions. + If *codeset* is omitted, then the current binding is returned. .. function:: textdomain(domain=None) @@ -67,28 +68,14 @@ class-based API instead. :func:`_` in the local namespace (see examples below). -.. function:: lgettext(message) - - Equivalent to :func:`gettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. - - .. function:: dgettext(domain, message) - Like :func:`gettext`, but look the message up in the specified *domain*. - - -.. function:: ldgettext(domain, message) - - Equivalent to :func:`dgettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. + Like :func:`.gettext`, but look the message up in the specified *domain*. .. function:: ngettext(singular, plural, n) - Like :func:`gettext`, but consider plural forms. If a translation is found, + Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some languages have more than two plural forms). If no translation is found, return *singular* if *n* is 1; return *plural* otherwise. @@ -101,24 +88,33 @@ class-based API instead. formulas for a variety of languages. -.. function:: lngettext(singular, plural, n) - - Equivalent to :func:`ngettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. - - .. function:: dngettext(domain, singular, plural, n) Like :func:`ngettext`, but look the message up in the specified *domain*. +.. function:: lgettext(message) +.. function:: ldgettext(domain, message) +.. function:: lngettext(singular, plural, n) .. function:: ldngettext(domain, singular, plural, n) - Equivalent to :func:`dngettext`, but the translation is returned in the - preferred system encoding, if no other encoding was explicitly set with + Equivalent to the corresponding functions without the ``l`` prefix + (:func:`.gettext`, :func:`dgettext`, :func:`ngettext` and :func:`dngettext`), + but the translation is returned as a byte string encoded in the preferred + system encoding if no other encoding was explicitly set with :func:`bind_textdomain_codeset`. + .. warning:: + + These functions should be avoided in Python 3, because they return + encoded bytes. It's much better to use alternatives which return + Unicode strings instead, since most Python applications will want to + manipulate human readable text as strings instead of bytes. Further, + it's possible that you may get unexpected Unicode-related exceptions + if there are encoding problems with the translated strings. It is + possible that the ``l*()`` functions will be deprecated in future Python + versions due to their inherent problems and limitations. + Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but this was deemed not useful and so it is currently unimplemented. @@ -138,7 +134,7 @@ Class-based API The class-based API of the :mod:`gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended -way of localizing your Python applications and modules. :mod:`gettext` defines +way of localizing your Python applications and modules. :mod:`!gettext` defines a "translations" class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this "translations" class can also install themselves in the built-in namespace as the function @@ -179,8 +175,9 @@ class can also install themselves in the built-in namespace as the function names are cached. The actual class instantiated is either *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the :meth:`lgettext` and - :meth:`lngettext` methods. + the charset used to encode translated strings in the + :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` + methods. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -222,7 +219,7 @@ Translation classes are what actually implement the translation of original source file message strings to translated message strings. The base class used by all translation classes is :class:`NullTranslations`; this provides the basic interface you can use to write your own specialized translation classes. Here -are the methods of :class:`NullTranslations`: +are the methods of :class:`!NullTranslations`: .. class:: NullTranslations(fp=None) @@ -250,26 +247,29 @@ are the methods of :class:`NullTranslations`: .. method:: gettext(message) - If a fallback has been set, forward :meth:`gettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. - - - .. method:: lgettext(message) - - If a fallback has been set, forward :meth:`lgettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + If a fallback has been set, forward :meth:`!gettext` to the fallback. + Otherwise, return *message*. Overridden in derived classes. .. method:: ngettext(singular, plural, n) - If a fallback has been set, forward :meth:`ngettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + If a fallback has been set, forward :meth:`!ngettext` to the fallback. + Otherwise, return *singular* if *n* is 1; return *plural* otherwise. + Overridden in derived classes. + .. method:: lgettext(message) .. method:: lngettext(singular, plural, n) - If a fallback has been set, forward :meth:`lngettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. + Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation + is returned as a byte string encoded in the preferred system encoding + if no encoding was explicitly set with :meth:`set_output_charset`. + Overridden in derived classes. + + .. warning:: + + These methods should be avoided in Python 3. See the warning for the + :func:`lgettext` function. .. method:: info() @@ -279,32 +279,28 @@ are the methods of :class:`NullTranslations`: .. method:: charset() - Return the "protected" :attr:`_charset` variable, which is the encoding of - the message catalog file. + Return the encoding of the message catalog file. .. method:: output_charset() - Return the "protected" :attr:`_output_charset` variable, which defines the - encoding used to return translated messages in :meth:`lgettext` and - :meth:`lngettext`. + Return the encoding used to return translated messages in :meth:`.lgettext` + and :meth:`.lngettext`. .. method:: set_output_charset(charset) - Change the "protected" :attr:`_output_charset` variable, which defines the - encoding used to return translated messages. + Change the encoding used to return translated messages. .. method:: install(names=None) - This method installs :meth:`self.gettext` into the built-in namespace, + This method installs :meth:`.gettext` into the built-in namespace, binding it to ``_``. If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'`` (bound to - :meth:`self.gettext`), ``'ngettext'`` (bound to :meth:`self.ngettext`), + addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, ``'lgettext'`` and ``'lngettext'``. Note that this is only one way, albeit the most convenient way, to make @@ -349,49 +345,52 @@ If the :file:`.mo` file's magic number is invalid, the major version number is unexpected, or if other problems occur while reading the file, instantiating a :class:`GNUTranslations` class can raise :exc:`OSError`. -The following methods are overridden from the base class implementation: - +.. class:: GNUTranslations -.. method:: GNUTranslations.gettext(message) + The following methods are overridden from the base class implementation: - Look up the *message* id in the catalog and return the corresponding message - string, as a Unicode string. If there is no entry in the catalog for the - *message* id, and a fallback has been set, the look up is forwarded to the - fallback's :meth:`gettext` method. Otherwise, the *message* id is returned. + .. method:: gettext(message) + Look up the *message* id in the catalog and return the corresponding message + string, as a Unicode string. If there is no entry in the catalog for the + *message* id, and a fallback has been set, the look up is forwarded to the + fallback's :meth:`~NullTranslations.gettext` method. Otherwise, the + *message* id is returned. -.. method:: GNUTranslations.lgettext(message) - Equivalent to :meth:`gettext`, but the translation is returned as a - bytestring encoded in the selected output charset, or in the preferred system - encoding if no encoding was explicitly set with :meth:`set_output_charset`. + .. method:: ngettext(singular, plural, n) + Do a plural-forms lookup of a message id. *singular* is used as the message id + for purposes of lookup in the catalog, while *n* is used to determine which + plural form to use. The returned message string is a Unicode string. -.. method:: GNUTranslations.ngettext(singular, plural, n) + If the message id is not found in the catalog, and a fallback is specified, + the request is forwarded to the fallback's :meth:`~NullTranslations.ngettext` + method. Otherwise, when *n* is 1 *singular* is returned, and *plural* is + returned in all other cases. - Do a plural-forms lookup of a message id. *singular* is used as the message id - for purposes of lookup in the catalog, while *n* is used to determine which - plural form to use. The returned message string is a Unicode string. + Here is an example:: - If the message id is not found in the catalog, and a fallback is specified, the - request is forwarded to the fallback's :meth:`ngettext` method. Otherwise, when - *n* is 1 *singular* is returned, and *plural* is returned in all other cases. + n = len(os.listdir('.')) + cat = GNUTranslations(somefile) + message = cat.ngettext( + 'There is %(num)d file in this directory', + 'There are %(num)d files in this directory', + n) % {'num': n} - Here is an example:: - n = len(os.listdir('.')) - cat = GNUTranslations(somefile) - message = cat.ngettext( - 'There is %(num)d file in this directory', - 'There are %(num)d files in this directory', - n) % {'num': n} + .. method:: lgettext(message) + .. method:: lngettext(singular, plural, n) + Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation + is returned as a byte string encoded in the preferred system encoding + if no encoding was explicitly set with + :meth:`~NullTranslations.set_output_charset`. -.. method:: GNUTranslations.lngettext(singular, plural, n) + .. warning:: - Equivalent to :meth:`gettext`, but the translation is returned as a - bytestring encoded in the selected output charset, or in the preferred system - encoding if no encoding was explicitly set with :meth:`set_output_charset`. + These methods should be avoided in Python 3. See the warning for the + :func:`lgettext` function. Solaris message catalog support @@ -509,7 +508,7 @@ module:: import gettext t = gettext.translation('spam', '/usr/share/locale') - _ = t.lgettext + _ = t.gettext Localizing your application diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 9d563566d49d7f..3a27a5b566f58d 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -506,18 +506,23 @@ to users and later verify them to make sure they weren't tampered with:: >>> AUTH_SIZE = 16 >>> >>> def sign(cookie): - ... h = blake2b(data=cookie, digest_size=AUTH_SIZE, key=SECRET_KEY) - ... return h.hexdigest() + ... h = blake2b(digest_size=AUTH_SIZE, key=SECRET_KEY) + ... h.update(cookie) + ... return h.hexdigest().encode('utf-8') >>> - >>> cookie = b'user:vatrogasac' + >>> def verify(cookie, sig): + ... good_sig = sign(cookie) + ... return compare_digest(good_sig, sig) + >>> + >>> cookie = b'user-alice' >>> sig = sign(cookie) >>> print("{0},{1}".format(cookie.decode('utf-8'), sig)) - user:vatrogasac,349cf904533767ed2d755279a8df84d0 - >>> compare_digest(cookie, sig) + user-alice,b'43b3c982cf697e0c5ab22172d1ca7421' + >>> verify(cookie, sig) True - >>> compare_digest(b'user:policajac', sig) + >>> verify(b'user-bob', sig) False - >>> compare_digesty(cookie, '0102030405060708090a0b0c0d0e0f00') + >>> verify(cookie, b'0102030405060708090a0b0c0d0e0f00') False Even though there's a native keyed hashing mode, BLAKE2 can, of course, be used diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fb5c1df611d8f2..b29020bc7ca5ca 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -105,7 +105,8 @@ of which this module provides three different variants: Contains the output stream for writing a response back to the client. Proper adherence to the HTTP protocol must be used when writing to - this stream. + this stream in order to achieve successful interoperation with HTTP + clients. .. versionchanged:: 3.6 This is an :class:`io.BufferedIOBase` stream. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a629bc50dbc749..af153593e3c46d 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -190,7 +190,9 @@ Format Paragraph paragraph will be formatted to less than N columns, where N defaults to 72. Strip trailing whitespace - Remove any space characters after the last non-space character of a line. + Remove trailing space and other whitespace characters after the last + non-whitespace character of a line by applying str.rstrip to each line, + including lines within multiline strings. .. index:: single: Run script @@ -244,7 +246,7 @@ Go to File/Line single: stack viewer Debugger (toggle) - When actived, code entered in the Shell or run from an Editor will run + When activated, code entered in the Shell or run from an Editor will run under the debugger. In the Editor, breakpoints can be set with the context menu. This feature is still incomplete and somewhat experimental. @@ -372,7 +374,7 @@ the :kbd:`Command` key on Mac OSX. * :kbd:`C-l` center window around the insertion point - * :kbd:`C-b` go backwards one character without deleting (usually you can + * :kbd:`C-b` go backward one character without deleting (usually you can also use the cursor key for this) * :kbd:`C-f` go forward one character without deleting (usually you can @@ -394,7 +396,7 @@ After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python -Shell window one tab), number depends on Indent width. Currently tabs +Shell window one tab), number depends on Indent width. Currently, tabs are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands in the edit menu. @@ -418,7 +420,7 @@ If there is only one possible completion for the characters entered, a :kbd:`C-space` will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some +classes in the current namespaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific. If a string of characters is typed, the ACW selection will jump to the @@ -446,7 +448,7 @@ longer or disable the extension. Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *acccessible* +A calltip is shown when one types :kbd:`(` after the name of an *accessible* function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or :kbd:`)` is typed. When the cursor is in the argument part of a definition, @@ -552,18 +554,62 @@ If there are arguments: ``sys.argv`` reflects the arguments passed to IDLE itself. +Startup failure +^^^^^^^^^^^^^^^ + +IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +'RESTART'). If the user process fails to connect to the GUI process, it +displays a ``Tk`` error box with a 'cannot connect' message that directs the +user here. It then exits. + +A common cause of failure is a user-written file with the same name as a +standard library module, such as *random.py* and *tkinter.py*. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file. + +Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections. + +Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over. + +A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem. + +When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one's home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files. + +If IDLE quits with no message, and it was not started from a console, try +starting from a console (``python -m idlelib)`` and see if a message appears. + + IDLE-console differences ^^^^^^^^^^^^^^^^^^^^^^^^ -As much as possible, the result of executing Python code with IDLE is the -same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, -``sys.modules`` starts with more entries. +With rare exceptions, the result of executing Python code with IDLE is +intended to be the same as executing the same code in a console window. +However, the different interface and operation occasionally affect +visible results. For instance, ``sys.modules`` starts with more entries. IDLE also replaces ``sys.stdin``, ``sys.stdout``, and ``sys.stderr`` with objects that get input from and send output to the Shell window. -When this window has the focus, it controls the keyboard and screen. -This is normally transparent, but functions that directly access the keyboard +When Shell has the focus, it controls the keyboard and screen. This is +normally transparent, but functions that directly access the keyboard and screen will not work. If ``sys`` is reset with ``importlib.reload(sys)``, IDLE's changes are lost and things like ``input``, ``raw_input``, and ``print`` will not work correctly. @@ -573,6 +619,29 @@ Some consoles only work with a single physical line at a time. IDLE uses ``exec`` to run each statement. As a result, ``'__builtins__'`` is always defined for each statement. +Developing tkinter applications +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +IDLE is intentionally different from standard Python in order to +facilitate development of tkinter programs. Enter ``import tkinter as tk; +root = tk.Tk()`` in standard Python and nothing appears. Enter the same +in IDLE and a tk window appears. In standard Python, one must also enter +``root.update()`` to see the window. IDLE does the equivalent in the +background, about 20 times a second, which is about every 50 milleseconds. +Next enter ``b = tk.Button(root, text='button'); b.pack()``. Again, +nothing visibly changes in standard Python until one enters ``root.update()``. + +Most tkinter programs run ``root.mainloop()``, which usually does not +return until the tk app is destroyed. If the program is run with +``python -i`` or from an IDLE editor, a ``>>>`` shell prompt does not +appear until ``mainloop()`` returns, at which time there is nothing left +to interact with. + +When running a tkinter program from an IDLE editor, one can comment out +the mainloop call. One then gets a shell prompt immediately and can +interact with the live application. One just has to remember to +re-enable the mainloop call when running in standard Python. + Running without a subprocess ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -583,7 +652,7 @@ If firewall software complains anyway, you can ignore it. If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch. @@ -619,32 +688,15 @@ Setting preferences The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab. Extensions ^^^^^^^^^^ -IDLE contains an extension facility. Peferences for extensions can be -changed with Configure Extensions. See the beginning of config-extensions.def -in the idlelib directory for further information. The default extensions -are currently: - -* FormatParagraph - -* AutoExpand - -* ZoomHeight - -* ScriptBinding - -* CallTips - -* ParenMatch - -* AutoComplete - -* CodeContext - -* RstripExtension +IDLE contains an extension facility. Preferences for extensions can be +changed with the Extensions tab of the preferences dialog. See the +beginning of config-extensions.def in the idlelib directory for further +information. The only current default extension is zzdummy, an example +also used for testing. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 1fd56983d0a9e9..6406e306c49faa 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -32,7 +32,7 @@ generically as an :term:`importer`) to participate in the import process. :ref:`import` The language reference for the :keyword:`import` statement. - `Packages specification `__ + `Packages specification `__ Original specification of packages. Some semantics have changed since the writing of this document (e.g. redirecting based on ``None`` in :data:`sys.modules`). diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 41a784d982d6c2..6be28a2b31cb66 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -442,7 +455,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) @@ -905,10 +920,8 @@ Classes and functions are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the locals dictionary of the given frame. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) @@ -944,10 +957,8 @@ Classes and functions :func:`getargvalues`. The format\* arguments are the corresponding optional formatting functions that are called to turn names and values into strings. - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. + .. note:: + This function was inadvertently marked as deprecated in Python 3.5. .. function:: getmro(cls) @@ -1270,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1283,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index b0d0a8c8c8ccd7..86a590bb67c17e 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -53,7 +53,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, keyfunc] sub-iterators grouped by value of keyfunc(v) +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` :func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8103c614aaf450..829218d558439b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -43,7 +43,7 @@ Encoding basic Python object hierarchies:: Compact encoding:: >>> import json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',', ':')) + >>> json.dumps([1, 2, 3, {'4': 5, '6': 7}], separators=(',', ':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: @@ -503,7 +503,7 @@ Encoders and Decoders Exceptions ---------- -.. exception:: JSONDecodeError(msg, doc, pos, end=None) +.. exception:: JSONDecodeError(msg, doc, pos) Subclass of :exc:`ValueError` with the following additional attributes: diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 5aaf4a398f7441..b04442bc16234a 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -542,17 +542,23 @@ library. Access to message catalogs -------------------------- +.. function:: gettext(msg) +.. function:: dgettext(domain, msg) +.. function:: dcgettext(domain, msg, category) +.. function:: textdomain(domain) +.. function:: bindtextdomain(domain, dir) + The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`gettext`, -:func:`dgettext`, :func:`dcgettext`, :func:`textdomain`, :func:`bindtextdomain`, -and :func:`bind_textdomain_codeset`. These are similar to the same functions in +provide this interface. It consists of the functions :func:`!gettext`, +:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, +and :func:`!bind_textdomain_codeset`. These are similar to the same functions in the :mod:`gettext` module, but use the C library's binary format for message catalogs, and the C library's search algorithms for locating message catalogs. Python applications should normally find no need to invoke these functions, and should use :mod:`gettext` instead. A known exception to this rule are applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :func:`dcgettext`. For these applications, it may be +:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be necessary to bind the text domain, so that the libraries can properly locate their message catalogs. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 5edb23de83bcff..cce6c23e611e31 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,7 +39,7 @@ Reading and writing compressed files object`. The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like object` object), in + :class:`str`, :class:`bytes` or :term:`path-like ` object), in which case the named file is opened, or it can be an existing file object to read from or write to. @@ -76,7 +76,7 @@ Reading and writing compressed files An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate directly on a named file. The *filename* argument specifies either the file object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like object` object). When wrapping an + :class:`bytes` or :term:`path-like ` object). When wrapping an existing file object, the wrapped file will not be closed when the :class:`LZMAFile` is closed. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6effc7142d..d65afc20041133 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 464248c3ea7990..67b7a7178534d3 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -186,78 +186,78 @@ than one MIME-type database; it provides an interface similar to the one of the loaded "on top" of the default database. -.. attribute:: MimeTypes.suffix_map + .. attribute:: MimeTypes.suffix_map - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + Dictionary mapping suffixes to suffixes. This is used to allow recognition of + encoded files for which the encoding and the type are indicated by the same + extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` + to allow the encoding and type to be recognized separately. This is initially a + copy of the global :data:`suffix_map` defined in the module. -.. attribute:: MimeTypes.encodings_map + .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. This is initially a + copy of the global :data:`encodings_map` defined in the module. -.. attribute:: MimeTypes.types_map + .. attribute:: MimeTypes.types_map - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + Tuple containing two dictionaries, mapping filename extensions to MIME types: + the first dictionary is for the non-standards types and the second one is for + the standard types. They are initialized by :data:`common_types` and + :data:`types_map`. -.. attribute:: MimeTypes.types_map_inv + .. attribute:: MimeTypes.types_map_inv - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + Tuple containing two dictionaries, mapping MIME types to a list of filename + extensions: the first dictionary is for the non-standards types and the + second one is for the standard types. They are initialized by + :data:`common_types` and :data:`types_map`. -.. method:: MimeTypes.guess_extension(type, strict=True) + .. method:: MimeTypes.guess_extension(type, strict=True) - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. + Similar to the :func:`guess_extension` function, using the tables stored as part + of the object. -.. method:: MimeTypes.guess_type(url, strict=True) + .. method:: MimeTypes.guess_type(url, strict=True) - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. + Similar to the :func:`guess_type` function, using the tables stored as part of + the object. -.. method:: MimeTypes.guess_all_extensions(type, strict=True) + .. method:: MimeTypes.guess_all_extensions(type, strict=True) - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. + Similar to the :func:`guess_all_extensions` function, using the tables stored + as part of the object. -.. method:: MimeTypes.read(filename, strict=True) + .. method:: MimeTypes.read(filename, strict=True) - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. + Load MIME information from a file named *filename*. This uses :meth:`readfp` to + parse the file. - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. + If *strict* is ``True``, information will be added to list of standard types, + else to the list of non-standard types. -.. method:: MimeTypes.readfp(fp, strict=True) + .. method:: MimeTypes.readfp(fp, strict=True) - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. + Load MIME type information from an open file *fp*. The file must have the format of + the standard :file:`mime.types` files. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. -.. method:: MimeTypes.read_windows_registry(strict=True) + .. method:: MimeTypes.read_windows_registry(strict=True) - Load MIME type information from the Windows registry. Availability: Windows. + Load MIME type information from the Windows registry. Availability: Windows. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. - .. versionadded:: 3.2 + .. versionadded:: 3.2 diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 96d1424126caed..2f770b6322309e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1024,7 +1024,7 @@ Connection objects are usually created using :func:`Pipe` -- see also .. method:: recv() Return an object sent from the other end of the connection using - :meth:`send`. Blocks until there its something to receive. Raises + :meth:`send`. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 8121b480cb6574..60bf23a549b114 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -17,9 +17,10 @@ The :mod:`operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is -equivalent to the expression ``x+y``. The function names are those used for -special class methods; variants without leading and trailing ``__`` are also -provided for convenience. +equivalent to the expression ``x+y``. Many function names are those used for +special methods, without the double underscores. For backward compatibility, +many of these have a variant with the double underscores kept. The variants +without the double underscores are preferred for clarity. The functions fall into categories that perform object comparisons, logical operations, mathematical operations and sequence operations. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 37fa2a2868dac9..974ab2d481e210 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2859,7 +2859,7 @@ These functions are all available on Linux only. :ref:`not following symlinks `. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` fpr *path* and *attribute*. + Accepts a :term:`path-like object` for *path* and *attribute*. .. function:: listxattr(path=None, *, follow_symlinks=True) diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50b6bd5e1..d40abe167653b7 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7c37bb7d248c91..6225a3a1f07996 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,7 +76,7 @@ The typical usage to inspect a crashed program is:: >>> import mymodule >>> mymodule.test() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "./mymodule.py", line 4, in test test2() File "./mymodule.py", line 3, in test2 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 959d9b98a84691..5796e3acb6a797 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` @@ -215,11 +215,11 @@ functions: and gathers profiling statistics from the execution. If no file name is present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified + instance and prints a simple profiling report. If the sort value is specified, it is passed to this :class:`~pstats.Stats` instance to control how the results are sorted. -.. function:: runctx(command, globals, locals, filename=None) +.. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the globals and locals dictionaries for the *command* string. This routine @@ -444,9 +444,10 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: + select a percentage of lines), or a string that will interpreted as a + regular expression (to pattern match the standard name that is printed). + If several restrictions are provided, then they are applied sequentially. + For example:: print_stats(.1, 'foo:') diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 075a8b5139b196..e43b9aecd86835 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -869,7 +869,7 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index fe5ebcc67c840b..56fc4ed2c9420d 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -42,6 +42,12 @@ module-level functions and methods on that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + The third-party `regex `_ module, + which has an API compatible with the standard library :mod:`re` module, + but offers additional functionality and a more thorough Unicode support. + .. _re-syntax: @@ -484,9 +490,10 @@ form. .. function:: compile(pattern, flags=0) - Compile a regular expression pattern into a regular expression object, which - can be used for matching using its :func:`~regex.match` and - :func:`~regex.search` methods, described below. + Compile a regular expression pattern into a :ref:`regular expression object + `, which can be used for matching using its + :func:`~regex.match`, :func:`~regex.search` and other methods, described + below. The expression's behaviour can be modified by specifying a *flags* value. Values can be any of the following variables, combined using bitwise OR (the @@ -535,9 +542,11 @@ form. .. data:: I IGNORECASE - Perform case-insensitive matching; expressions like ``[A-Z]`` will match - lowercase letters, too. This is not affected by the current locale - and works for Unicode characters as expected. + Perform case-insensitive matching; expressions like ``[A-Z]`` will also + match lowercase letters. The current locale does not change the effect of + this flag. Full Unicode matching (such as ``Ü`` matching ``ü``) also + works unless the :const:`re.ASCII` flag is also used to disable non-ASCII + matches. .. data:: L @@ -778,11 +787,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. @@ -811,15 +831,15 @@ form. .. attribute:: pos - The index of *pattern* where compilation failed. + The index in *pattern* where compilation failed (may be ``None``). .. attribute:: lineno - The line corresponding to *pos*. + The line corresponding to *pos* (may be ``None``). .. attribute:: colno - The column corresponding to *pos*. + The column corresponding to *pos* (may be ``None``). .. versionchanged:: 3.5 Added additional attributes. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118ebe05788..bd5442c6a27aa0 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 1624d88aaed38d..6d864a836de075 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -68,7 +68,7 @@ constants below: .. class:: SelectorKey A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file decriptor, selected event + associate a file object to its underlying file descriptor, selected event mask and attached data. It is returned by several :class:`BaseSelector` methods. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf339ae1bc8..41e5bafa53d468 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 039b666475a247..46d71def08abd6 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -306,8 +306,10 @@ The :mod:`signal` module defines the following functions: a library to wakeup a poll or select call, allowing the signal to be fully processed. - The old wakeup fd is returned. *fd* must be non-blocking. It is up to the - library to remove any bytes before calling poll or select again. + The old wakeup fd is returned (or -1 if file descriptor wakeup was not + enabled). If *fd* is -1, file descriptor wakeup is disabled. + If not -1, *fd* must be non-blocking. It is up to the library to remove + any bytes from *fd* before calling poll or select again. Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the signal numbers list. diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index e383201aab1461..85ee8a75cf77a9 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -13,6 +13,12 @@ This module offers several classes to implement SMTP (email) servers. +.. seealso:: + + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. :mod:`smtpd` should be considered deprecated. + Several server implementations are present; one is a generic do-nothing implementation, which can be overridden, while the other two offer specific mail-sending strategies. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5635577da0f946..9fef7d7f03f1f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To @@ -927,14 +932,6 @@ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. ``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). -So if you are within a transaction and issue a command like ``CREATE TABLE -...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly -before executing that command. There are two reasons for doing that. The first -is that some of these commands don't work within transactions. The other reason -is that sqlite3 needs to keep track of the transaction state (if a transaction -is active or not). The current transaction state is exposed through the -:attr:`Connection.in_transaction` attribute of the connection object. - You can control which kind of ``BEGIN`` statements sqlite3 implicitly executes (or none at all) via the *isolation_level* parameter to the :func:`connect` call, or via the :attr:`isolation_level` property of connections. @@ -945,6 +942,9 @@ Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +The current transaction state is exposed through the +:attr:`Connection.in_transaction` attribute of the connection object. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 07b97715a985b5..495cd590f8972e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -193,11 +193,11 @@ instead. .. table:: ======================== ============ ============ ============= ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **TLS** **TLSv1** **TLSv1.1** **TLSv1.2** + *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** ------------------------ ------------ ------------ ------------- --------- ----------- ----------- *SSLv2* yes no no [1]_ no no no *SSLv3* no yes no [2]_ no no no - *TLS* (*SSLv23*) no [1]_ no [2]_ yes yes yes yes + *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes *TLSv1* no no yes yes no no *TLSv1.1* no no yes no yes no *TLSv1.2* no no yes no no yes @@ -206,6 +206,9 @@ instead. .. rubric:: Footnotes .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default. + .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in + OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just + TLS 1.3. .. note:: @@ -215,7 +218,7 @@ instead. The *ciphers* parameter sets the available ciphers for this SSL object. It should be a string in the `OpenSSL cipher list format - `_. + `_. The parameter ``do_handshake_on_connect`` specifies whether to do the SSL handshake automatically after doing a :meth:`socket.connect`, or whether the @@ -294,6 +297,11 @@ purposes. 3DES was dropped from the default cipher string. + .. versionchanged:: 3.6.3 + + TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, + and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string. + Random generation ^^^^^^^^^^^^^^^^^ @@ -374,9 +382,9 @@ Certificate handling Verify that *cert* (in decoded format as returned by :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818` and :rfc:`6125`. In addition to HTTPS, this function - should be suitable for checking the identity of servers in various - SSL-based protocols such as FTPS, IMAPS, POPS and others. + in :rfc:`2818`, :rfc:`5280` and :rfc:`6125`. In addition to HTTPS, this + function should be suitable for checking the identity of servers in + various SSL-based protocols such as FTPS, IMAPS, POPS and others. :exc:`CertificateError` is raised on failure. On success, the function returns nothing:: @@ -610,13 +618,13 @@ Constants .. data:: PROTOCOL_TLS Selects the highest protocol version that both the client and server support. - Despite the name, this option can select "TLS" protocols as well as "SSL". + Despite the name, this option can select both "SSL" and "TLS" protocols. .. versionadded:: 3.6 .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +633,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -760,6 +768,16 @@ Constants .. versionadded:: 3.4 +.. data:: OP_NO_TLSv1_3 + + Prevents a TLSv1.3 connection. This option is only applicable in conjunction + with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.3 as + the protocol version. TLS 1.3 is available with OpenSSL 1.1.1 or later. + When Python has been compiled against an older version of OpenSSL, the + flag defaults to *0*. + + .. versionadded:: 3.6.3 + .. data:: OP_CIPHER_SERVER_PREFERENCE Use the server's cipher ordering preference, rather than the client's. @@ -820,7 +838,7 @@ Constants .. data:: HAS_SNI Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`4366`). + Indication* extension (as defined in :rfc:`6066`). .. versionadded:: 3.2 @@ -834,6 +852,12 @@ Constants .. versionadded:: 3.3 +.. data:: HAS_TLSv1_3 + + Whether the OpenSSL library has built-in support for the TLS 1.3 protocol. + + .. versionadded:: 3.6.3 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -846,7 +870,7 @@ Constants The version string of the OpenSSL library loaded by the interpreter:: >>> ssl.OPENSSL_VERSION - 'OpenSSL 0.9.8k 25 Mar 2009' + 'OpenSSL 1.0.2k 26 Jan 2017' .. versionadded:: 3.2 @@ -856,7 +880,7 @@ Constants OpenSSL library:: >>> ssl.OPENSSL_VERSION_INFO - (0, 9, 8, 11, 15) + (1, 0, 2, 11, 15) .. versionadded:: 3.2 @@ -865,9 +889,9 @@ Constants The raw version number of the OpenSSL library, as a single integer:: >>> ssl.OPENSSL_VERSION_NUMBER - 9470143 + 268443839 >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x9080bf' + '0x100020bf' .. versionadded:: 3.2 @@ -948,7 +972,7 @@ SSL Sockets :ref:`notes on non-blocking sockets `. Usually, :class:`SSLSocket` are not created directly, but using the - the :meth:`SSLContext.wrap_socket` method. + :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 The :meth:`sendfile` method was added. @@ -1421,7 +1445,7 @@ to speed up repeated connections from the same clients. Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format - `_. + `_. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. @@ -1442,8 +1466,9 @@ to speed up repeated connections from the same clients. This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is False. - OpenSSL 1.1.0+ will abort the handshake and raise :exc:`SSLError` when - both sides support ALPN but cannot agree on a protocol. + OpenSSL 1.1.0 to 1.1.0e will abort the handshake and raise :exc:`SSLError` + when both sides support ALPN but cannot agree on a protocol. 1.1.0f+ + behaves like 1.0.2, :meth:`SSLSocket.selected_alpn_protocol` returns None. .. versionadded:: 3.5 @@ -2107,8 +2132,8 @@ provided. When compared to :class:`SSLSocket`, this object lacks the following features: - - Any form of network IO incluging methods such as ``recv()`` and - ``send()``. + - Any form of network IO; ``recv()`` and ``send()`` read and write only to + the underlying :class:`MemoryBIO` buffers. - There is no *do_handshake_on_connect* machinery. You must always manually call :meth:`~SSLSocket.do_handshake` to start the handshake. @@ -2306,14 +2331,11 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent - `RFC 1750: Randomness Recommendations for Security `_ - D. Eastlake et. al. - - `RFC 3280: Internet X.509 Public Key Infrastructure Certificate and CRL Profile `_ - Housley et. al. + `RFC 4086: Randomness Requirements for Security `_ + Donald E., Jeffrey I. Schiller - `RFC 4366: Transport Layer Security (TLS) Extensions `_ - Blake-Wilson et. al. + `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_ + D. Cooper `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_ T. Dierks et. al. @@ -2323,3 +2345,9 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA + + `RFC 7525: Recommendations for Secure Use of Transport Layer Security (TLS) and Datagram Transport Layer Security (DTLS) `_ + IETF + + `Mozilla's Server Side TLS recommendations `_ + Mozilla diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9a4f42caa408a5..b8c4d59363d2a6 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -39,31 +39,26 @@ Truth Value Testing single: false Any object can be tested for truth value, for use in an :keyword:`if` or -:keyword:`while` condition or as operand of the Boolean operations below. The -following values are considered false: +:keyword:`while` condition or as operand of the Boolean operations below. - .. index:: single: None (Built-in object) - -* ``None`` - - .. index:: single: False (Built-in object) - -* ``False`` - -* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``. +.. index:: single: true -* any empty sequence, for example, ``''``, ``()``, ``[]``. +By default, an object is considered true unless its class defines either a +:meth:`__bool__` method that returns ``False`` or a :meth:`__len__` method that +returns zero, when called with the object. [1]_ Here are most of the built-in +objects considered false: -* any empty mapping, for example, ``{}``. + .. index:: + single: None (Built-in object) + single: False (Built-in object) -* instances of user-defined classes, if the class defines a :meth:`__bool__` or - :meth:`__len__` method, when that method returns the integer zero or - :class:`bool` value ``False``. [1]_ +* constants defined to be false: ``None`` and ``False``. -.. index:: single: true +* zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, + ``Fraction(0, 1)`` -All other values are considered true --- so objects of many types are always -true. +* empty sequences and collections: ``''``, ``()``, ``[]``, ``{}``, ``set()``, + ``range(0)`` .. index:: operator: or @@ -108,11 +103,11 @@ Notes: (1) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`False`. + argument if the first one is false. (2) This is a short-circuit operator, so it only evaluates the second - argument if the first one is :const:`True`. + argument if the first one is true. (3) ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is @@ -354,7 +349,7 @@ Notes: The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedNumericType.txt + See http://www.unicode.org/Public/9.0.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. @@ -394,10 +389,12 @@ Bitwise Operations on Integer Types pair: bitwise; operations pair: shifting; operations pair: masking; operations + operator: | operator: ^ operator: & operator: << operator: >> + operator: ~ Bitwise operations only make sense for integers. Negative numbers are treated as their 2's complement value (this assumes that there are enough bits so that @@ -829,7 +826,7 @@ restrictions imposed by *s*. The ``in`` and ``not in`` operations have the same priorities as the comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. +operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types @@ -1159,7 +1156,7 @@ application). :ref:`mutable ` sequence operations. Lists also provide the following additional method: - .. method:: list.sort(*, key=None, reverse=None) + .. method:: list.sort(*, key=None, reverse=False) This method sorts the list in place, using only ``<`` comparisons between items. Exceptions are not suppressed - if any comparison operations @@ -1719,10 +1716,10 @@ expression support in the :mod:`re` module). .. method:: str.join(iterable) - Return a string which is the concatenation of the strings in the - :term:`iterable` *iterable*. A :exc:`TypeError` will be raised if there are - any non-string values in *iterable*, including :class:`bytes` objects. The - separator between elements is the string providing this method. + Return a string which is the concatenation of the strings in *iterable*. + A :exc:`TypeError` will be raised if there are any non-string values in + *iterable*, including :class:`bytes` objects. The separator between + elements is the string providing this method. .. method:: str.ljust(width[, fillchar]) @@ -2264,8 +2261,8 @@ The :mod:`array` module supports efficient storage of basic data types like .. _typebytes: -Bytes ------ +Bytes Objects +------------- .. index:: object: bytes @@ -2274,65 +2271,67 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -Firstly, the syntax for bytes literals is largely the same as that for string -literals, except that a ``b`` prefix is added: +.. class:: bytes([source[, encoding[, errors]]]) + + Firstly, the syntax for bytes literals is largely the same as that for string + literals, except that a ``b`` prefix is added: -* Single quotes: ``b'still allows embedded "double" quotes'`` -* Double quotes: ``b"still allows embedded 'single' quotes"``. -* Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` + * Single quotes: ``b'still allows embedded "double" quotes'`` + * Double quotes: ``b"still allows embedded 'single' quotes"``. + * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` -Only ASCII characters are permitted in bytes literals (regardless of the -declared source code encoding). Any binary values over 127 must be entered -into bytes literals using the appropriate escape sequence. + Only ASCII characters are permitted in bytes literals (regardless of the + declared source code encoding). Any binary values over 127 must be entered + into bytes literals using the appropriate escape sequence. -As with string literals, bytes literals may also use a ``r`` prefix to disable -processing of escape sequences. See :ref:`strings` for more about the various -forms of bytes literal, including supported escape sequences. + As with string literals, bytes literals may also use a ``r`` prefix to disable + processing of escape sequences. See :ref:`strings` for more about the various + forms of bytes literal, including supported escape sequences. -While bytes literals and representations are based on ASCII text, bytes -objects actually behave like immutable sequences of integers, with each -value in the sequence restricted such that ``0 <= x < 256`` (attempts to -violate this restriction will trigger :exc:`ValueError`. This is done -deliberately to emphasise that while many binary formats include ASCII based -elements and can be usefully manipulated with some text-oriented algorithms, -this is not generally the case for arbitrary binary data (blindly applying -text processing algorithms to binary data formats that are not ASCII -compatible will usually lead to data corruption). + While bytes literals and representations are based on ASCII text, bytes + objects actually behave like immutable sequences of integers, with each + value in the sequence restricted such that ``0 <= x < 256`` (attempts to + violate this restriction will trigger :exc:`ValueError`. This is done + deliberately to emphasise that while many binary formats include ASCII based + elements and can be usefully manipulated with some text-oriented algorithms, + this is not generally the case for arbitrary binary data (blindly applying + text processing algorithms to binary data formats that are not ASCII + compatible will usually lead to data corruption). -In addition to the literal forms, bytes objects can be created in a number of -other ways: + In addition to the literal forms, bytes objects can be created in a number of + other ways: -* A zero-filled bytes object of a specified length: ``bytes(10)`` -* From an iterable of integers: ``bytes(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytes(obj)`` + * A zero-filled bytes object of a specified length: ``bytes(10)`` + * From an iterable of integers: ``bytes(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytes(obj)`` -Also see the :ref:`bytes ` built-in. + Also see the :ref:`bytes ` built-in. -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytes type has an additional class method to read data in that format: + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytes type has an additional class method to read data in that format: -.. classmethod:: bytes.fromhex(string) + .. classmethod:: fromhex(string) - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII spaces being ignored. + This :class:`bytes` class method returns a bytes object, decoding the + given string object. The string must contain two hexadecimal digits per + byte, with ASCII whitespace being ignored. - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' + >>> bytes.fromhex('2Ef0 F1f2 ') + b'.\xf0\xf1\xf2' -A reverse conversion function exists to transform a bytes object into its -hexadecimal representation. + A reverse conversion function exists to transform a bytes object into its + hexadecimal representation. -.. method:: bytes.hex() + .. method:: hex() - Return a string object containing two hexadecimal digits for each - byte in the instance. + Return a string object containing two hexadecimal digits for each + byte in the instance. - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' + >>> b'\xf0\xf1\xf2'.hex() + 'f0f1f2' - .. versionadded:: 3.5 + .. versionadded:: 3.5 Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes @@ -2362,45 +2361,49 @@ Bytearray Objects .. index:: object: bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. There is no dedicated literal syntax for bytearray objects, instead -they are always created by calling the constructor: +objects. + +.. class:: bytearray([source[, encoding[, errors]]]) -* Creating an empty instance: ``bytearray()`` -* Creating a zero-filled instance with a given length: ``bytearray(10)`` -* From an iterable of integers: ``bytearray(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` + There is no dedicated literal syntax for bytearray objects, instead + they are always created by calling the constructor: -As bytearray objects are mutable, they support the -:ref:`mutable ` sequence operations in addition to the -common bytes and bytearray operations described in :ref:`bytes-methods`. + * Creating an empty instance: ``bytearray()`` + * Creating a zero-filled instance with a given length: ``bytearray(10)`` + * From an iterable of integers: ``bytearray(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` -Also see the :ref:`bytearray ` built-in. + As bytearray objects are mutable, they support the + :ref:`mutable ` sequence operations in addition to the + common bytes and bytearray operations described in :ref:`bytes-methods`. -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytearray type has an additional class method to read data in that format: + Also see the :ref:`bytearray ` built-in. -.. classmethod:: bytearray.fromhex(string) + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytearray type has an additional class method to read data in that format: - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII spaces being ignored. + .. classmethod:: fromhex(string) - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') + This :class:`bytearray` class method returns bytearray object, decoding + the given string object. The string must contain two hexadecimal digits + per byte, with ASCII whitespace being ignored. -A reverse conversion function exists to transform a bytearray object into its -hexadecimal representation. + >>> bytearray.fromhex('2Ef0 F1f2 ') + bytearray(b'.\xf0\xf1\xf2') -.. method:: bytearray.hex() + A reverse conversion function exists to transform a bytearray object into its + hexadecimal representation. - Return a string object containing two hexadecimal digits for each - byte in the instance. + .. method:: hex() + + Return a string object containing two hexadecimal digits for each + byte in the instance. - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' + >>> bytearray(b'\xf0\xf1\xf2').hex() + 'f0f1f2' - .. versionadded:: 3.5 + .. versionadded:: 3.5 Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be @@ -2539,11 +2542,11 @@ arbitrary binary data. bytearray.join(iterable) Return a bytes or bytearray object which is the concatenation of the - binary data sequences in the :term:`iterable` *iterable*. A - :exc:`TypeError` will be raised if there are any values in *iterable* - that are not :term:`bytes-like objects `, including - :class:`str` objects. The separator between elements is the contents - of the bytes or bytearray object providing this method. + binary data sequences in *iterable*. A :exc:`TypeError` will be raised + if there are any values in *iterable* that are not :term:`bytes-like + objects `, including :class:`str` objects. The + separator between elements is the contents of the bytes or + bytearray object providing this method. .. staticmethod:: bytes.maketrans(from, to) @@ -3982,9 +3985,7 @@ The constructors for both classes work the same: Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, the *elem* set is temporarily mutated during the search and then - restored. During the search, the *elem* set should not be read or mutated - since it does not have a meaningful value. + frozenset, a temporary one is created from *elem*. .. _typesmapping: diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad2abe82453622..27f3e825961e1a 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -38,7 +38,7 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - shell=False, timeout=None, check=False, \ + shell=False, cwd=None, timeout=None, check=False, \ encoding=None, errors=None) Run the command described by *args*. Wait for command to complete, then @@ -466,9 +466,13 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + *cwd* before executing the child. *cwd* can be a :class:`str` and + :term:`path-like ` object. In particular, the function + looks for *executable* (or for the first item in *args*) relative to *cwd* + if the executable path is a relative path. + + .. versionchanged:: 3.6 + *cwd* parameter accepts a :term:`path-like object`. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. @@ -852,7 +856,7 @@ Prior to Python 3.5, these three functions comprised the high level API to subprocess. You can now use :func:`run` in many cases, but lots of existing code calls these functions. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) Run the command described by *args*. Wait for command to complete, then return the :attr:`~Popen.returncode` attribute. @@ -878,7 +882,7 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) Run command with arguments. Wait for command to complete. If the return code was zero then return, otherwise raise :exc:`CalledProcessError`. The @@ -908,7 +912,7 @@ calls these functions. .. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ - encoding=None, errors=None, \ + cwd=None, encoding=None, errors=None, \ universal_newlines=False, timeout=None) Run command with arguments and return its output. @@ -1162,27 +1166,32 @@ handling consistency are valid for these functions. .. function:: getstatusoutput(cmd) - Return ``(status, output)`` of executing *cmd* in a shell. + Return ``(exitcode, output)`` of executing *cmd* in a shell. Execute the string *cmd* in a shell with :meth:`Popen.check_output` and - return a 2-tuple ``(status, output)``. The locale encoding is used; + return a 2-tuple ``(exitcode, output)``. The locale encoding is used; see the notes on :ref:`frequently-used-arguments` for more details. A trailing newline is stripped from the output. - The exit status for the command can be interpreted - according to the rules for the C function :c:func:`wait`. Example:: + The exit code for the command can be interpreted as the return code + of subprocess. Example:: >>> subprocess.getstatusoutput('ls /bin/ls') (0, '/bin/ls') >>> subprocess.getstatusoutput('cat /bin/junk') - (256, 'cat: /bin/junk: No such file or directory') + (1, 'cat: /bin/junk: No such file or directory') >>> subprocess.getstatusoutput('/bin/junk') - (256, 'sh: /bin/junk: not found') + (127, 'sh: /bin/junk: not found') + >>> subprocess.getstatusoutput('/bin/kill $$') + (-15, '') Availability: POSIX & Windows .. versionchanged:: 3.3.4 - Windows support added + Windows support was added. + + The function now returns (exitcode, output) instead of (status, output) + as it did in Python 3.3.3 and earlier. See :func:`WEXITSTATUS`. .. function:: getoutput(cmd) diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 1ecc7a7cf92bd0..c8357e4fcc85e2 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -118,7 +118,7 @@ AU_read objects, as returned by :func:`.open` above, have the following methods: .. method:: AU_read.getnchannels() - Returns number of audio channels (1 for mone, 2 for stereo). + Returns number of audio channels (1 for mono, 2 for stereo). .. method:: AU_read.getsampwidth() diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 08b74a9affd24f..f066a765d00ec7 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -255,7 +255,6 @@ You can use :mod:`sysconfig` as a script with Python's *-m* option: AIX_GENUINE_CPLUSPLUS = "0" AR = "ar" ARFLAGS = "rc" - ASDLGEN = "./Parser/asdl_c.py" ... This call will print in the standard output the information returned by diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fbabb2023..dfe688a2f93e0c 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f809753dfdd7..337c061107299c 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261ffb21f94..c59aca1e189086 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above diff --git a/Doc/library/test.rst b/Doc/library/test.rst index fab3e1fe4cc6ad..9d4ff7ad8b45a3 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -570,7 +570,8 @@ The :mod:`test.support` module defines the following functions: def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()): + +.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not found on *other_api*, except for a defined list of items to be diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 2792dfdce04c6c..c375754629e1f4 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -291,10 +291,10 @@ since it is impossible to detect the termination of alien threads. .. attribute:: ident The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the - :func:`_thread.get_ident()` function. Thread identifiers may be recycled - when a thread exits and another thread is created. The identifier is - available even after the thread has exited. + been started. This is a nonzero integer. See the :func:`get_ident` + function. Thread identifiers may be recycled when a thread exits and + another thread is created. The identifier is available even after the + thread has exited. .. method:: is_alive() @@ -371,8 +371,9 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - .. versionchanged:: 3.3 - Changed from a factory function to a class. + Note that ``Lock`` is actually a factory function which returns an instance + of the most efficient version of the concrete Lock class that is supported + by the platform. .. method:: acquire(blocking=True, timeout=-1) @@ -874,8 +875,8 @@ Barrier Objects This class provides a simple synchronization primitive for use by a fixed number of threads that need to wait for each other. Each of the threads tries to pass the barrier by calling the :meth:`~Barrier.wait` method and will block until -all of the threads have made the call. At this points, the threads are released -simultaneously. +all of the threads have made their :meth:`~Barrier.wait` calls. At this point, +the threads are released simultaneously. The barrier can be reused any number of times for the same number of threads. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ae17f6f4f0d236..d2e2ac4410aba3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -17,11 +17,23 @@ semantics of these functions varies among platforms. An explanation of some terminology and conventions is in order. +.. _epoch: + .. index:: single: epoch -* The :dfn:`epoch` is the point where the time starts. On January 1st of that - year, at 0 hours, the "time since the epoch" is zero. For Unix, the epoch is - 1970. To find out what the epoch is, look at ``gmtime(0)``. +* The :dfn:`epoch` is the point where the time starts, and is platform + dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). + To find out what the epoch is on a given platform, look at + ``time.gmtime(0)``. + +.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second + +.. index:: seconds since the epoch + +* The term :dfn:`seconds since the epoch` refers to the total number + of elapsed seconds since the epoch, typically excluding + `leap seconds`_. Leap seconds are excluded from this total on all + POSIX-compliant platforms. .. index:: single: Year 2038 @@ -467,7 +479,7 @@ The module defines the following functions and data items: (2) The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing leap seconds and value ``61`` is supported + timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. (3) @@ -572,12 +584,28 @@ The module defines the following functions and data items: .. function:: time() - Return the time in seconds since the epoch as a floating point number. + Return the time in seconds since the epoch_ as a floating point + number. The specific date of the epoch and the handling of + `leap seconds`_ is platform dependent. + On Windows and most Unix systems, the epoch is January 1, 1970, + 00:00:00 (UTC) and leap seconds are not counted towards the time + in seconds since the epoch. This is commonly referred to as + `Unix time `_. + To find out what the epoch is on a given platform, look at + ``gmtime(0)``. + Note that even though the time is always returned as a floating point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back between - the two calls. + lower value than a previous call if the system clock has been set back + between the two calls. + + The number returned by :func:`.time` may be converted into a more common + time format (i.e. year, month, day, hour, etc...) in UTC by passing it to + :func:`gmtime` function or in local time by passing it to the + :func:`localtime` function. In both cases a + :class:`struct_time` object is returned, from which the components + of the calendar date may be accessed as attributes. .. data:: timezone diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 3b772765aca260..5793c54ae8d1a3 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -134,21 +134,21 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None) - Automatically determine how many times to call :meth:`.timeit`. + Automatically determine how many times to call :meth:`.timeit`. - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. + This is a convenience function that calls :meth:`.timeit` repeatedly + so that the total time >= 0.2 second, returning the eventual + (number of loops, time taken for that number of loops). It calls + :meth:`.timeit` with *number* set to successive powers of ten (10, + 100, 1000, ...) up to a maximum of one billion, until the time taken + is at least 0.2 second, or the maximum is reached. - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. + If *callback* is given and is not ``None``, it will be called after + each trial with two arguments: ``callback(number, time_taken)``. - .. versionadded:: 3.6 + .. versionadded:: 3.6 .. method:: Timer.repeat(repeat=3, number=1000000) diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 3dad182cccdf89..927f7f3c6c1b71 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1099,26 +1099,42 @@ ttk.Treeview If *selop* is not specified, returns selected items. Otherwise, it will act according to the following selection methods. + .. deprecated-removed:: 3.6 3.8 + Using ``selection()`` for changing the selection state is deprecated. + Use the following selection methods instead. - .. method:: selection_set(items) + + .. method:: selection_set(*items) *items* becomes the new selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + - .. method:: selection_add(items) + .. method:: selection_add(*items) Add *items* to the selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + - .. method:: selection_remove(items) + .. method:: selection_remove(*items) Remove *items* from the selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. - .. method:: selection_toggle(items) + + .. method:: selection_toggle(*items) Toggle the selection state of each item in *items*. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + .. method:: set(item, column=None, value=None) diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index ff55aacbd44c5b..afed36b9e46170 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -16,8 +16,9 @@ implemented in Python. The scanner in this module returns comments as tokens as well, making it useful for implementing "pretty-printers," including colorizers for on-screen displays. -To simplify token stream handling, all :ref:`operators` and :ref:`delimiters` -tokens are returned using the generic :data:`token.OP` token type. The exact +To simplify token stream handling, all :ref:`operator ` and +:ref:`delimiter ` tokens and :data:`Ellipsis` are returned using +the generic :data:`~token.OP` token type. The exact type can be determined by checking the ``exact_type`` property on the :term:`named tuple` returned from :func:`tokenize.tokenize`. diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index b0ac81271c5c18..5cb7029adf5e9e 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -13,6 +13,12 @@ annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. +.. seealso:: + + `Coverage.py `_ + A popular third-party coverage tool that provides HTML + output along with advanced features such as branch coverage. + .. _trace-cli: Command-Line Usage diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 066ee96fc004ab..55d331c996a187 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -45,9 +45,9 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` * it prints the exception *etype* and *value* after the stack trace - * if *etype* is :exc:`SyntaxError` and *value* has the appropriate format, it - prints the line where the syntax error occurred with a caret indicating the - approximate position of the error. + * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate + format, it prints the line where the syntax error occurred with a caret + indicating the approximate position of the error. The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the @@ -55,6 +55,9 @@ The module defines the following functions: printed as well, like the interpreter itself does when printing an unhandled exception. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -131,6 +134,9 @@ The module defines the following functions: containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does :func:`print_exception`. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: format_exc(limit=None, chain=True) @@ -372,6 +378,7 @@ exception and traceback: print("*** print_tb:") traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) print("*** print_exception:") + # exc_type below is ignored on 3.5 and later traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout) print("*** print_exc:") @@ -381,6 +388,7 @@ exception and traceback: print(formatted_lines[0]) print(formatted_lines[-1]) print("*** format_exception:") + # exc_type below is ignored on 3.5 and later print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) print("*** extract_tb:") diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 1986972549c503..2b5d3e567d6cf2 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1797,7 +1797,7 @@ Input methods :param prompt: string Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, propmt is a text mostly describing + the title of the dialog window, prompt is a text mostly describing what information to input. Return the string input. If the dialog is canceled, return ``None``. :: @@ -2368,7 +2368,7 @@ The demo scripts are: | wikipedia | a pattern from the wikipedia | :func:`clone`, | | | article on turtle graphics | :func:`undo` | +----------------+------------------------------+-----------------------+ -| yingyang | another elementary example | :func:`circle` | +| yinyang | another elementary example | :func:`circle` | +----------------+------------------------------+-----------------------+ Have fun! diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6c8982ba743526..bd04f731a1238b 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -111,8 +111,7 @@ More precisely, the expression ``some_value is Derived(some_value)`` is always true at runtime. This also means that it is not possible to create a subtype of ``Derived`` -since it is an identity function at runtime, not an actual type. Similarly, it -is not possible to create another :func:`NewType` based on a ``Derived`` type:: +since it is an identity function at runtime, not an actual type:: from typing import NewType @@ -121,9 +120,16 @@ is not possible to create another :func:`NewType` based on a ``Derived`` type:: # Fails at runtime and does not typecheck class AdminUserId(UserId): pass - # Also does not typecheck +However, it is possible to create a :func:`NewType` based on a 'derived' ``NewType``:: + + from typing import NewType + + UserId = NewType('UserId', int) + ProUserId = NewType('ProUserId', UserId) +and typechecking for ``ProUserId`` will work as expected. + See :pep:`484` for more details. .. note:: @@ -508,6 +514,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -574,6 +588,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.6.1 + .. class:: List(list, MutableSequence[T]) Generic version of :class:`list`. @@ -656,7 +672,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -740,9 +768,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -754,6 +785,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -780,10 +814,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -792,7 +836,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -901,7 +945,7 @@ The module defines the following classes, functions and decorators: Union[int, str] == Union[str, int] - * When a class and its subclass are present, the former is skipped, e.g.:: + * When a class and its subclass are present, the latter is skipped, e.g.:: Union[int, object] == object @@ -970,9 +1014,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance @@ -1003,5 +1047,10 @@ The module defines the following classes, functions and decorators: if TYPE_CHECKING: import expensive_mod - def fun(): - local_var: expensive_mod.some_type = other_fun() + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + Note that the first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 643180953fd23d..2a9777609527d0 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -158,7 +158,7 @@ Examples: 9 >>> unicodedata.decimal('a') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: not a decimal >>> unicodedata.category('A') # 'L'etter, 'u'ppercase 'Lu' diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c6d0ec92b6e9ab..a552cbfc70ad2e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -303,14 +303,14 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and with the specified - arguments. + Assert that the mock was called exactly once and that that call was + with the specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') + >>> mock('other', bar='values') + >>> mock.assert_called_once_with('other', bar='values') Traceback (most recent call last): ... AssertionError: Expected 'mock' to be called once. Called 2 times. @@ -322,7 +322,8 @@ the *new_callable* argument to :func:`patch`. The assert passes if the mock has *ever* been called, unlike :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one. + only pass if the call is the most recent one, and in the case of + :meth:`assert_called_once_with` it must also be the only call. >>> mock = Mock(return_value=None) >>> mock(1, 2, arg='thing') diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 92e567d12824fa..2099bd1e2e979c 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1170,6 +1170,9 @@ Test cases :meth:`.assertRegex`. .. versionadded:: 3.2 :meth:`.assertNotRegex`. + .. versionadded:: 3.5 + The name ``assertNotRegexpMatches`` is a deprecated alias + for :meth:`.assertNotRegex`. .. method:: assertCountEqual(first, second, msg=None) @@ -1435,9 +1438,9 @@ For historical reasons, some of the :class:`TestCase` methods had one or more aliases that are now deprecated. The following table lists the correct names along with their deprecated aliases: - ============================== ====================== ====================== + ============================== ====================== ======================= Method Name Deprecated alias Deprecated alias - ============================== ====================== ====================== + ============================== ====================== ======================= :meth:`.assertEqual` failUnlessEqual assertEquals :meth:`.assertNotEqual` failIfEqual assertNotEquals :meth:`.assertTrue` failUnless assert\_ @@ -1446,8 +1449,9 @@ along with their deprecated aliases: :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals :meth:`.assertRegex` assertRegexpMatches + :meth:`.assertNotRegex` assertNotRegexpMatches :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ====================== + ============================== ====================== ======================= .. deprecated:: 3.1 the fail* aliases listed in the second column. @@ -1455,8 +1459,9 @@ along with their deprecated aliases: the assert* aliases listed in the third column. .. deprecated:: 3.2 ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex` - + :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. + .. deprecated:: 3.5 + the ``assertNotRegexpMatches`` name in favor of :meth:`.assertNotRegex`. .. _testsuite-objects: diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 676321b46a2232..1cc69e62e63318 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -118,6 +118,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -236,6 +239,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index e289b971e7c269..b02a006d733b6e 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -166,8 +166,8 @@ Extension types can easily be made to support weak references; see performed by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakKeyDictionary` objects have the following additional methods. These -expose the internal references directly. The references are not guaranteed to +:class:`WeakKeyDictionary` objects have an additional method that +exposes the internal references directly. The references are not guaranteed to be "live" at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer @@ -192,9 +192,9 @@ than needed. by the program during iteration may cause items in the dictionary to vanish "by magic" (as a side effect of garbage collection). -:class:`WeakValueDictionary` objects have the following additional methods. -These method have the same issues as the and :meth:`keyrefs` method of -:class:`WeakKeyDictionary` objects. +:class:`WeakValueDictionary` objects have an additional method that has the +same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` +objects. .. method:: WeakValueDictionary.valuerefs() diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2e9e814693df57..40470e8736e7e4 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -248,7 +248,7 @@ utility to most DOM users. .. rubric:: Footnotes -.. [#] The encoding name included in the XML output should conform to +.. [1] The encoding name included in the XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not valid in an XML document's declaration, even though Python accepts it as an encoding name. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index b50255434de945..5c0f469ad7a5cf 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -108,7 +108,7 @@ DOMEventStream Objects :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals :data:`CHARACTERS`. - The current node does not contain informations about its children, unless + The current node does not contain information about its children, unless :func:`expandNode` is called. .. method:: expandNode(node) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index b54eace41188b0..7d814ad406eb1b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1192,7 +1192,7 @@ Exceptions .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f103380661..a5d42118ba5176 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -451,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c245f7b3fe..eaae2bb04b7482 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/make.bat b/Doc/make.bat index da1f8765a4f358..6cb315fda40583 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -1,130 +1,167 @@ -@echo off -setlocal - -pushd %~dp0 - -set this=%~n0 - -if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build -if "%PYTHON%" EQU "" set PYTHON=py - -if "%1" NEQ "htmlhelp" goto :skiphhcsearch -if exist "%HTMLHELP%" goto :skiphhcsearch - -rem Search for HHC in likely places -set HTMLHELP= -where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch -where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" ( - echo. - echo.The HTML Help Workshop was not found. Set the HTMLHELP variable - echo.to the path to hhc.exe or download and install it from - echo.http://msdn.microsoft.com/en-us/library/ms669985 - exit /B 1 -) -:skiphhcsearch - -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v - -if "%BUILDDIR%" EQU "" set BUILDDIR=build - -rem Targets that don't require sphinx-build -if "%1" EQU "" goto help -if "%1" EQU "help" goto help -if "%1" EQU "check" goto check -if "%1" EQU "serve" goto serve -if "%1" == "clean" ( - rmdir /q /s %BUILDDIR% - goto end -) - -%SPHINXBUILD% >nul 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - popd - exit /B 1 -) - -rem Targets that do require sphinx-build and have their own label -if "%1" EQU "htmlview" goto htmlview - -rem Everything else -goto build - -:help -echo.usage: %this% BUILDER [filename ...] -echo. -echo.Call %this% with the desired Sphinx builder as the first argument, e.g. -echo.``%this% html`` or ``%this% doctest``. Interesting targets that are -echo.always available include: -echo. -echo. Provided by Sphinx: -echo. html, htmlhelp, latex, text -echo. suspicious, linkcheck, changes, doctest -echo. Provided by this script: -echo. clean, check, serve, htmlview -echo. -echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or -echo.the documentation for your version of Sphinx for more exhaustive lists -echo.of available targets and descriptions of each. -echo. -echo.This script assumes that the SPHINXBUILD environment variable contains -echo.a legitimate command for calling sphinx-build, or that sphinx-build is -echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can -echo.be passed by setting the SPHINXOPTS environment variable. -goto end - -:build -if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% -) -cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* - -if "%1" EQU "htmlhelp" ( - cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp - rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 - if not errorlevel 2 cmd /C exit /b 0 -) - -echo. -if errorlevel 1 ( - echo.Build failed (exit code %ERRORLEVEL%^), check for error messages - echo.above. Any output will be found in %BUILDDIR%\%1 -) else ( - echo.Build succeeded. All output should be in %BUILDDIR%\%1 -) -goto end - -:htmlview -if NOT "%2" EQU "" ( - echo.Can't specify filenames to build with htmlview target, ignoring. -) -cmd /C %this% html - -if EXIST %BUILDDIR%\html\index.html ( - echo.Opening %BUILDDIR%\html\index.html in the default web browser... - start %BUILDDIR%\html\index.html -) - -goto end - -:check -cmd /C %PYTHON% tools\rstlint.py -i tools -goto end - -:serve -cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html -goto end - -:end -popd +@echo off +setlocal + +pushd %~dp0 + +set this=%~n0 + +call ..\PCBuild\find_python.bat %PYTHON% +if not defined SPHINXBUILD if defined PYTHON ( + %PYTHON% -c "import sphinx" > nul 2> nul + if errorlevel 1 ( + echo Installing sphinx with %PYTHON% + %PYTHON% -m pip install sphinx + if errorlevel 1 exit /B + ) + set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()" +) + +if not defined BLURB if defined PYTHON ( + %PYTHON% -c "import blurb" > nul 2> nul + if errorlevel 1 ( + echo Installing blurb with %PYTHON% + %PYTHON% -m pip install blurb + if errorlevel 1 exit /B + ) + set BLURB=%PYTHON% -m blurb +) + +if not defined PYTHON set PYTHON=py +if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build +if not defined BLURB set BLURB=blurb + +if "%1" NEQ "htmlhelp" goto :skiphhcsearch +if exist "%HTMLHELP%" goto :skiphhcsearch + +rem Search for HHC in likely places +set HTMLHELP= +where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch +where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" +if not exist "%HTMLHELP%" ( + echo. + echo.The HTML Help Workshop was not found. Set the HTMLHELP variable + echo.to the path to hhc.exe or download and install it from + echo.http://msdn.microsoft.com/en-us/library/ms669985 + exit /B 1 +) +:skiphhcsearch + +if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v + +if "%BUILDDIR%" EQU "" set BUILDDIR=build + +rem Targets that don't require sphinx-build +if "%1" EQU "" goto help +if "%1" EQU "help" goto help +if "%1" EQU "check" goto check +if "%1" EQU "serve" goto serve +if "%1" == "clean" ( + rmdir /q /s "%BUILDDIR%" + goto end +) + +%SPHINXBUILD% >nul 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + popd + exit /B 1 +) + +rem Targets that do require sphinx-build and have their own label +if "%1" EQU "htmlview" goto htmlview + +rem Everything else +goto build + +:help +echo.usage: %this% BUILDER [filename ...] +echo. +echo.Call %this% with the desired Sphinx builder as the first argument, e.g. +echo.``%this% html`` or ``%this% doctest``. Interesting targets that are +echo.always available include: +echo. +echo. Provided by Sphinx: +echo. html, htmlhelp, latex, text +echo. suspicious, linkcheck, changes, doctest +echo. Provided by this script: +echo. clean, check, serve, htmlview +echo. +echo.All arguments past the first one are passed through to sphinx-build as +echo.filenames to build or are ignored. See README.rst in this directory or +echo.the documentation for your version of Sphinx for more exhaustive lists +echo.of available targets and descriptions of each. +echo. +echo.This script assumes that the SPHINXBUILD environment variable contains +echo.a legitimate command for calling sphinx-build, or that sphinx-build is +echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can +echo.be passed by setting the SPHINXOPTS environment variable. +goto end + +:build +if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" + +if exist ..\Misc\NEWS ( + echo.Copying Misc\NEWS to build\NEWS + copy ..\Misc\NEWS build\NEWS > nul +) else if exist ..\Misc\NEWS.D ( + if defined BLURB ( + echo.Merging Misc/NEWS with %BLURB% + %BLURB% merge -f build\NEWS + ) else ( + echo.No Misc/NEWS file and Blurb is not available. + exit /B 1 + ) +) + +if NOT "%PAPER%" == "" ( + set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% +) +cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" + +if "%1" EQU "htmlhelp" ( + "%HTMLHELP%" "%BUILDDIR%\htmlhelp\python%DISTVERSION:.=%.hhp" + rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 + if not errorlevel 2 cmd /C exit /b 0 +) + +echo. +if errorlevel 1 ( + echo.Build failed (exit code %ERRORLEVEL%^), check for error messages + echo.above. Any output will be found in %BUILDDIR%\%1 +) else ( + echo.Build succeeded. All output should be in %BUILDDIR%\%1 +) +goto end + +:htmlview +if NOT "%2" EQU "" ( + echo.Can't specify filenames to build with htmlview target, ignoring. +) +cmd /C %this% html + +if EXIST "%BUILDDIR%\html\index.html" ( + echo.Opening "%BUILDDIR%\html\index.html" in the default web browser... + start "%BUILDDIR%\html\index.html" +) + +goto end + +:check +cmd /S /C "%PYTHON% tools\rstlint.py -i tools" +goto end + +:serve +cmd /S /C "%PYTHON% ..\Tools\scripts\serve.py "%BUILDDIR%\html"" +goto end + +:end +popd diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 095a2380b379bc..230caf843a5d9d 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -320,9 +320,9 @@ Sequences A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in function :func:`bytes` can be used to - construct bytes objects. Also, bytes objects can be decoded to strings - via the :meth:`~bytes.decode` method. + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. Mutable sequences .. index:: @@ -349,9 +349,9 @@ Sequences .. index:: bytearray A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable (and hence - unhashable), byte arrays otherwise provide the same interface and - functionality as immutable bytes objects. + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. .. index:: module: array @@ -1119,9 +1119,9 @@ Basic customization (usually an instance of *cls*). Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super(currentclass, - cls).__new__(cls[, ...])`` with appropriate arguments and then modifying the - newly-created instance as necessary before returning it. + superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` + with appropriate arguments and then modifying the newly-created instance + as necessary before returning it. If :meth:`__new__` returns an instance of *cls*, then the new instance's :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where @@ -1145,7 +1145,7 @@ Basic customization class constructor expression. If a base class has an :meth:`__init__` method, the derived class's :meth:`__init__` method, if any, must explicitly call it to ensure proper initialization of the base class part of the - instance; for example: ``BaseClass.__init__(self, [args...])``. + instance; for example: ``super().__init__([args...])``. Because :meth:`__new__` and :meth:`__init__` work together in constructing objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), @@ -1253,8 +1253,8 @@ Basic customization .. index:: builtin: bytes - Called by :func:`bytes` to compute a byte-string representation of an - object. This should return a ``bytes`` object. + Called by :ref:`bytes ` to compute a byte-string representation + of an object. This should return a :class:`bytes` object. .. index:: single: string; __format__() (object method) @@ -1578,8 +1578,8 @@ Class Binding ``A.__dict__['x'].__get__(None, A)``. Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, - obj).m()`` searches ``obj.__class__.__mro__`` for the base class ``A`` + If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` + searches ``obj.__class__.__mro__`` for the base class ``A`` immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. @@ -1875,8 +1875,8 @@ Metaclass example ^^^^^^^^^^^^^^^^^ The potential uses for metaclasses are boundless. Some ideas that have been -explored include logging, interface checking, automatic delegation, automatic -property creation, proxies, frameworks, and automatic resource +explored include enum, logging, interface checking, automatic delegation, +automatic property creation, proxies, frameworks, and automatic resource locking/synchronization. Here is an example of a metaclass that uses an :class:`collections.OrderedDict` @@ -2011,6 +2011,14 @@ through the container; for mappings, :meth:`__iter__` should be the same as :meth:`__bool__` method and whose :meth:`__len__` method returns zero is considered to be false in a Boolean context. + .. impl-detail:: + + In CPython, the length is required to be at most :attr:`sys.maxsize`. + If the length is larger than :attr:`!sys.maxsize` some features (such as + :func:`len`) may raise :exc:`OverflowError`. To prevent raising + :exc:`!OverflowError` by truth value testing, an object must define a + :meth:`__bool__` method. + .. method:: object.__length_hint__(self) @@ -2021,6 +2029,7 @@ through the container; for mappings, :meth:`__iter__` should be the same as .. versionadded:: 3.4 + .. note:: Slicing is done exclusively with the following three methods. A call like :: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92ed460f3..d08abdf3343fe4 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -164,15 +164,6 @@ Builtins and restricted execution .. index:: pair: restricted; execution -The builtins namespace associated with the execution of a code block is actually -found by looking up the name ``__builtins__`` in its global namespace; this -should be a dictionary or a module (in the latter case the module's dictionary -is used). By default, when in the :mod:`__main__` module, ``__builtins__`` is -the built-in module :mod:`builtins`; when in any other module, -``__builtins__`` is an alias for the dictionary of the :mod:`builtins` module -itself. ``__builtins__`` can be set to a user-created dictionary to create a -weak form of restricted execution. - .. impl-detail:: Users should not touch ``__builtins__``; it is strictly an implementation @@ -180,6 +171,15 @@ weak form of restricted execution. :keyword:`import` the :mod:`builtins` module and modify its attributes appropriately. +The builtins namespace associated with the execution of a code block +is actually found by looking up the name ``__builtins__`` in its +global namespace; this should be a dictionary or a module (in the +latter case the module's dictionary is used). By default, when in the +:mod:`__main__` module, ``__builtins__`` is the built-in module +:mod:`builtins`; when in any other module, ``__builtins__`` is an +alias for the dictionary of the :mod:`builtins` module itself. + + .. _dynamic-features: Interaction with dynamic features @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 3a4b80557caf67..ff890a815b5bbe 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -190,7 +190,7 @@ Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading -expression, may contan additonal :keyword:`for` or :keyword:`async for` +expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. If a comprehension contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an @@ -636,7 +636,7 @@ which are used to control the execution of a generator function. without yielding another value, an :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitalbe is run that exception + raises a different exception, then when the awaitable is run that exception propagates to the caller of the awaitable. .. index:: exception: GeneratorExit @@ -905,7 +905,7 @@ keyword arguments (and any ``**expression`` arguments -- see below). So:: 2 1 >>> f(a=1, *(2,)) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: f() got multiple values for keyword argument 'a' >>> f(1, *(2,)) 1 2 @@ -1317,7 +1317,7 @@ built-in types. * Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can be compared only within each of their types, with the restriction that ranges do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises + results in inequality, and ordering comparison across these types raises :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). @@ -1691,8 +1692,8 @@ precedence and have a left-to-right chaining feature as described in the | ``+``, ``-`` | Addition and subtraction | +-----------------------------------------------+-------------------------------------+ | ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication division, | -| | remainder [#]_ | +| | multiplication, division, floor | +| | division, remainder [#]_ | +-----------------------------------------------+-------------------------------------+ | ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +-----------------------------------------------+-------------------------------------+ diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 5e2c1c8b0758d8..8cf16cad4c9c9c 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -36,7 +36,7 @@ implement import semantics. When a module is first imported, Python searches for the module and if found, it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, an :exc:`ModuleNotFoundError` is raised. Python implements various +cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various strategies to search for the named module when the import machinery is invoked. These strategies can be modified and extended by using various hooks described in the sections below. @@ -167,7 +167,7 @@ arguments to the :keyword:`import` statement, or from the parameters to the This name will be used in various phases of the import search, and it may be the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, an :exc:`ModuleNotFoundError` is raised. +If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. The module cache @@ -185,7 +185,7 @@ object. During import, the module name is looked up in :data:`sys.modules` and if present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then an +process completes. However, if the value is ``None``, then a :exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will continue searching for the module. @@ -194,7 +194,7 @@ associated module (as other modules may hold references to it), but it will invalidate the cache entry for the named module, causing Python to search anew for the named module upon its next import. The key can also be assigned to ``None``, forcing the next import -of the module to result in an :exc:`ModuleNotFoundError`. +of the module to result in a :exc:`ModuleNotFoundError`. Beware though, as if you keep a reference to the module object, invalidate its cache entry in :data:`sys.modules`, and then re-import the @@ -298,7 +298,7 @@ The second argument is the path entries to use for the module search. For top-level modules, the second argument is ``None``, but for submodules or subpackages, the second argument is the value of the parent package's ``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, an :exc:`ModuleNotFoundError` is raised. The third argument +be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument is an existing module object that will be the target of loading later. The import system passes in a target module only during reload. @@ -431,7 +431,7 @@ on the module object. If the method returns ``None``, the import machinery will create the new module itself. .. versionadded:: 3.4 - The create_module() method of loaders. + The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 The :meth:`~importlib.abc.Loader.load_module` method was replaced by @@ -464,8 +464,11 @@ import machinery will create the new module itself. .. versionchanged:: 3.5 A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. + ``create_module()`` is not. + +.. versionchanged:: 3.6 + An :exc:`ImportError` is raised when ``exec_module()`` is defined but + ``create_module()`` is not. Submodules ---------- @@ -613,7 +616,7 @@ the module. module.__path__ --------------- -By definition, if a module has an ``__path__`` attribute, it is a package, +By definition, if a module has a ``__path__`` attribute, it is a package, regardless of its value. A package's ``__path__`` attribute is used during imports of its subpackages. @@ -887,7 +890,7 @@ import statements within that module. To selectively prevent import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNoFoundError` directly from +it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. @@ -962,7 +965,7 @@ References The import machinery has evolved considerably since Python's early days. The original `specification for packages -`_ is still available to read, +`_ is still available to read, although some details have changed since the writing of that document. The original specification for :data:`sys.meta_path` was :pep:`302`, with diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017afff0a3d..4a5abf62768155 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -313,7 +313,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -446,9 +446,6 @@ instance of the :class:`bytes` type instead of the :class:`str` type. They may only contain ASCII characters; bytes with a numeric value of 128 or greater must be expressed with escapes. -As of Python 3.3 it is possible again to prefix string literals with a -``u`` prefix to simplify maintenance of dual 2.x and 3.x codebases. - Both string and bytes literals may optionally be prefixed with a letter ``'r'`` or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` @@ -676,6 +673,12 @@ Some examples of formatted string literals:: >>> value = decimal.Decimal("12.34567") >>> f"result: {value:{width}.{precision}}" # nested fields 'result: 12.35' + >>> today = datetime(year=2017, month=1, day=27) + >>> f"{today:%b %d, %Y}" # using date format specifier + 'January 27, 2017' + >>> number = 1024 + >>> f"{number:#0x}" # using integer format specifier + '0x400' A consequence of sharing the same syntax as regular string literals is that characters in the replacement fields must not conflict with the @@ -696,6 +699,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. @@ -782,10 +796,6 @@ Some examples of floating point literals:: 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 -Note that numeric literals do not include a sign; a phrase like ``-1`` is -actually an expression composed of the unary operator ``-`` and the literal -``1``. - .. versionchanged:: 3.6 Underscores are now allowed for grouping purposes in literals. @@ -864,4 +874,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/9.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index e152b16ee32d4f..8d17383853a3c4 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -587,7 +587,7 @@ printed:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: @@ -606,7 +606,7 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: @@ -614,9 +614,27 @@ attached as the new exception's :attr:`__context__` attribute:: File "", line 4, in RuntimeError: Something bad happened +Exception chaining can be explicitly suppressed by specifying :const:`None` in +the ``from`` clause:: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + Additional information on exceptions can be found in section :ref:`exceptions`, and information about handling exceptions is in section :ref:`try`. +.. versionchanged:: 3.3 + :const:`None` is now permitted as ``Y`` in ``raise X from Y``. + +.. versionadded:: 3.3 + The ``__suppress_context__`` attribute to suppress automatic display of the + exception context. .. _break: @@ -913,7 +931,7 @@ annotation. .. impl-detail:: - The current implementation does not enforce some of these restriction, but + The current implementation does not enforce some of these restrictions, but programs should not abuse this freedom, as future implementations may enforce them or silently change the meaning of the program. @@ -922,7 +940,7 @@ annotation. builtin: eval builtin: compile -**Programmer's note:** the :keyword:`global` is a directive to the parser. It +**Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`global` statement. In particular, a :keyword:`global` statement contained in a string or code object supplied to the built-in :func:`exec` function does not affect the code diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 273191bbd3c025..8f507e461504bd 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -15,14 +15,15 @@ from time import asctime from pprint import pformat from docutils.io import StringOutput +from docutils.parsers.rst import Directive from docutils.utils import new_document from docutils import nodes, utils from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title -from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter from sphinx.writers.latex import LaTeXTranslator @@ -34,7 +35,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/3.6/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -79,7 +80,7 @@ def new_depart_literal_block(self, node): def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) - text = 'issue ' + issue + text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: @@ -225,7 +235,7 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('([Ii])ssue #([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -253,7 +263,7 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', + content = issue_re.sub(r'`bpo-\1 `__', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js new file mode 100644 index 00000000000000..c2b342b919a28b --- /dev/null +++ b/Doc/tools/static/switchers.js @@ -0,0 +1,144 @@ +(function() { + 'use strict'; + + // Parses versions in URL segments like: + // "3", "dev", "release/2.7" or "3.6rc2" + var version_regexs = [ + '(?:\\d)', + '(?:\\d\\.\\d[\\w\\d\\.]*)', + '(?:dev)', + '(?:release/\\d.\\d[\\x\\d\\.]*)']; + + var all_versions = { + '3.7': 'dev (3.7)', + '3.6': '3.6', + '2.7': '2.7', + }; + + var all_languages = { + 'en': 'English', + 'fr': 'French', + 'ja': 'Japanese', + }; + + function build_version_select(current_version, current_release) { + var buf = [''); + return buf.join(''); + } + + function build_language_select(current_language) { + var buf = [''); + return buf.join(''); + } + + function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + var url = urls.shift(); + if (urls.length == 0) { + window.location.href = url; + return; + } + $.ajax({ + url: url, + success: function() { + window.location.href = url; + }, + error: function() { + navigate_to_first_existing(urls); + } + }); + } + + function on_version_switch() { + var selected_version = $(this).children('option:selected').attr('value') + '/'; + var url = window.location.href; + var current_language = language_segment_from_url(url); + var current_version = version_segment_in_url(url); + var new_url = url.replace('.org/' + current_language + current_version, + '.org/' + current_language + selected_version); + if (new_url != url) { + navigate_to_first_existing([ + new_url, + url.replace('.org/' + current_language + current_version, + '.org/' + selected_version), + 'https://docs.python.org/' + current_language + selected_version, + 'https://docs.python.org/' + selected_version, + 'https://docs.python.org/' + ]); + } + } + + function on_language_switch() { + var selected_language = $(this).children('option:selected').attr('value') + '/'; + var url = window.location.href; + var current_language = language_segment_from_url(url); + var current_version = version_segment_in_url(url); + if (selected_language == 'en/') // Special 'default' case for english. + selected_language = ''; + var new_url = url.replace('.org/' + current_language + current_version, + '.org/' + selected_language + current_version); + if (new_url != url) { + navigate_to_first_existing([ + new_url, + 'https://docs.python.org/' + ]); + } + } + + // Returns the path segment of the language as a string, like 'fr/' + // or '' if not found. + function language_segment_from_url(url) { + var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)'; + var match = url.match(language_regexp); + if (match !== null) + return match[1]; + return ''; + } + + // Returns the path segment of the version as a string, like '3.6/' + // or '' if not found. + function version_segment_in_url(url) { + var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)'; + var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; + var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; + var match = url.match(version_regexp); + if (match !== null) + return match[1]; + return '' + } + + $(document).ready(function() { + var release = DOCUMENTATION_OPTIONS.VERSION; + var language_segment = language_segment_from_url(window.location.href); + var current_language = language_segment.replace(/\/+$/g, '') || 'en'; + var version = release.substr(0, 3); + var version_select = build_version_select(version, release); + + $('.version_switcher_placeholder').html(version_select); + $('.version_switcher_placeholder select').bind('change', on_version_switch); + + var language_select = build_language_select(current_language); + + $('.language_switcher_placeholder').html(language_select); + $('.language_switcher_placeholder select').bind('change', on_language_switch); + }); +})(); diff --git a/Doc/tools/static/version_switch.js b/Doc/tools/static/version_switch.js deleted file mode 100644 index 8b36a61671831d..00000000000000 --- a/Doc/tools/static/version_switch.js +++ /dev/null @@ -1,67 +0,0 @@ -(function() { - 'use strict'; - - var all_versions = { - '3.7': 'dev (3.7)', - '3.6': '3.6', - '3.5': '3.5', - '3.4': '3.4', - '3.3': '3.3', - '2.7': '2.7', - }; - - function build_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function patch_url(url, new_version) { - var url_re = /\.org\/(\d|py3k|dev|((release\/)?\d\.\d[\w\d\.]*))\//, - new_url = url.replace(url_re, '.org/' + new_version + '/'); - - if (new_url == url && !new_url.match(url_re)) { - // python 2 url without version? - new_url = url.replace(/\.org\//, '.org/' + new_version + '/'); - } - return new_url; - } - - function on_switch() { - var selected = $(this).children('option:selected').attr('value'); - - var url = window.location.href, - new_url = patch_url(url, selected); - - if (new_url != url) { - // check beforehand if url exists, else redirect to version's start page - $.ajax({ - url: new_url, - success: function() { - window.location.href = new_url; - }, - error: function() { - window.location.href = 'https://docs.python.org/' + selected; - } - }); - } - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var version = release.substr(0, 3); - var select = build_select(version, release); - - $('.version_switcher_placeholder').html(select); - $('.version_switcher_placeholder select').bind('change', on_switch); - }); -})(); diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e03119ae9730..482ffc0bde8ae7 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -106,7 +106,7 @@ howto/pyporting,,::,Programming Language :: Python :: 2 howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"for example ""joe:password@example.com""" +howto/urllib2,,:password,"""joe:password@example.com""" library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) @@ -130,9 +130,6 @@ library/exceptions,,:err,err.object[err.start:err.end] library/functions,,:step,a[start:stop:step] library/functions,,:stop,"a[start:stop, i]" library/functions,,:stop,a[start:stop:step] -library/hashlib,,:vatrogasac,>>> cookie = b'user:vatrogasac' -library/hashlib,,:vatrogasac,"user:vatrogasac,349cf904533767ed2d755279a8df84d0" -library/hashlib,,:policajac,">>> compare_digest(b'user:policajac', sig)" library/hashlib,,:LEAF,"h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH," library/http.client,,:port,host:port library/http.cookies,,`,!#$%&'*+-.^_`|~: @@ -302,6 +299,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional,">> m[::2].tolist() library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' @@ -323,7 +322,3 @@ whatsnew/3.5,,:warning,'WARNING:root:warning\n' whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception -whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" -whatsnew/changelog,,`,"for readability (was ""`"")." diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 243d8107779361..21af621efce95b 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -3,8 +3,11 @@

{{ _('This Page') }}

{%- endif %} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3abc81c7..d49ebdd14ba8a1 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -18,23 +18,23 @@

Download Python {{ release }} Documentation

- - + + - - + + - - + + - - + + - +
FormatPacked as .zipPacked as .tar.bz2
PDF (US-Letter paper size)Download (ca. 8 MB)Download (ca. 8 MB)Download (ca. 13 MB)Download (ca. 13 MB)
PDF (A4 paper size)Download (ca. 8 MB)Download (ca. 8 MB)Download (ca. 13 MB)Download (ca. 13 MB)
HTMLDownload (ca. 6 MB)Download (ca. 4 MB)Download (ca. 9 MB)Download (ca. 6 MB)
Plain TextDownload (ca. 2 MB)Download (ca. 1.5 MB)Download (ca. 3 MB)Download (ca. 2 MB)
EPUBDownload (ca. 4.5 MB)Download (ca. 5.5 MB)
@@ -42,7 +42,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 00000000000000..6e43be23230b54 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1076c1f51b7d19..d795c0a5586bc8 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -1,5 +1,12 @@ -{% extends "defindex.html" %} -{% block tables %} +{% extends "layout.html" %} +{%- block htmltitle -%} +{{ shorttitle }} +{%- endblock -%} +{% block body %} +

{{ docstitle|e }}

+

+ {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} +

{% trans %}Parts of the documentation:{% endtrans %}

diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 413c0a7699cbe8..a3c4bc8e3a90a7 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -3,7 +3,6 @@

{% trans %}Download{% endtrans %}

{% trans %}Docs for other versions{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 640d8b30493299..c2106678ac60f4 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -4,7 +4,8 @@ style="vertical-align: middle; margin-top: -1px"/>
  • Python{{ reldelim1 }}
  • - {%- if versionswitcher is defined %} + {%- if switchers is defined %} + {{ language or 'en' }} {{ release }} {% trans %}Documentation {% endtrans %}{{ reldelim1 }} {%- else %} @@ -41,7 +42,7 @@ {% if builder != "htmlhelp" %} {% if not embedded %}{% endif %} - {% if versionswitcher is defined and not embedded %}{% endif %} + {% if switchers is defined and not embedded %}{% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %} - + + + +
  • Python »
  • - 3.5.2 Documentation » + 3.7.0a0 Documentation »
  • The Python Standard Library »
    • @@ -90,7 +93,7 @@

    Navigation

    @@ -418,7 +423,7 @@

    25.5.2.2. CompletionsC-space will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some +classes in the current namespaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific.

    If a string of characters is typed, the ACW selection will jump to the entry most closely matching those characters. Entering a tab will @@ -440,7 +445,7 @@

    25.5.2.2. Completions

    25.5.2.3. Calltips

    -

    A calltip is shown when one types ( after the name of an acccessible +

    A calltip is shown when one types ( after the name of an accessible function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or ) is typed. When the cursor is in the argument part of a definition, @@ -532,16 +537,52 @@

    25.5.3.1. Command line usagesys.argv reflects the arguments passed to IDLE itself. +
    +

    25.5.3.2. Startup failure

    +

    IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +‘RESTART’). If the user process fails to connect to the GUI process, it +displays a Tk error box with a ‘cannot connect’ message that directs the +user here. It then exits.

    +

    A common cause of failure is a user-written file with the same name as a +standard library module, such as random.py and tkinter.py. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file.

    +

    Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections.

    +

    Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over.

    +

    A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem.

    +

    When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one’s home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files.

    +

    If IDLE quits with no message, and it was not started from a console, try +starting from a console (python -m idlelib) and see if a message appears.

    +
    -

    25.5.3.2. IDLE-console differences

    -

    As much as possible, the result of executing Python code with IDLE is the -same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, -sys.modules starts with more entries.

    +

    25.5.3.3. IDLE-console differences

    +

    With rare exceptions, the result of executing Python code with IDLE is +intended to be the same as executing the same code in a console window. +However, the different interface and operation occasionally affect +visible results. For instance, sys.modules starts with more entries.

    IDLE also replaces sys.stdin, sys.stdout, and sys.stderr with objects that get input from and send output to the Shell window. -When this window has the focus, it controls the keyboard and screen. -This is normally transparent, but functions that directly access the keyboard +When Shell has the focus, it controls the keyboard and screen. This is +normally transparent, but functions that directly access the keyboard and screen will not work. If sys is reset with importlib.reload(sys), IDLE’s changes are lost and things like input, raw_input, and print will not work correctly.

    @@ -550,15 +591,35 @@

    25.5.3.2. IDLE-console differencesexec to run each statement. As a result, '__builtins__' is always defined for each statement.

    +
    +

    25.5.3.4. Developing tkinter applications

    +

    IDLE is intentionally different from standard Python in order to +facilitate development of tkinter programs. Enter import tkinter as tk; +root = tk.Tk() in standard Python and nothing appears. Enter the same +in IDLE and a tk window appears. In standard Python, one must also enter +root.update() to see the window. IDLE does the equivalent in the +background, about 20 times a second, which is about every 50 milleseconds. +Next enter b = tk.Button(root, text='button'); b.pack(). Again, +nothing visibly changes in standard Python until one enters root.update().

    +

    Most tkinter programs run root.mainloop(), which usually does not +return until the tk app is destroyed. If the program is run with +python -i or from an IDLE editor, a >>> shell prompt does not +appear until mainloop() returns, at which time there is nothing left +to interact with.

    +

    When running a tkinter program from an IDLE editor, one can comment out +the mainloop call. One then gets a shell prompt immediately and can +interact with the live application. One just has to remember to +re-enable the mainloop call when running in standard Python.

    +
    -

    25.5.3.3. Running without a subprocess

    +

    25.5.3.5. Running without a subprocess

    By default, IDLE executes user code in a separate subprocess via a socket, which uses the internal loopback interface. This connection is not externally visible and no data is sent to or received from the Internet. If firewall software complains anyway, you can ignore it.

    If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch.

    If IDLE is started with the -n command line switch it will run in a @@ -590,26 +651,16 @@

    25.5.4.1. Additional help sources

    The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab.

    25.5.4.3. Extensions

    -

    IDLE contains an extension facility. Peferences for extensions can be -changed with Configure Extensions. See the beginning of config-extensions.def -in the idlelib directory for further information. The default extensions -are currently:

    -
      -
    • FormatParagraph
      • -
    • AutoExpand
      • -
    • ZoomHeight
      • -
    • ScriptBinding
      • -
    • CallTips
      • -
    • ParenMatch
      • -
    • AutoComplete
      • -
    • CodeContext
      • -
    • RstripExtension
      • -
    +

    IDLE contains an extension facility. Preferences for extensions can be +changed with the Extensions tab of the preferences dialog. See the +beginning of config-extensions.def in the idlelib directory for further +information. The only current default extension is zzdummy, an example +also used for testing.

    @@ -646,8 +697,10 @@

    Table Of Contents

  • 25.5.3. Startup and code execution
  • 25.5.4. Help and preferences
    - © Copyright 2001-2016, Python Software Foundation. + © Copyright 2001-2017, Python Software Foundation.
    The Python Software Foundation is a non-profit corporation. Please donate.
    - Last updated on Sep 12, 2016. + Last updated on Sep 15, 2017. Found a bug?
    Created using Sphinx 1.3.6. diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index 77e01a31c08f60..fa6112a339444f 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -260,7 +260,7 @@ def copy_strip(): open(dst, 'wb') as out: for line in inn: out.write(line.rstrip() + b'\n') - print('idle.html copied to help.html') + print(f'{src} copied to {dst}') def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 071bd3ec0f219c..77b4b18962066c 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -2,20 +2,34 @@ """ import os -from sys import version +import sys +from platform import python_version, architecture -from tkinter import * +from tkinter import Toplevel, Frame, Label, Button, PhotoImage +from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW, E from idlelib import textview +def build_bits(): + "Return bits for platform." + if sys.platform == 'darwin': + return '64' if sys.maxsize > 2**32 else '32' + else: + return architecture()[0][:2] + + class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title, _htest=False): - """ + def __init__(self, parent, title=None, *, _htest=False, _utest=False): + """Create popup, do not return until tk widget destroyed. + + parent - parent of this dialog + title - string which is title of popup dialog _htest - bool, change box location when running htest + _utest - bool, don't wait_window when running unittest """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -23,127 +37,164 @@ def __init__(self, parent, title, _htest=False): self.geometry("+%d+%d" % ( parent.winfo_rootx()+30, parent.winfo_rooty()+(30 if not _htest else 100))) - self.bg = "#707070" - self.fg = "#ffffff" - self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) - self.title(title) + self.bg = "#bbbbbb" + self.fg = "#000000" + self.create_widgets() + self.resizable(height=False, width=False) + self.title(title or + f'About IDLE {python_version()} ({build_bits()} bit)') self.transient(parent) self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Ok) + self.protocol("WM_DELETE_WINDOW", self.ok) self.parent = parent - self.buttonOk.focus_set() - self.bind('',self.Ok) #dismiss dialog - self.bind('',self.Ok) #dismiss dialog - self.wait_window() - - def CreateWidgets(self): - release = version[:version.index(' ')] - frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - frameButtons = Frame(self) - frameButtons.pack(side=BOTTOM, fill=X) - frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok) - self.buttonOk.pack(padx=5, pady=5) - #self.picture = Image('photo', data=self.pictureData) - frameBg = Frame(frameMain, bg=self.bg) - frameBg.pack(expand=TRUE, fill=BOTH) - labelTitle = Label(frameBg, text='IDLE', fg=self.fg, bg=self.bg, - font=('courier', 24, 'bold')) - labelTitle.grid(row=0, column=0, sticky=W, padx=10, pady=10) - #labelPicture = Label(frameBg, text='[picture]') - #image=self.picture, bg=self.bg) - #labelPicture.grid(row=1, column=1, sticky=W, rowspan=2, - # padx=0, pady=3) - byline = "Python's Integrated DeveLopment Environment" + 5*'\n' - labelDesc = Label(frameBg, text=byline, justify=LEFT, - fg=self.fg, bg=self.bg) - labelDesc.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - labelEmail = Label(frameBg, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - labelEmail.grid(row=6, column=0, columnspan=2, - sticky=W, padx=10, pady=0) - labelWWW = Label(frameBg, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', - justify=LEFT, fg=self.fg, bg=self.bg) - labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + self.button_ok.focus_set() + self.bind('', self.ok) # dismiss dialog + self.bind('', self.ok) # dismiss dialog + self._current_textview = None + self._utest = _utest + + if not _utest: + self.deiconify() + self.wait_window() + + def create_widgets(self): + frame = Frame(self, borderwidth=2, relief=SUNKEN) + frame_buttons = Frame(self) + frame_buttons.pack(side=BOTTOM, fill=X) + frame.pack(side=TOP, expand=True, fill=BOTH) + self.button_ok = Button(frame_buttons, text='Close', + command=self.ok) + self.button_ok.pack(padx=5, pady=5) + + frame_background = Frame(frame, bg=self.bg) + frame_background.pack(expand=True, fill=BOTH) + + header = Label(frame_background, text='IDLE', fg=self.fg, + bg=self.bg, font=('courier', 24, 'bold')) + header.grid(row=0, column=0, sticky=E, padx=10, pady=10) + + tk_patchlevel = self.tk.call('info', 'patchlevel') + ext = '.png' if tk_patchlevel >= '8.6' else '.gif' + icon = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'Icons', f'idle_48{ext}') + self.icon_image = PhotoImage(master=self._root(), file=icon) + logo = Label(frame_background, image=self.icon_image, bg=self.bg) + logo.grid(row=0, column=0, sticky=W, rowspan=2, padx=10, pady=10) + + byline_text = "Python's Integrated Development\nand Learning Environment" + 5*'\n' + byline = Label(frame_background, text=byline_text, justify=LEFT, + fg=self.fg, bg=self.bg) + byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) + email = Label(frame_background, text='email: idle-dev@python.org', + justify=LEFT, fg=self.fg, bg=self.bg) + email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) + docs = Label(frame_background, text='https://docs.python.org/' + + python_version()[:3] + '/library/idle.html', + justify=LEFT, fg=self.fg, bg=self.bg) + docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - labelPythonVer = Label(frameBg, text='Python version: ' + - release, fg=self.fg, bg=self.bg) - labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkVer = self.tk.call('info', 'patchlevel') - labelTkVer = Label(frameBg, text='Tk version: '+ - tkVer, fg=self.fg, bg=self.bg) - labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) - py_button_f = Frame(frameBg, bg=self.bg) - py_button_f.grid(row=10, column=0, columnspan=2, sticky=NSEW) - buttonLicense = Button(py_button_f, text='License', width=8, - highlightbackground=self.bg, - command=self.ShowLicense) - buttonLicense.pack(side=LEFT, padx=10, pady=10) - buttonCopyright = Button(py_button_f, text='Copyright', width=8, + + pyver = Label(frame_background, + text='Python version: ' + python_version(), + fg=self.fg, bg=self.bg) + pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) + tkver = Label(frame_background, text='Tk version: ' + tk_patchlevel, + fg=self.fg, bg=self.bg) + tkver.grid(row=9, column=1, sticky=W, padx=2, pady=0) + py_buttons = Frame(frame_background, bg=self.bg) + py_buttons.grid(row=10, column=0, columnspan=2, sticky=NSEW) + self.py_license = Button(py_buttons, text='License', width=8, + highlightbackground=self.bg, + command=self.show_py_license) + self.py_license.pack(side=LEFT, padx=10, pady=10) + self.py_copyright = Button(py_buttons, text='Copyright', width=8, + highlightbackground=self.bg, + command=self.show_py_copyright) + self.py_copyright.pack(side=LEFT, padx=10, pady=10) + self.py_credits = Button(py_buttons, text='Credits', width=8, highlightbackground=self.bg, - command=self.ShowCopyright) - buttonCopyright.pack(side=LEFT, padx=10, pady=10) - buttonCredits = Button(py_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowPythonCredits) - buttonCredits.pack(side=LEFT, padx=10, pady=10) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + command=self.show_py_credits) + self.py_credits.pack(side=LEFT, padx=10, pady=10) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle_v = Label(frameBg, text='IDLE version: ' + release, - fg=self.fg, bg=self.bg) - idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) - idle_button_f = Frame(frameBg, bg=self.bg) - idle_button_f.grid(row=13, column=0, columnspan=3, sticky=NSEW) - idle_about_b = Button(idle_button_f, text='README', width=8, - highlightbackground=self.bg, - command=self.ShowIDLEAbout) - idle_about_b.pack(side=LEFT, padx=10, pady=10) - idle_news_b = Button(idle_button_f, text='NEWS', width=8, - highlightbackground=self.bg, - command=self.ShowIDLENEWS) - idle_news_b.pack(side=LEFT, padx=10, pady=10) - idle_credits_b = Button(idle_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowIDLECredits) - idle_credits_b.pack(side=LEFT, padx=10, pady=10) - # License, et all, are of type _sitebuiltins._Printer - def ShowLicense(self): + idlever = Label(frame_background, + text='IDLE version: ' + python_version(), + fg=self.fg, bg=self.bg) + idlever.grid(row=12, column=0, sticky=W, padx=10, pady=0) + idle_buttons = Frame(frame_background, bg=self.bg) + idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) + self.readme = Button(idle_buttons, text='README', width=8, + highlightbackground=self.bg, + command=self.show_readme) + self.readme.pack(side=LEFT, padx=10, pady=10) + self.idle_news = Button(idle_buttons, text='NEWS', width=8, + highlightbackground=self.bg, + command=self.show_idle_news) + self.idle_news.pack(side=LEFT, padx=10, pady=10) + self.idle_credits = Button(idle_buttons, text='Credits', width=8, + highlightbackground=self.bg, + command=self.show_idle_credits) + self.idle_credits.pack(side=LEFT, padx=10, pady=10) + + # License, copyright, and credits are of type _sitebuiltins._Printer + def show_py_license(self): + "Handle License button event." self.display_printer_text('About - License', license) - def ShowCopyright(self): + def show_py_copyright(self): + "Handle Copyright button event." self.display_printer_text('About - Copyright', copyright) - def ShowPythonCredits(self): + def show_py_credits(self): + "Handle Python Credits button event." self.display_printer_text('About - Python Credits', credits) # Encode CREDITS.txt to utf-8 for proper version of Loewis. # Specify others as ascii until need utf-8, so catch errors. - def ShowIDLECredits(self): + def show_idle_credits(self): + "Handle Idle Credits button event." self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') - def ShowIDLEAbout(self): + def show_readme(self): + "Handle Readme button event." self.display_file_text('About - Readme', 'README.txt', 'ascii') - def ShowIDLENEWS(self): + def show_idle_news(self): + "Handle News button event." self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') def display_printer_text(self, title, printer): + """Create textview for built-in constants. + + Built-in constants have type _sitebuiltins._Printer. The + text is extracted from the built-in and then sent to a text + viewer with self as the parent and title as the title of + the popup. + """ printer._Printer__setup() text = '\n'.join(printer._Printer__lines) - textview.view_text(self, title, text) + self._current_textview = textview.view_text( + self, title, text, _utest=self._utest) def display_file_text(self, title, filename, encoding=None): + """Create textview for filename. + + The filename needs to be in the current directory. The path + is sent to a text viewer with self as the parent, title as + the title of the popup, and the file encoding. + """ fn = os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) - textview.view_file(self, title, fn, encoding) + self._current_textview = textview.view_file( + self, title, fn, encoding, _utest=self._utest) - def Ok(self, event=None): + def ok(self, event=None): + "Dismiss help_about dialog." self.destroy() diff --git a/Lib/idlelib/idle.bat b/Lib/idlelib/idle.bat index 3d619a37eed366..e77b96e9b5149b 100755 --- a/Lib/idlelib/idle.bat +++ b/Lib/idlelib/idle.bat @@ -1,4 +1,4 @@ -@echo off -rem Start IDLE using the appropriate Python interpreter -set CURRDIR=%~dp0 -start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +@echo off +rem Start IDLE using the appropriate Python interpreter +set CURRDIR=%~dp0 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index dc7a28697c1988..c580fb9e79352a 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -159,3 +159,59 @@ complete, though some tests need improvement. To run all htests, run the htest file from an editor or from the command line with: python -m idlelib.idle_test.htest + + +5. Test Coverage + +Install the coverage package into your Python 3.6 site-packages +directory. (Its exact location depends on the OS). +> python3 -m pip install coverage +(On Windows, replace 'python3 with 'py -3.6' or perhaps just 'python'.) + +The problem with running coverage with repository python is that +coverage uses absolute imports for its submodules, hence it needs to be +in a directory in sys.path. One solution: copy the package to the +directory containing the cpython repository. Call it 'dev'. Then run +coverage either directly or from a script in that directory so that +'dev' is prepended to sys.path. + +Either edit or add dev/.coveragerc so it looks something like this. +--- +# .coveragerc sets coverage options. +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + .*# htest # + if not _utest: + if _htest: +--- +The additions for IDLE are 'branch = True', to test coverage both ways, +and the last three exclude lines, to exclude things peculiar to IDLE +that are not executed during tests. + +A script like the following cover.bat (for Windows) is very handy. +--- +@echo off +rem Usage: cover filename [test_ suffix] # proper case required by coverage +rem filename without .py, 2nd parameter if test is not test_filename +setlocal +set py=f:\dev\3x\pcbuild\win32\python_d.exe +set src=idlelib.%1 +if "%2" EQU "" set tst=f:/dev/3x/Lib/idlelib/idle_test/test_%1.py +if "%2" NEQ "" set tst=f:/dev/ex/Lib/idlelib/idle_test/test_%2.py + +%py% -m coverage run --pylib --source=%src% %tst% +%py% -m coverage report --show-missing +%py% -m coverage html +start htmlcov\3x_Lib_idlelib_%1_py.html +rem Above opens new report; htmlcov\index.html displays report index +--- +The second parameter was added for tests of module x not named test_x. +(There were several before modules were renamed, now only one is left.) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 6f676ae865ccc6..442f55e283a484 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -8,7 +8,7 @@ end the test. In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attrubute is also X (the usual situation). +or function) whose .__name__ attribute is also X (the usual situation). The first parameter of X must be 'parent'. When called, the parent argument will be the root window. X must create a child Toplevel window (or subclass thereof). The Toplevel may be a test widget or @@ -86,7 +86,7 @@ def _wrapper(parent): # htest # "Typing ') should hide the calltip.\n" } -_class_browser_spec = { +_module_browser_spec = { 'file': 'browser', 'kwds': {}, 'msg': "Inspect names of module, class(with superclass if " @@ -306,15 +306,6 @@ def _wrapper(parent): # htest # " is an invalid add page and remove page name.\n" } -TextViewer_spec = { - 'file': 'textview', - 'kwds': {'title': 'Test textview', - 'text':'The quick brown fox jumps over the lazy dog.\n'*35, - '_htest': True}, - 'msg': "Test for read-only property of text.\n" - "Text is selectable. Window is scrollable.", - } - _tooltip_spec = { 'file': 'tooltip', 'kwds': {}, @@ -338,6 +329,15 @@ def _wrapper(parent): # htest # "by printing to the console or the IDLE shell.\n" } +ViewWindow_spec = { + 'file': 'textview', + 'kwds': {'title': 'Test textview', + 'text': 'The quick brown fox jumps over the lazy dog.\n'*35, + '_htest': True}, + 'msg': "Test for read-only property of text.\n" + "Select text, scroll window, close" + } + _widget_redirector_spec = { 'file': 'redirector', 'kwds': {}, diff --git a/Lib/idlelib/idle_test/mock_idle.py b/Lib/idlelib/idle_test/mock_idle.py index c7b49ef02b8851..f279a52fd511f0 100644 --- a/Lib/idlelib/idle_test/mock_idle.py +++ b/Lib/idlelib/idle_test/mock_idle.py @@ -6,28 +6,33 @@ from idlelib.idle_test.mock_tk import Text class Func: - '''Mock function captures args and returns result set by test. + '''Record call, capture args, return/raise result set by test. - Attributes: - self.called - records call even if no args, kwds passed. - self.result - set by init, returned by call. - self.args - captures positional arguments. - self.kwds - captures keyword arguments. + When mock function is called, set or use attributes: + self.called - increment call number even if no args, kwds passed. + self.args - capture positional arguments. + self.kwds - capture keyword arguments. + self.result - return or raise value set in __init__. + self.return_self - return self instead, to mock query class return. - Most common use will probably be to mock methods. + Most common use will probably be to mock instance methods. + Given class instance, can set and delete as instance attribute. Mock_tk.Var and Mbox_func are special variants of this. ''' - def __init__(self, result=None): - self.called = False + def __init__(self, result=None, return_self=False): + self.called = 0 self.result = result + self.return_self = return_self self.args = None self.kwds = None def __call__(self, *args, **kwds): - self.called = True + self.called += 1 self.args = args self.kwds = kwds if isinstance(self.result, BaseException): raise self.result + elif self.return_self: + return self else: return self.result diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py new file mode 100644 index 00000000000000..c8184957821ead --- /dev/null +++ b/Lib/idlelib/idle_test/test_browser.py @@ -0,0 +1,253 @@ +""" Test idlelib.browser. + +Coverage: 88% +(Higher, because should exclude 3 lines that .coveragerc won't exclude.) +""" + +import os.path +import unittest +from idlelib import _pyclbr as pyclbr + +from idlelib import browser, filelist +from idlelib.tree import TreeNode +from test.support import requires +from unittest import mock +from tkinter import Tk +from idlelib.idle_test.mock_idle import Func +from collections import deque + + +class ModuleBrowserTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.mb = browser.ModuleBrowser(cls.root, __file__, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.mb.close() + cls.root.destroy() + del cls.root, cls.mb + + def test_init(self): + mb = self.mb + eq = self.assertEqual + eq(mb.path, __file__) + eq(pyclbr._modules, {}) + self.assertIsInstance(mb.node, TreeNode) + + def test_settitle(self): + mb = self.mb + self.assertIn(os.path.basename(__file__), mb.top.title()) + self.assertEqual(mb.top.iconname(), 'Module Browser') + + def test_rootnode(self): + mb = self.mb + rn = mb.rootnode() + self.assertIsInstance(rn, browser.ModuleBrowserTreeItem) + + def test_close(self): + mb = self.mb + mb.top.destroy = Func() + mb.node.destroy = Func() + mb.close() + self.assertTrue(mb.top.destroy.called) + self.assertTrue(mb.node.destroy.called) + del mb.top.destroy, mb.node.destroy + + +# Nested tree same as in test_pyclbr.py except for supers on C0. C1. +mb = pyclbr +module, fname = 'test', 'test.py' +f0 = mb.Function(module, 'f0', fname, 1) +f1 = mb._nest_function(f0, 'f1', 2) +f2 = mb._nest_function(f1, 'f2', 3) +c1 = mb._nest_class(f0, 'c1', 5) +C0 = mb.Class(module, 'C0', ['base'], fname, 6) +F1 = mb._nest_function(C0, 'F1', 8) +C1 = mb._nest_class(C0, 'C1', 11, ['']) +C2 = mb._nest_class(C1, 'C2', 12) +F3 = mb._nest_function(C2, 'F3', 14) +mock_pyclbr_tree = {'f0': f0, 'C0': C0} + +# Adjust C0.name, C1.name so tests do not depend on order. +browser.transform_children(mock_pyclbr_tree, 'test') # C0(base) +browser.transform_children(C0.children) # C1() + +# The class below checks that the calls above are correct +# and that duplicate calls have no effect. + + +class TransformChildrenTest(unittest.TestCase): + + def test_transform_module_children(self): + eq = self.assertEqual + transform = browser.transform_children + # Parameter matches tree module. + tcl = list(transform(mock_pyclbr_tree, 'test')) + eq(tcl, [f0, C0]) + eq(tcl[0].name, 'f0') + eq(tcl[1].name, 'C0(base)') + # Check that second call does not change suffix. + tcl = list(transform(mock_pyclbr_tree, 'test')) + eq(tcl[1].name, 'C0(base)') + # Nothing to traverse if parameter name isn't same as tree module. + tcl = list(transform(mock_pyclbr_tree, 'different name')) + eq(tcl, []) + + def test_transform_node_children(self): + eq = self.assertEqual + transform = browser.transform_children + # Class with two children, one name altered. + tcl = list(transform(C0.children)) + eq(tcl, [F1, C1]) + eq(tcl[0].name, 'F1') + eq(tcl[1].name, 'C1()') + tcl = list(transform(C0.children)) + eq(tcl[1].name, 'C1()') + # Function with two children. + eq(list(transform(f0.children)), [f1, c1]) + + +class ModuleBrowserTreeItemTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.mbt = browser.ModuleBrowserTreeItem(fname) + + def test_init(self): + self.assertEqual(self.mbt.file, fname) + + def test_gettext(self): + self.assertEqual(self.mbt.GetText(), fname) + + def test_geticonname(self): + self.assertEqual(self.mbt.GetIconName(), 'python') + + def test_isexpandable(self): + self.assertTrue(self.mbt.IsExpandable()) + + def test_listchildren(self): + save_rex = browser.pyclbr.readmodule_ex + save_tc = browser.transform_children + browser.pyclbr.readmodule_ex = Func(result=mock_pyclbr_tree) + browser.transform_children = Func(result=[f0, C0]) + try: + self.assertEqual(self.mbt.listchildren(), [f0, C0]) + finally: + browser.pyclbr.readmodule_ex = save_rex + browser.transform_children = save_tc + + def test_getsublist(self): + mbt = self.mbt + mbt.listchildren = Func(result=[f0, C0]) + sub0, sub1 = mbt.GetSubList() + del mbt.listchildren + self.assertIsInstance(sub0, browser.ChildBrowserTreeItem) + self.assertIsInstance(sub1, browser.ChildBrowserTreeItem) + self.assertEqual(sub0.name, 'f0') + self.assertEqual(sub1.name, 'C0(base)') + + + def test_ondoubleclick(self): + mbt = self.mbt + fopen = browser.file_open = mock.Mock() + + with mock.patch('os.path.exists', return_value=False): + mbt.OnDoubleClick() + fopen.assert_not_called() + + with mock.patch('os.path.exists', return_value=True): + mbt.OnDoubleClick() + fopen.assert_called() + fopen.called_with(fname) + + del browser.file_open + + +class ChildBrowserTreeItemTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + CBT = browser.ChildBrowserTreeItem + cls.cbt_f1 = CBT(f1) + cls.cbt_C1 = CBT(C1) + cls.cbt_F1 = CBT(F1) + + @classmethod + def tearDownClass(cls): + del cls.cbt_C1, cls.cbt_f1, cls.cbt_F1 + + def test_init(self): + eq = self.assertEqual + eq(self.cbt_C1.name, 'C1()') + self.assertFalse(self.cbt_C1.isfunction) + eq(self.cbt_f1.name, 'f1') + self.assertTrue(self.cbt_f1.isfunction) + + def test_gettext(self): + self.assertEqual(self.cbt_C1.GetText(), 'class C1()') + self.assertEqual(self.cbt_f1.GetText(), 'def f1(...)') + + def test_geticonname(self): + self.assertEqual(self.cbt_C1.GetIconName(), 'folder') + self.assertEqual(self.cbt_f1.GetIconName(), 'python') + + def test_isexpandable(self): + self.assertTrue(self.cbt_C1.IsExpandable()) + self.assertTrue(self.cbt_f1.IsExpandable()) + self.assertFalse(self.cbt_F1.IsExpandable()) + + def test_getsublist(self): + eq = self.assertEqual + CBT = browser.ChildBrowserTreeItem + + f1sublist = self.cbt_f1.GetSubList() + self.assertIsInstance(f1sublist[0], CBT) + eq(len(f1sublist), 1) + eq(f1sublist[0].name, 'f2') + + eq(self.cbt_F1.GetSubList(), []) + + def test_ondoubleclick(self): + fopen = browser.file_open = mock.Mock() + goto = fopen.return_value.gotoline = mock.Mock() + self.cbt_F1.OnDoubleClick() + fopen.assert_called() + goto.assert_called() + goto.assert_called_with(self.cbt_F1.obj.lineno) + del browser.file_open + # Failure test would have to raise OSError or AttributeError. + + +class NestedChildrenTest(unittest.TestCase): + "Test that all the nodes in a nested tree are added to the BrowserTree." + + def test_nested(self): + queue = deque() + actual_names = [] + # The tree items are processed in breadth first order. + # Verify that processing each sublist hits every node and + # in the right order. + expected_names = ['f0', 'C0(base)', + 'f1', 'c1', 'F1', 'C1()', + 'f2', 'C2', + 'F3'] + CBT = browser.ChildBrowserTreeItem + queue.extend((CBT(f0), CBT(C0))) + while queue: + cb = queue.popleft() + sublist = cb.GetSubList() + queue.extend(sublist) + self.assertIn(cb.name, cb.GetText()) + self.assertIn(cb.GetIconName(), ('python', 'folder')) + self.assertIs(cb.IsExpandable(), sublist != []) + actual_names.append(cb.name) + self.assertEqual(actual_names, expected_names) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 0b11602ca9e414..f6b71306ac8baa 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -46,6 +46,7 @@ def test_builtins(self): # Python class that inherits builtin methods class List(list): "List() doc" + # Simulate builtin with no docstring for default tip test class SB: __call__ = None @@ -53,11 +54,14 @@ def gtest(obj, out): self.assertEqual(signature(obj), out) if List.__doc__ is not None: - gtest(List, List.__doc__) + gtest(List, List.__doc__) # This and append_doc changed in 3.7. gtest(list.__new__, - 'Create and return a new object. See help(type) for accurate signature.') + '(*args, **kwargs)\nCreate and return a new object.' + ' See help(type) for accurate signature.') gtest(list.__init__, + '(self, /, *args, **kwargs)' + ct._argument_positional + '\n' + 'Initialize self. See help(type(self)) for accurate signature.') + append_doc = "L.append(object) -> None -- append object to end" gtest(list.append, append_doc) gtest([].append, append_doc) @@ -65,6 +69,14 @@ def gtest(obj, out): gtest(types.MethodType, "method(function, instance)") gtest(SB(), default_tip) + import re + p = re.compile('') + gtest(re.sub, '''(pattern, repl, string, count=0, flags=0)\nReturn the string obtained by replacing the leftmost +non-overlapping occurrences of the pattern in string by the +replacement repl. repl can be either a string or a callable; +if a string, backslash escapes in it are processed. If it is +a callable, it's passed the match object and must return''') + gtest(p.sub, '''(repl, string, count=0)\nReturn the string obtained by replacing the leftmost non-overlapping occurrences o...''') def test_signature_wrap(self): if textwrap.TextWrapper.__doc__ is not None: @@ -81,9 +93,9 @@ def f(): pass def test_multiline_docstring(self): # Test fewer lines than max. - self.assertEqual(signature(list), - "list() -> new empty list\n" - "list(iterable) -> new list initialized from iterable's items") + self.assertEqual(signature(range), + "range(stop) -> range object\n" + "range(start, stop[, step]) -> range object") # Test max lines self.assertEqual(signature(bytes), '''\ @@ -132,12 +144,20 @@ def test_starred_parameter(self): # test that starred first parameter is *not* removed from argspec class C: def m1(*args): pass - def m2(**kwds): pass c = C() - for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"), - (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),): + for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"),): self.assertEqual(signature(meth), mtip) + def test_invalid_method_signature(self): + class C: + def m2(**kwargs): pass + class Test: + def __call__(*, a): pass + + mtip = ct._invalid_method + self.assertEqual(signature(C().m2), mtip) + self.assertEqual(signature(Test()), mtip) + def test_non_ascii_name(self): # test that re works to delete a first parameter name that # includes non-ascii chars, such as various forms of A. @@ -156,17 +176,23 @@ def test_attribute_exception(self): class NoCall: def __getattr__(self, name): raise BaseException - class Call(NoCall): + class CallA(NoCall): + def __call__(oui, a, b, c): + pass + class CallB(NoCall): def __call__(self, ci): pass - for meth, mtip in ((NoCall, default_tip), (Call, default_tip), - (NoCall(), ''), (Call(), '(ci)')): + + for meth, mtip in ((NoCall, default_tip), (CallA, default_tip), + (NoCall(), ''), (CallA(), '(a, b, c)'), + (CallB(), '(ci)')): self.assertEqual(signature(meth), mtip) def test_non_callables(self): for obj in (0, 0.0, '0', b'0', [], {}): self.assertEqual(signature(obj), '') + class Get_entityTest(unittest.TestCase): def test_bad_entity(self): self.assertIsNone(ct.get_entity('1/0')) diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index a3fa1a341a1642..abfec7993e0744 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -1,30 +1,608 @@ '''Test idlelib.config. -Much is tested by opening config dialog live or in test_configdialog. -Coverage: 27% +Coverage: 96% (100% for IdleConfParser, IdleUserConfParser*, ConfigChanges). +* Exception is OSError clause in Save method. +Much of IdleConf is also exercised by ConfigDialog and test_configdialog. ''' -from sys import modules -from test.support import captured_stderr -from tkinter import Tk +import copy +import sys +import os +import tempfile +from test.support import captured_stderr, findfile import unittest +from unittest import mock +import idlelib from idlelib import config +from idlelib.idle_test.mock_idle import Func # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. -# Replace user parsers with empty parsers that cannot be saved. +# Replace user parsers with empty parsers that cannot be saved +# due to getting '' as the filename when created. idleConf = config.idleConf usercfg = idleConf.userCfg testcfg = {} -usermain = testcfg['main'] = config.IdleUserConfParser('') # filename +usermain = testcfg['main'] = config.IdleUserConfParser('') userhigh = testcfg['highlight'] = config.IdleUserConfParser('') userkeys = testcfg['keys'] = config.IdleUserConfParser('') +userextn = testcfg['extensions'] = config.IdleUserConfParser('') def setUpModule(): idleConf.userCfg = testcfg + idlelib.testing = True def tearDownModule(): idleConf.userCfg = usercfg + idlelib.testing = False + + +class IdleConfParserTest(unittest.TestCase): + """Test that IdleConfParser works""" + + config = """ + [one] + one = false + two = true + three = 10 + + [two] + one = a string + two = true + three = false + """ + + def test_get(self): + parser = config.IdleConfParser('') + parser.read_string(self.config) + eq = self.assertEqual + + # Test with type argument. + self.assertIs(parser.Get('one', 'one', type='bool'), False) + self.assertIs(parser.Get('one', 'two', type='bool'), True) + eq(parser.Get('one', 'three', type='int'), 10) + eq(parser.Get('two', 'one'), 'a string') + self.assertIs(parser.Get('two', 'two', type='bool'), True) + self.assertIs(parser.Get('two', 'three', type='bool'), False) + + # Test without type should fallback to string. + eq(parser.Get('two', 'two'), 'true') + eq(parser.Get('two', 'three'), 'false') + + # If option not exist, should return None, or default. + self.assertIsNone(parser.Get('not', 'exist')) + eq(parser.Get('not', 'exist', default='DEFAULT'), 'DEFAULT') + + def test_get_option_list(self): + parser = config.IdleConfParser('') + parser.read_string(self.config) + get_list = parser.GetOptionList + self.assertCountEqual(get_list('one'), ['one', 'two', 'three']) + self.assertCountEqual(get_list('two'), ['one', 'two', 'three']) + self.assertEqual(get_list('not exist'), []) + + def test_load_nothing(self): + parser = config.IdleConfParser('') + parser.Load() + self.assertEqual(parser.sections(), []) + + def test_load_file(self): + # Borrow test/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1') + parser = config.IdleConfParser(config_path) + parser.Load() + + self.assertEqual(parser.Get('Foo Bar', 'foo'), 'newbar') + self.assertEqual(parser.GetOptionList('Foo Bar'), ['foo']) + + +class IdleUserConfParserTest(unittest.TestCase): + """Test that IdleUserConfParser works""" + + def new_parser(self, path=''): + return config.IdleUserConfParser(path) + + def test_set_option(self): + parser = self.new_parser() + parser.add_section('Foo') + # Setting new option in existing section should return True. + self.assertTrue(parser.SetOption('Foo', 'bar', 'true')) + # Setting existing option with same value should return False. + self.assertFalse(parser.SetOption('Foo', 'bar', 'true')) + # Setting exiting option with new value should return True. + self.assertTrue(parser.SetOption('Foo', 'bar', 'false')) + self.assertEqual(parser.Get('Foo', 'bar'), 'false') + + # Setting option in new section should create section and return True. + self.assertTrue(parser.SetOption('Bar', 'bar', 'true')) + self.assertCountEqual(parser.sections(), ['Bar', 'Foo']) + self.assertEqual(parser.Get('Bar', 'bar'), 'true') + + def test_remove_option(self): + parser = self.new_parser() + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + + self.assertTrue(parser.RemoveOption('Foo', 'bar')) + self.assertFalse(parser.RemoveOption('Foo', 'bar')) + self.assertFalse(parser.RemoveOption('Not', 'Exist')) + + def test_add_section(self): + parser = self.new_parser() + self.assertEqual(parser.sections(), []) + + # Should not add duplicate section. + # Configparser raises DuplicateError, IdleParser not. + parser.AddSection('Foo') + parser.AddSection('Foo') + parser.AddSection('Bar') + self.assertCountEqual(parser.sections(), ['Bar', 'Foo']) + + def test_remove_empty_sections(self): + parser = self.new_parser() + + parser.AddSection('Foo') + parser.AddSection('Bar') + parser.SetOption('Idle', 'name', 'val') + self.assertCountEqual(parser.sections(), ['Bar', 'Foo', 'Idle']) + parser.RemoveEmptySections() + self.assertEqual(parser.sections(), ['Idle']) + + def test_is_empty(self): + parser = self.new_parser() + + parser.AddSection('Foo') + parser.AddSection('Bar') + self.assertTrue(parser.IsEmpty()) + self.assertEqual(parser.sections(), []) + + parser.SetOption('Foo', 'bar', 'false') + parser.AddSection('Bar') + self.assertFalse(parser.IsEmpty()) + self.assertCountEqual(parser.sections(), ['Foo']) + + def test_remove_file(self): + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 'test.cfg') + parser = self.new_parser(path) + parser.RemoveFile() # Should not raise exception. + + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + parser.Save() + self.assertTrue(os.path.exists(path)) + parser.RemoveFile() + self.assertFalse(os.path.exists(path)) + + def test_save(self): + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 'test.cfg') + parser = self.new_parser(path) + parser.AddSection('Foo') + parser.SetOption('Foo', 'bar', 'true') + + # Should save to path when config is not empty. + self.assertFalse(os.path.exists(path)) + parser.Save() + self.assertTrue(os.path.exists(path)) + + # Should remove the file from disk when config is empty. + parser.remove_section('Foo') + parser.Save() + self.assertFalse(os.path.exists(path)) + + +class IdleConfTest(unittest.TestCase): + """Test for idleConf""" + + @classmethod + def setUpClass(cls): + cls.config_string = {} + + conf = config.IdleConf(_utest=True) + if __name__ != '__main__': + idle_dir = os.path.dirname(__file__) + else: + idle_dir = os.path.abspath(sys.path[0]) + for ctype in conf.config_types: + config_path = os.path.join(idle_dir, '../config-%s.def' % ctype) + with open(config_path, 'r') as f: + cls.config_string[ctype] = f.read() + + cls.orig_warn = config._warn + config._warn = Func() + + @classmethod + def tearDownClass(cls): + config._warn = cls.orig_warn + + def new_config(self, _utest=False): + return config.IdleConf(_utest=_utest) + + def mock_config(self): + """Return a mocked idleConf + + Both default and user config used the same config-*.def + """ + conf = config.IdleConf(_utest=True) + for ctype in conf.config_types: + conf.defaultCfg[ctype] = config.IdleConfParser('') + conf.defaultCfg[ctype].read_string(self.config_string[ctype]) + conf.userCfg[ctype] = config.IdleUserConfParser('') + conf.userCfg[ctype].read_string(self.config_string[ctype]) + + return conf + + @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system') + def test_get_user_cfg_dir_unix(self): + "Test to get user config directory under unix" + conf = self.new_config(_utest=True) + + # Check normal way should success + with mock.patch('os.path.expanduser', return_value='/home/foo'): + with mock.patch('os.path.exists', return_value=True): + self.assertEqual(conf.GetUserCfgDir(), '/home/foo/.idlerc') + + # Check os.getcwd should success + with mock.patch('os.path.expanduser', return_value='~'): + with mock.patch('os.getcwd', return_value='/home/foo/cpython'): + with mock.patch('os.mkdir'): + self.assertEqual(conf.GetUserCfgDir(), + '/home/foo/cpython/.idlerc') + + # Check user dir not exists and created failed should raise SystemExit + with mock.patch('os.path.join', return_value='/path/not/exists'): + with self.assertRaises(SystemExit): + with self.assertRaises(FileNotFoundError): + conf.GetUserCfgDir() + + @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for windows system') + def test_get_user_cfg_dir_windows(self): + "Test to get user config directory under windows" + conf = self.new_config(_utest=True) + + # Check normal way should success + with mock.patch('os.path.expanduser', return_value='C:\\foo'): + with mock.patch('os.path.exists', return_value=True): + self.assertEqual(conf.GetUserCfgDir(), 'C:\\foo\\.idlerc') + + # Check os.getcwd should success + with mock.patch('os.path.expanduser', return_value='~'): + with mock.patch('os.getcwd', return_value='C:\\foo\\cpython'): + with mock.patch('os.mkdir'): + self.assertEqual(conf.GetUserCfgDir(), + 'C:\\foo\\cpython\\.idlerc') + + # Check user dir not exists and created failed should raise SystemExit + with mock.patch('os.path.join', return_value='/path/not/exists'): + with self.assertRaises(SystemExit): + with self.assertRaises(FileNotFoundError): + conf.GetUserCfgDir() + + def test_create_config_handlers(self): + conf = self.new_config(_utest=True) + + # Mock out idle_dir + idle_dir = '/home/foo' + with mock.patch.dict({'__name__': '__foo__'}): + with mock.patch('os.path.dirname', return_value=idle_dir): + conf.CreateConfigHandlers() + + # Check keys are equal + self.assertCountEqual(conf.defaultCfg.keys(), conf.config_types) + self.assertCountEqual(conf.userCfg.keys(), conf.config_types) + + # Check conf parser are correct type + for default_parser in conf.defaultCfg.values(): + self.assertIsInstance(default_parser, config.IdleConfParser) + for user_parser in conf.userCfg.values(): + self.assertIsInstance(user_parser, config.IdleUserConfParser) + + # Check config path are correct + for config_type, parser in conf.defaultCfg.items(): + self.assertEqual(parser.file, + os.path.join(idle_dir, 'config-%s.def' % config_type)) + for config_type, parser in conf.userCfg.items(): + self.assertEqual(parser.file, + os.path.join(conf.userdir, 'config-%s.cfg' % config_type)) + + def test_load_cfg_files(self): + conf = self.new_config(_utest=True) + + # Borrow test/cfgparser.1 from test_configparser. + config_path = findfile('cfgparser.1') + conf.defaultCfg['foo'] = config.IdleConfParser(config_path) + conf.userCfg['foo'] = config.IdleUserConfParser(config_path) + + # Load all config from path + conf.LoadCfgFiles() + + eq = self.assertEqual + + # Check defaultCfg is loaded + eq(conf.defaultCfg['foo'].Get('Foo Bar', 'foo'), 'newbar') + eq(conf.defaultCfg['foo'].GetOptionList('Foo Bar'), ['foo']) + + # Check userCfg is loaded + eq(conf.userCfg['foo'].Get('Foo Bar', 'foo'), 'newbar') + eq(conf.userCfg['foo'].GetOptionList('Foo Bar'), ['foo']) + + def test_save_user_cfg_files(self): + conf = self.mock_config() + + with mock.patch('idlelib.config.IdleUserConfParser.Save') as m: + conf.SaveUserCfgFiles() + self.assertEqual(m.call_count, len(conf.userCfg)) + + def test_get_option(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetOption('main', 'EditorWindow', 'width'), '80') + eq(conf.GetOption('main', 'EditorWindow', 'width', type='int'), 80) + with mock.patch('idlelib.config._warn') as _warn: + eq(conf.GetOption('main', 'EditorWindow', 'font', type='int'), None) + eq(conf.GetOption('main', 'EditorWindow', 'NotExists'), None) + eq(conf.GetOption('main', 'EditorWindow', 'NotExists', default='NE'), 'NE') + eq(_warn.call_count, 4) + + def test_set_option(self): + conf = self.mock_config() + + conf.SetOption('main', 'Foo', 'bar', 'newbar') + self.assertEqual(conf.GetOption('main', 'Foo', 'bar'), 'newbar') + + def test_get_section_list(self): + conf = self.mock_config() + + self.assertCountEqual( + conf.GetSectionList('default', 'main'), + ['General', 'EditorWindow', 'Indent', 'Theme', + 'Keys', 'History', 'HelpFiles']) + self.assertCountEqual( + conf.GetSectionList('user', 'main'), + ['General', 'EditorWindow', 'Indent', 'Theme', + 'Keys', 'History', 'HelpFiles']) + + with self.assertRaises(config.InvalidConfigSet): + conf.GetSectionList('foobar', 'main') + with self.assertRaises(config.InvalidConfigType): + conf.GetSectionList('default', 'notexists') + + def test_get_highlight(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetHighlight('IDLE Classic', 'normal'), {'foreground': '#000000', + 'background': '#ffffff'}) + eq(conf.GetHighlight('IDLE Classic', 'normal', 'fg'), '#000000') + eq(conf.GetHighlight('IDLE Classic', 'normal', 'bg'), '#ffffff') + with self.assertRaises(config.InvalidFgBg): + conf.GetHighlight('IDLE Classic', 'normal', 'fb') + + # Test cursor (this background should be normal-background) + eq(conf.GetHighlight('IDLE Classic', 'cursor'), {'foreground': 'black', + 'background': '#ffffff'}) + + # Test get user themes + conf.SetOption('highlight', 'Foobar', 'normal-foreground', '#747474') + conf.SetOption('highlight', 'Foobar', 'normal-background', '#171717') + with mock.patch('idlelib.config._warn'): + eq(conf.GetHighlight('Foobar', 'normal'), {'foreground': '#747474', + 'background': '#171717'}) + + def test_get_theme_dict(self): + "XXX: NOT YET DONE" + conf = self.mock_config() + + # These two should be the same + self.assertEqual( + conf.GetThemeDict('default', 'IDLE Classic'), + conf.GetThemeDict('user', 'IDLE Classic')) + + with self.assertRaises(config.InvalidTheme): + conf.GetThemeDict('bad', 'IDLE Classic') + + def test_get_current_theme_and_keys(self): + conf = self.mock_config() + + self.assertEqual(conf.CurrentTheme(), conf.current_colors_and_keys('Theme')) + self.assertEqual(conf.CurrentKeys(), conf.current_colors_and_keys('Keys')) + + def test_current_colors_and_keys(self): + conf = self.mock_config() + + self.assertEqual(conf.current_colors_and_keys('Theme'), 'IDLE Classic') + + def test_default_keys(self): + current_platform = sys.platform + conf = self.new_config(_utest=True) + + sys.platform = 'win32' + self.assertEqual(conf.default_keys(), 'IDLE Classic Windows') + + sys.platform = 'darwin' + self.assertEqual(conf.default_keys(), 'IDLE Classic OSX') + + sys.platform = 'some-linux' + self.assertEqual(conf.default_keys(), 'IDLE Modern Unix') + + # Restore platform + sys.platform = current_platform + + def test_get_extensions(self): + userextn.read_string(''' + [ZzDummy] + enable = True + [DISABLE] + enable = False + ''') + eq = self.assertEqual + iGE = idleConf.GetExtensions + eq(iGE(shell_only=True), []) + eq(iGE(), ['ZzDummy']) + eq(iGE(editor_only=True), ['ZzDummy']) + eq(iGE(active_only=False), ['ZzDummy', 'DISABLE']) + eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE']) + userextn.remove_section('ZzDummy') + userextn.remove_section('DISABLE') + + + def test_remove_key_bind_names(self): + conf = self.mock_config() + + self.assertCountEqual( + conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), + ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch','ZzDummy']) + + def test_get_extn_name_for_event(self): + userextn.read_string(''' + [ZzDummy] + enable = True + ''') + eq = self.assertEqual + eq(idleConf.GetExtnNameForEvent('z-in'), 'ZzDummy') + eq(idleConf.GetExtnNameForEvent('z-out'), None) + userextn.remove_section('ZzDummy') + + def test_get_extension_keys(self): + userextn.read_string(''' + [ZzDummy] + enable = True + ''') + self.assertEqual(idleConf.GetExtensionKeys('ZzDummy'), + {'<>': ['']}) + userextn.remove_section('ZzDummy') +# need option key test +## key = [''] if sys.platform == 'darwin' else [''] +## eq(conf.GetExtensionKeys('ZoomHeight'), {'<>': key}) + + def test_get_extension_bindings(self): + userextn.read_string(''' + [ZzDummy] + enable = True + ''') + eq = self.assertEqual + iGEB = idleConf.GetExtensionBindings + eq(iGEB('NotExists'), {}) + expect = {'<>': [''], + '<>': ['']} + eq(iGEB('ZzDummy'), expect) + userextn.remove_section('ZzDummy') + + def test_get_keybinding(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetKeyBinding('IDLE Modern Unix', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Unix', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Windows', '<>'), + ['', '']) + eq(conf.GetKeyBinding('IDLE Classic Mac', '<>'), ['']) + eq(conf.GetKeyBinding('IDLE Classic OSX', '<>'), ['']) + + # Test keybinding not exists + eq(conf.GetKeyBinding('NOT EXISTS', '<>'), []) + eq(conf.GetKeyBinding('IDLE Modern Unix', 'NOT EXISTS'), []) + + def test_get_current_keyset(self): + current_platform = sys.platform + conf = self.mock_config() + + # Ensure that platform isn't darwin + sys.platform = 'some-linux' + self.assertEqual(conf.GetCurrentKeySet(), conf.GetKeySet(conf.CurrentKeys())) + + # This should not be the same, since replace ') + self.assertEqual(conf.GetKeySet('IDLE Modern Unix')['<>'], '') + + def test_is_core_binding(self): + # XXX: Should move out the core keys to config file or other place + conf = self.mock_config() + + self.assertTrue(conf.IsCoreBinding('copy')) + self.assertTrue(conf.IsCoreBinding('cut')) + self.assertTrue(conf.IsCoreBinding('del-word-right')) + self.assertFalse(conf.IsCoreBinding('not-exists')) + + def test_extra_help_source_list(self): + # Test GetExtraHelpSourceList and GetAllExtraHelpSourcesList in same + # place to prevent prepare input data twice. + conf = self.mock_config() + + # Test default with no extra help source + self.assertEqual(conf.GetExtraHelpSourceList('default'), []) + self.assertEqual(conf.GetExtraHelpSourceList('user'), []) + with self.assertRaises(config.InvalidConfigSet): + self.assertEqual(conf.GetExtraHelpSourceList('bad'), []) + self.assertCountEqual( + conf.GetAllExtraHelpSourcesList(), + conf.GetExtraHelpSourceList('default') + conf.GetExtraHelpSourceList('user')) + + # Add help source to user config + conf.userCfg['main'].SetOption('HelpFiles', '4', 'Python;https://python.org') # This is bad input + conf.userCfg['main'].SetOption('HelpFiles', '3', 'Python:https://python.org') # This is bad input + conf.userCfg['main'].SetOption('HelpFiles', '2', 'Pillow;https://pillow.readthedocs.io/en/latest/') + conf.userCfg['main'].SetOption('HelpFiles', '1', 'IDLE;C:/Programs/Python36/Lib/idlelib/help.html') + self.assertEqual(conf.GetExtraHelpSourceList('user'), + [('IDLE', 'C:/Programs/Python36/Lib/idlelib/help.html', '1'), + ('Pillow', 'https://pillow.readthedocs.io/en/latest/', '2'), + ('Python', 'https://python.org', '4')]) + self.assertCountEqual( + conf.GetAllExtraHelpSourcesList(), + conf.GetExtraHelpSourceList('default') + conf.GetExtraHelpSourceList('user')) + + def test_get_font(self): + from test.support import requires + from tkinter import Tk + from tkinter.font import Font + conf = self.mock_config() + + requires('gui') + root = Tk() + root.withdraw() + + f = Font.actual(Font(name='TkFixedFont', exists=True, root=root)) + self.assertEqual( + conf.GetFont(root, 'main', 'EditorWindow'), + (f['family'], 10 if f['size'] <= 0 else f['size'], f['weight'])) + + # Cleanup root + root.destroy() + del root + + def test_get_core_keys(self): + conf = self.mock_config() + + eq = self.assertEqual + eq(conf.GetCoreKeys()['<>'], ['']) + eq(conf.GetCoreKeys()['<>'], ['', '']) + eq(conf.GetCoreKeys()['<>'], ['']) + eq(conf.GetCoreKeys('IDLE Classic Windows')['<>'], + ['', '']) + eq(conf.GetCoreKeys('IDLE Classic OSX')['<>'], ['']) + eq(conf.GetCoreKeys('IDLE Classic Unix')['<>'], + ['', '']) + eq(conf.GetCoreKeys('IDLE Modern Unix')['<>'], + ['', '']) class CurrentColorKeysTest(unittest.TestCase): @@ -138,6 +716,92 @@ def test_user_override_keys(self): userkeys.remove_section('Custom Keys') +class ChangesTest(unittest.TestCase): + + empty = {'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} + + def load(self): # Test_add_option verifies that this works. + changes = self.changes + changes.add_option('main', 'Msec', 'mitem', 'mval') + changes.add_option('highlight', 'Hsec', 'hitem', 'hval') + changes.add_option('keys', 'Ksec', 'kitem', 'kval') + return changes + + loaded = {'main': {'Msec': {'mitem': 'mval'}}, + 'highlight': {'Hsec': {'hitem': 'hval'}}, + 'keys': {'Ksec': {'kitem':'kval'}}, + 'extensions': {}} + + def setUp(self): + self.changes = config.ConfigChanges() + + def test_init(self): + self.assertEqual(self.changes, self.empty) + + def test_add_option(self): + changes = self.load() + self.assertEqual(changes, self.loaded) + changes.add_option('main', 'Msec', 'mitem', 'mval') + self.assertEqual(changes, self.loaded) + + def test_save_option(self): # Static function does not touch changes. + save_option = self.changes.save_option + self.assertTrue(save_option('main', 'Indent', 'what', '0')) + self.assertFalse(save_option('main', 'Indent', 'what', '0')) + self.assertEqual(usermain['Indent']['what'], '0') + + self.assertTrue(save_option('main', 'Indent', 'use-spaces', '0')) + self.assertEqual(usermain['Indent']['use-spaces'], '0') + self.assertTrue(save_option('main', 'Indent', 'use-spaces', '1')) + self.assertFalse(usermain.has_option('Indent', 'use-spaces')) + usermain.remove_section('Indent') + + def test_save_added(self): + changes = self.load() + self.assertTrue(changes.save_all()) + self.assertEqual(usermain['Msec']['mitem'], 'mval') + self.assertEqual(userhigh['Hsec']['hitem'], 'hval') + self.assertEqual(userkeys['Ksec']['kitem'], 'kval') + changes.add_option('main', 'Msec', 'mitem', 'mval') + self.assertFalse(changes.save_all()) + usermain.remove_section('Msec') + userhigh.remove_section('Hsec') + userkeys.remove_section('Ksec') + + def test_save_help(self): + # Any change to HelpFiles overwrites entire section. + changes = self.changes + changes.save_option('main', 'HelpFiles', 'IDLE', 'idledoc') + changes.add_option('main', 'HelpFiles', 'ELDI', 'codeldi') + changes.save_all() + self.assertFalse(usermain.has_option('HelpFiles', 'IDLE')) + self.assertTrue(usermain.has_option('HelpFiles', 'ELDI')) + + def test_save_default(self): # Cover 2nd and 3rd false branches. + changes = self.changes + changes.add_option('main', 'Indent', 'use-spaces', '1') + # save_option returns False; cfg_type_changed remains False. + + # TODO: test that save_all calls usercfg Saves. + + def test_delete_section(self): + changes = self.load() + changes.delete_section('main', 'fake') # Test no exception. + self.assertEqual(changes, self.loaded) # Test nothing deleted. + for cfgtype, section in (('main', 'Msec'), ('keys', 'Ksec')): + testcfg[cfgtype].SetOption(section, 'name', 'value') + changes.delete_section(cfgtype, section) + with self.assertRaises(KeyError): + changes[cfgtype][section] # Test section gone from changes + testcfg[cfgtype][section] # and from mock userCfg. + # TODO test for save call. + + def test_clear(self): + changes = self.load() + changes.clear() + self.assertEqual(changes, self.empty) + + class WarningTest(unittest.TestCase): def test_warn(self): diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index ee3f2c835c98b7..9074e23aab35d1 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -4,29 +4,98 @@ ''' from idlelib import config_key from test.support import requires -requires('gui') +import sys import unittest -from tkinter import Tk, Text +from tkinter import Tk +from idlelib.idle_test.mock_idle import Func +from idlelib.idle_test.mock_tk import Var, Mbox_func -class GetKeysTest(unittest.TestCase): +class ValidationTest(unittest.TestCase): + "Test validation methods: OK, KeysOK, bind_ok." + + class Validator(config_key.GetKeysDialog): + def __init__(self, *args, **kwargs): + config_key.GetKeysDialog.__init__(self, *args, **kwargs) + class listKeysFinal: + get = Func() + self.listKeysFinal = listKeysFinal + GetModifiers = Func() + showerror = Mbox_func() @classmethod def setUpClass(cls): + requires('gui') cls.root = Tk() cls.root.withdraw() + keylist = [[''], ['', '']] + cls.dialog = cls.Validator( + cls.root, 'Title', '<>', keylist, _utest=True) @classmethod def tearDownClass(cls): - cls.root.update() # Stop "can't run event command" warning. + cls.dialog.Cancel() + cls.root.update_idletasks() cls.root.destroy() - del cls.root + del cls.dialog, cls.root + + def setUp(self): + self.dialog.showerror.message = '' + # A test that needs a particular final key value should set it. + # A test that sets a non-blank modifier list should reset it to []. + + def test_ok_empty(self): + self.dialog.keyString.set(' ') + self.dialog.OK() + self.assertEqual(self.dialog.result, '') + self.assertEqual(self.dialog.showerror.message, 'No key specified.') + + def test_ok_good(self): + self.dialog.keyString.set('') + self.dialog.listKeysFinal.get.result = 'F11' + self.dialog.OK() + self.assertEqual(self.dialog.result, '') + self.assertEqual(self.dialog.showerror.message, '') + + def test_keys_no_ending(self): + self.assertFalse(self.dialog.KeysOK('')) + self.assertIn('No modifier', self.dialog.showerror.message) + + def test_keys_no_modifier_ok(self): + self.dialog.listKeysFinal.get.result = 'F11' + self.assertTrue(self.dialog.KeysOK('')) + self.assertEqual(self.dialog.showerror.message, '') + + def test_keys_shift_bad(self): + self.dialog.listKeysFinal.get.result = 'a' + self.dialog.GetModifiers.result = ['Shift'] + self.assertFalse(self.dialog.KeysOK('')) + self.assertIn('shift modifier', self.dialog.showerror.message) + self.dialog.GetModifiers.result = [] + + def test_keys_dup(self): + for mods, final, seq in (([], 'F12', ''), + (['Control'], 'x', ''), + (['Control'], 'X', '')): + with self.subTest(m=mods, f=final, s=seq): + self.dialog.listKeysFinal.get.result = final + self.dialog.GetModifiers.result = mods + self.assertFalse(self.dialog.KeysOK(seq)) + self.assertIn('already in use', self.dialog.showerror.message) + self.dialog.GetModifiers.result = [] + def test_bind_ok(self): + self.assertTrue(self.dialog.bind_ok('')) + self.assertEqual(self.dialog.showerror.message, '') - def test_init(self): - dia = config_key.GetKeysDialog( - self.root, 'test', '<>', [''], _utest=True) - dia.Cancel() + def test_bind_not_ok(self): + self.assertFalse(self.dialog.bind_ok('')) + self.assertIn('not accepted', self.dialog.showerror.message) if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 3f94493733ba0b..dc7f69c2af86b2 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -1,14 +1,17 @@ """Test idlelib.configdialog. Half the class creates dialog, half works with user customizations. -Coverage: 46% just by creating dialog, 56% with current tests. +Coverage: 95%. """ -from idlelib.configdialog import ConfigDialog, idleConf # test import +from idlelib import configdialog from test.support import requires requires('gui') -from tkinter import Tk import unittest -import idlelib.config as config +from unittest import mock +from idlelib.idle_test.mock_idle import Func +from tkinter import Tk, Frame, StringVar, IntVar, BooleanVar, DISABLED, NORMAL +from idlelib import config +from idlelib.configdialog import idleConf, changes, tracers # Tests should not depend on fortuitous user configurations. # They must not affect actual user .cfg files. @@ -21,107 +24,1391 @@ 'extensions': config.IdleUserConfParser(''), } -# ConfigDialog.changedItems is a 3-level hierarchical dictionary of -# pending changes that mirrors the multilevel user config dict. -# For testing, record args in a list for comparison with expected. -changes = [] -class TestDialog(ConfigDialog): - def AddChangedItem(self, *args): - changes.append(args) +root = None +dialog = None +mainpage = changes['main'] +highpage = changes['highlight'] +keyspage = changes['keys'] +extpage = changes['extensions'] def setUpModule(): - global root, configure + global root, dialog idleConf.userCfg = testcfg root = Tk() - root.withdraw() - configure = TestDialog(root, 'Test', _utest=True) - + # root.withdraw() # Comment out, see issue 30870 + dialog = configdialog.ConfigDialog(root, 'Test', _utest=True) def tearDownModule(): - global root, configure - idleConf.userCfg = testcfg - configure.remove_var_callbacks() - del configure + global root, dialog + idleConf.userCfg = usercfg + tracers.detach() + tracers.clear() + changes.clear() + del dialog root.update_idletasks() root.destroy() del root -class FontTabTest(unittest.TestCase): +class FontPageTest(unittest.TestCase): + """Test that font widgets enable users to make font changes. + Test that widget actions set vars, that var changes add three + options to changes and call set_samples, and that set_samples + changes the font of both sample boxes. + """ + @classmethod + def setUpClass(cls): + page = cls.page = dialog.fontpage + dialog.note.select(page) + page.set_samples = Func() # Mask instance method. + + @classmethod + def tearDownClass(cls): + del cls.page.set_samples # Unmask instance method. def setUp(self): changes.clear() - def test_font(self): - # Set values guaranteed not to be defaults. - dfont = idleConf.GetFont(root, 'main', 'EditorWindow') - dsize = str(dfont[1]) - dbold = dfont[2] == 'bold' - configure.fontName.set('Test Font') - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', dsize), - ('main', 'EditorWindow', 'font-bold', dbold)] - self.assertEqual(changes, expected) + def test_load_font_cfg(self): + # Leave widget load test to human visual check. + # TODO Improve checks when add IdleConf.get_font_values. + tracers.detach() + d = self.page + d.font_name.set('Fake') + d.font_size.set('1') + d.font_bold.set(True) + d.set_samples.called = 0 + d.load_font_cfg() + self.assertNotEqual(d.font_name.get(), 'Fake') + self.assertNotEqual(d.font_size.get(), '1') + self.assertFalse(d.font_bold.get()) + self.assertEqual(d.set_samples.called, 1) + tracers.attach() + + def test_fontlist_key(self): + # Up and Down keys should select a new font. + d = self.page + if d.fontlist.size() < 2: + self.skipTest('need at least 2 fonts') + fontlist = d.fontlist + fontlist.activate(0) + font = d.fontlist.get('active') + + # Test Down key. + fontlist.focus_force() + fontlist.update() + fontlist.event_generate('') + fontlist.event_generate('') + + down_font = fontlist.get('active') + self.assertNotEqual(down_font, font) + self.assertIn(d.font_name.get(), down_font.lower()) + + # Test Up key. + fontlist.focus_force() + fontlist.update() + fontlist.event_generate('') + fontlist.event_generate('') + + up_font = fontlist.get('active') + self.assertEqual(up_font, font) + self.assertIn(d.font_name.get(), up_font.lower()) + + def test_fontlist_mouse(self): + # Click on item should select that item. + d = self.page + if d.fontlist.size() < 2: + self.skipTest('need at least 2 fonts') + fontlist = d.fontlist + fontlist.activate(0) + + # Select next item in listbox + fontlist.focus_force() + fontlist.see(1) + fontlist.update() + x, y, dx, dy = fontlist.bbox(1) + x += dx // 2 + y += dy // 2 + fontlist.event_generate('', x=x, y=y) + fontlist.event_generate('', x=x, y=y) + + font1 = fontlist.get(1) + select_font = fontlist.get('anchor') + self.assertEqual(select_font, font1) + self.assertIn(d.font_name.get(), font1.lower()) + + def test_sizelist(self): + # Click on number should select that number + d = self.page + d.sizelist.variable.set(40) + self.assertEqual(d.font_size.get(), '40') + + def test_bold_toggle(self): + # Click on checkbutton should invert it. + d = self.page + d.font_bold.set(False) + d.bold_toggle.invoke() + self.assertTrue(d.font_bold.get()) + d.bold_toggle.invoke() + self.assertFalse(d.font_bold.get()) + + def test_font_set(self): + # Test that setting a font Variable results in 3 provisional + # change entries and a call to set_samples. Use values sure to + # not be defaults. + + default_font = idleConf.GetFont(root, 'main', 'EditorWindow') + default_size = str(default_font[1]) + default_bold = default_font[2] == 'bold' + d = self.page + d.font_size.set(default_size) + d.font_bold.set(default_bold) + d.set_samples.called = 0 + + d.font_name.set('Test Font') + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': default_size, + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 1) + changes.clear() + + d.font_size.set('20') + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 2) changes.clear() - configure.fontSize.set(20) - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', dbold)] - self.assertEqual(changes, expected) + + d.font_bold.set(not default_bold) + expected = {'EditorWindow': {'font': 'Test Font', + 'font-size': '20', + 'font-bold': str(not default_bold)}} + self.assertEqual(mainpage, expected) + self.assertEqual(d.set_samples.called, 3) + + def test_set_samples(self): + d = self.page + del d.set_samples # Unmask method for test + d.font_sample, d.highlight_sample = {}, {} + d.font_name.set('test') + d.font_size.set('5') + d.font_bold.set(1) + expected = {'font': ('test', '5', 'bold')} + + # Test set_samples. + d.set_samples() + self.assertTrue(d.font_sample == d.highlight_sample == expected) + + del d.font_sample, d.highlight_sample + d.set_samples = Func() # Re-mask for other tests. + + +class IndentTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.page = dialog.fontpage + + def test_load_tab_cfg(self): + d = self.page + d.space_num.set(16) + d.load_tab_cfg() + self.assertEqual(d.space_num.get(), 4) + + def test_indent_scale(self): + d = self.page changes.clear() - configure.fontBold.set(not dbold) - expected = [ - ('main', 'EditorWindow', 'font', 'Test Font'), - ('main', 'EditorWindow', 'font-size', '20'), - ('main', 'EditorWindow', 'font-bold', not dbold)] - self.assertEqual(changes, expected) + d.indent_scale.set(20) + self.assertEqual(d.space_num.get(), 16) + self.assertEqual(mainpage, {'Indent': {'num-spaces': '16'}}) + - #def test_sample(self): pass # TODO +class HighPageTest(unittest.TestCase): + """Test that highlight tab widgets enable users to make changes. - def test_tabspace(self): - configure.spaceNum.set(6) - self.assertEqual(changes, [('main', 'Indent', 'num-spaces', 6)]) + Test that widget actions set vars, that var changes add + options to changes and that themes work correctly. + """ + @classmethod + def setUpClass(cls): + page = cls.page = dialog.highpage + dialog.note.select(page) + page.set_theme_type = Func() + page.paint_theme_sample = Func() + page.set_highlight_target = Func() + page.set_color_sample = Func() -class HighlightTest(unittest.TestCase): + @classmethod + def tearDownClass(cls): + d = cls.page + del d.set_theme_type, d.paint_theme_sample + del d.set_highlight_target, d.set_color_sample def setUp(self): + d = self.page + # The following is needed for test_load_key_cfg, _delete_custom_keys. + # This may indicate a defect in some test or function. + for section in idleConf.GetSectionList('user', 'highlight'): + idleConf.userCfg['highlight'].remove_section(section) + changes.clear() + d.set_theme_type.called = 0 + d.paint_theme_sample.called = 0 + d.set_highlight_target.called = 0 + d.set_color_sample.called = 0 + + def test_load_theme_cfg(self): + tracers.detach() + d = self.page + eq = self.assertEqual + + # Use builtin theme with no user themes created. + idleConf.CurrentTheme = mock.Mock(return_value='IDLE Classic') + d.load_theme_cfg() + self.assertTrue(d.theme_source.get()) + # builtinlist sets variable builtin_name to the CurrentTheme default. + eq(d.builtin_name.get(), 'IDLE Classic') + eq(d.custom_name.get(), '- no custom themes -') + eq(d.custom_theme_on.state(), ('disabled',)) + eq(d.set_theme_type.called, 1) + eq(d.paint_theme_sample.called, 1) + eq(d.set_highlight_target.called, 1) + + # Builtin theme with non-empty user theme list. + idleConf.SetOption('highlight', 'test1', 'option', 'value') + idleConf.SetOption('highlight', 'test2', 'option2', 'value2') + d.load_theme_cfg() + eq(d.builtin_name.get(), 'IDLE Classic') + eq(d.custom_name.get(), 'test1') + eq(d.set_theme_type.called, 2) + eq(d.paint_theme_sample.called, 2) + eq(d.set_highlight_target.called, 2) + + # Use custom theme. + idleConf.CurrentTheme = mock.Mock(return_value='test2') + idleConf.SetOption('main', 'Theme', 'default', '0') + d.load_theme_cfg() + self.assertFalse(d.theme_source.get()) + eq(d.builtin_name.get(), 'IDLE Classic') + eq(d.custom_name.get(), 'test2') + eq(d.set_theme_type.called, 3) + eq(d.paint_theme_sample.called, 3) + eq(d.set_highlight_target.called, 3) + + del idleConf.CurrentTheme + tracers.attach() + + def test_theme_source(self): + eq = self.assertEqual + d = self.page + # Test these separately. + d.var_changed_builtin_name = Func() + d.var_changed_custom_name = Func() + # Builtin selected. + d.builtin_theme_on.invoke() + eq(mainpage, {'Theme': {'default': 'True'}}) + eq(d.var_changed_builtin_name.called, 1) + eq(d.var_changed_custom_name.called, 0) + changes.clear() + + # Custom selected. + d.custom_theme_on.state(('!disabled',)) + d.custom_theme_on.invoke() + self.assertEqual(mainpage, {'Theme': {'default': 'False'}}) + eq(d.var_changed_builtin_name.called, 1) + eq(d.var_changed_custom_name.called, 1) + del d.var_changed_builtin_name, d.var_changed_custom_name + + def test_builtin_name(self): + eq = self.assertEqual + d = self.page + item_list = ['IDLE Classic', 'IDLE Dark', 'IDLE New'] + + # Not in old_themes, defaults name to first item. + idleConf.SetOption('main', 'Theme', 'name', 'spam') + d.builtinlist.SetMenu(item_list, 'IDLE Dark') + eq(mainpage, {'Theme': {'name': 'IDLE Classic', + 'name2': 'IDLE Dark'}}) + eq(d.theme_message['text'], 'New theme, see Help') + eq(d.paint_theme_sample.called, 1) + + # Not in old themes - uses name2. + changes.clear() + idleConf.SetOption('main', 'Theme', 'name', 'IDLE New') + d.builtinlist.SetMenu(item_list, 'IDLE Dark') + eq(mainpage, {'Theme': {'name2': 'IDLE Dark'}}) + eq(d.theme_message['text'], 'New theme, see Help') + eq(d.paint_theme_sample.called, 2) + + # Builtin name in old_themes. + changes.clear() + d.builtinlist.SetMenu(item_list, 'IDLE Classic') + eq(mainpage, {'Theme': {'name': 'IDLE Classic', 'name2': ''}}) + eq(d.theme_message['text'], '') + eq(d.paint_theme_sample.called, 3) + + def test_custom_name(self): + d = self.page + + # If no selections, doesn't get added. + d.customlist.SetMenu([], '- no custom themes -') + self.assertNotIn('Theme', mainpage) + self.assertEqual(d.paint_theme_sample.called, 0) + + # Custom name selected. changes.clear() + d.customlist.SetMenu(['a', 'b', 'c'], 'c') + self.assertEqual(mainpage, {'Theme': {'name': 'c'}}) + self.assertEqual(d.paint_theme_sample.called, 1) + + def test_color(self): + d = self.page + d.on_new_color_set = Func() + # self.color is only set in get_color through ColorChooser. + d.color.set('green') + self.assertEqual(d.on_new_color_set.called, 1) + del d.on_new_color_set + + def test_highlight_target_list_mouse(self): + # Set highlight_target through targetlist. + eq = self.assertEqual + d = self.page + + d.targetlist.SetMenu(['a', 'b', 'c'], 'c') + eq(d.highlight_target.get(), 'c') + eq(d.set_highlight_target.called, 1) + + def test_highlight_target_text_mouse(self): + # Set highlight_target through clicking highlight_sample. + eq = self.assertEqual + d = self.page + + elem = {} + count = 0 + hs = d.highlight_sample + hs.focus_force() + hs.see(1.0) + hs.update_idletasks() + + def tag_to_element(elem): + for element, tag in d.theme_elements.items(): + elem[tag[0]] = element - #def test_colorchoose(self): pass # TODO + def click_it(start): + x, y, dx, dy = hs.bbox(start) + x += dx // 2 + y += dy // 2 + hs.event_generate('', x=0, y=0) + hs.event_generate('', x=x, y=y) + hs.event_generate('', x=x, y=y) + hs.event_generate('', x=x, y=y) + # Flip theme_elements to make the tag the key. + tag_to_element(elem) -class KeysTest(unittest.TestCase): + # If highlight_sample has a tag that isn't in theme_elements, there + # will be a KeyError in the test run. + for tag in hs.tag_names(): + for start_index in hs.tag_ranges(tag)[0::2]: + count += 1 + click_it(start_index) + eq(d.highlight_target.get(), elem[tag]) + eq(d.set_highlight_target.called, count) + + def test_set_theme_type(self): + eq = self.assertEqual + d = self.page + del d.set_theme_type + + # Builtin theme selected. + d.theme_source.set(True) + d.set_theme_type() + eq(d.builtinlist['state'], NORMAL) + eq(d.customlist['state'], DISABLED) + eq(d.button_delete_custom.state(), ('disabled',)) + + # Custom theme selected. + d.theme_source.set(False) + d.set_theme_type() + eq(d.builtinlist['state'], DISABLED) + eq(d.custom_theme_on.state(), ('selected',)) + eq(d.customlist['state'], NORMAL) + eq(d.button_delete_custom.state(), ()) + d.set_theme_type = Func() + + def test_get_color(self): + eq = self.assertEqual + d = self.page + orig_chooser = configdialog.tkColorChooser.askcolor + chooser = configdialog.tkColorChooser.askcolor = Func() + gntn = d.get_new_theme_name = Func() + + d.highlight_target.set('Editor Breakpoint') + d.color.set('#ffffff') + + # Nothing selected. + chooser.result = (None, None) + d.button_set_color.invoke() + eq(d.color.get(), '#ffffff') + + # Selection same as previous color. + chooser.result = ('', d.style.lookup(d.frame_color_set['style'], 'background')) + d.button_set_color.invoke() + eq(d.color.get(), '#ffffff') + + # Select different color. + chooser.result = ((222.8671875, 0.0, 0.0), '#de0000') + + # Default theme. + d.color.set('#ffffff') + d.theme_source.set(True) + + # No theme name selected therefore color not saved. + gntn.result = '' + d.button_set_color.invoke() + eq(gntn.called, 1) + eq(d.color.get(), '#ffffff') + # Theme name selected. + gntn.result = 'My New Theme' + d.button_set_color.invoke() + eq(d.custom_name.get(), gntn.result) + eq(d.color.get(), '#de0000') + + # Custom theme. + d.color.set('#ffffff') + d.theme_source.set(False) + d.button_set_color.invoke() + eq(d.color.get(), '#de0000') + + del d.get_new_theme_name + configdialog.tkColorChooser.askcolor = orig_chooser + + def test_on_new_color_set(self): + d = self.page + color = '#3f7cae' + d.custom_name.set('Python') + d.highlight_target.set('Selected Text') + d.fg_bg_toggle.set(True) + + d.color.set(color) + self.assertEqual(d.style.lookup(d.frame_color_set['style'], 'background'), color) + self.assertEqual(d.highlight_sample.tag_cget('hilite', 'foreground'), color) + self.assertEqual(highpage, + {'Python': {'hilite-foreground': color}}) + + def test_get_new_theme_name(self): + orig_sectionname = configdialog.SectionName + sn = configdialog.SectionName = Func(return_self=True) + d = self.page + + sn.result = 'New Theme' + self.assertEqual(d.get_new_theme_name(''), 'New Theme') + + configdialog.SectionName = orig_sectionname + + def test_save_as_new_theme(self): + d = self.page + gntn = d.get_new_theme_name = Func() + d.theme_source.set(True) + + # No name entered. + gntn.result = '' + d.button_save_custom.invoke() + self.assertNotIn(gntn.result, idleConf.userCfg['highlight']) + + # Name entered. + gntn.result = 'my new theme' + gntn.called = 0 + self.assertNotIn(gntn.result, idleConf.userCfg['highlight']) + d.button_save_custom.invoke() + self.assertIn(gntn.result, idleConf.userCfg['highlight']) + + del d.get_new_theme_name + + def test_create_new_and_save_new(self): + eq = self.assertEqual + d = self.page + + # Use default as previously active theme. + d.theme_source.set(True) + d.builtin_name.set('IDLE Classic') + first_new = 'my new custom theme' + second_new = 'my second custom theme' + + # No changes, so themes are an exact copy. + self.assertNotIn(first_new, idleConf.userCfg) + d.create_new(first_new) + eq(idleConf.GetSectionList('user', 'highlight'), [first_new]) + eq(idleConf.GetThemeDict('default', 'IDLE Classic'), + idleConf.GetThemeDict('user', first_new)) + eq(d.custom_name.get(), first_new) + self.assertFalse(d.theme_source.get()) # Use custom set. + eq(d.set_theme_type.called, 1) + + # Test that changed targets are in new theme. + changes.add_option('highlight', first_new, 'hit-background', 'yellow') + self.assertNotIn(second_new, idleConf.userCfg) + d.create_new(second_new) + eq(idleConf.GetSectionList('user', 'highlight'), [first_new, second_new]) + self.assertNotEqual(idleConf.GetThemeDict('user', first_new), + idleConf.GetThemeDict('user', second_new)) + # Check that difference in themes was in `hit-background` from `changes`. + idleConf.SetOption('highlight', first_new, 'hit-background', 'yellow') + eq(idleConf.GetThemeDict('user', first_new), + idleConf.GetThemeDict('user', second_new)) + + def test_set_highlight_target(self): + eq = self.assertEqual + d = self.page + del d.set_highlight_target + + # Target is cursor. + d.highlight_target.set('Cursor') + eq(d.fg_on.state(), ('disabled', 'selected')) + eq(d.bg_on.state(), ('disabled',)) + self.assertTrue(d.fg_bg_toggle) + eq(d.set_color_sample.called, 1) + + # Target is not cursor. + d.highlight_target.set('Comment') + eq(d.fg_on.state(), ('selected',)) + eq(d.bg_on.state(), ()) + self.assertTrue(d.fg_bg_toggle) + eq(d.set_color_sample.called, 2) + + d.set_highlight_target = Func() + + def test_set_color_sample_binding(self): + d = self.page + scs = d.set_color_sample + + d.fg_on.invoke() + self.assertEqual(scs.called, 1) + + d.bg_on.invoke() + self.assertEqual(scs.called, 2) + + def test_set_color_sample(self): + d = self.page + del d.set_color_sample + d.highlight_target.set('Selected Text') + d.fg_bg_toggle.set(True) + d.set_color_sample() + self.assertEqual( + d.style.lookup(d.frame_color_set['style'], 'background'), + d.highlight_sample.tag_cget('hilite', 'foreground')) + d.set_color_sample = Func() + + def test_paint_theme_sample(self): + eq = self.assertEqual + d = self.page + del d.paint_theme_sample + hs_tag = d.highlight_sample.tag_cget + gh = idleConf.GetHighlight + fg = 'foreground' + bg = 'background' + + # Create custom theme based on IDLE Dark. + d.theme_source.set(True) + d.builtin_name.set('IDLE Dark') + theme = 'IDLE Test' + d.create_new(theme) + d.set_color_sample.called = 0 + + # Base theme with nothing in `changes`. + d.paint_theme_sample() + eq(hs_tag('break', fg), gh(theme, 'break', fgBg='fg')) + eq(hs_tag('cursor', bg), gh(theme, 'normal', fgBg='bg')) + self.assertNotEqual(hs_tag('console', fg), 'blue') + self.assertNotEqual(hs_tag('console', bg), 'yellow') + eq(d.set_color_sample.called, 1) + + # Apply changes. + changes.add_option('highlight', theme, 'console-foreground', 'blue') + changes.add_option('highlight', theme, 'console-background', 'yellow') + d.paint_theme_sample() + + eq(hs_tag('break', fg), gh(theme, 'break', fgBg='fg')) + eq(hs_tag('cursor', bg), gh(theme, 'normal', fgBg='bg')) + eq(hs_tag('console', fg), 'blue') + eq(hs_tag('console', bg), 'yellow') + eq(d.set_color_sample.called, 2) + + d.paint_theme_sample = Func() + + def test_delete_custom(self): + eq = self.assertEqual + d = self.page + d.button_delete_custom.state(('!disabled',)) + yesno = d.askyesno = Func() + dialog.deactivate_current_config = Func() + dialog.activate_config_changes = Func() + + theme_name = 'spam theme' + idleConf.userCfg['highlight'].SetOption(theme_name, 'name', 'value') + highpage[theme_name] = {'option': 'True'} + + # Force custom theme. + d.theme_source.set(False) + d.custom_name.set(theme_name) + + # Cancel deletion. + yesno.result = False + d.button_delete_custom.invoke() + eq(yesno.called, 1) + eq(highpage[theme_name], {'option': 'True'}) + eq(idleConf.GetSectionList('user', 'highlight'), ['spam theme']) + eq(dialog.deactivate_current_config.called, 0) + eq(dialog.activate_config_changes.called, 0) + eq(d.set_theme_type.called, 0) + + # Confirm deletion. + yesno.result = True + d.button_delete_custom.invoke() + eq(yesno.called, 2) + self.assertNotIn(theme_name, highpage) + eq(idleConf.GetSectionList('user', 'highlight'), []) + eq(d.custom_theme_on.state(), ('disabled',)) + eq(d.custom_name.get(), '- no custom themes -') + eq(dialog.deactivate_current_config.called, 1) + eq(dialog.activate_config_changes.called, 1) + eq(d.set_theme_type.called, 1) + + del dialog.activate_config_changes, dialog.deactivate_current_config + del d.askyesno + + +class KeysPageTest(unittest.TestCase): + """Test that keys tab widgets enable users to make changes. + + Test that widget actions set vars, that var changes add + options to changes and that key sets works correctly. + """ + + @classmethod + def setUpClass(cls): + page = cls.page = dialog.keyspage + dialog.note.select(page) + page.set_keys_type = Func() + page.load_keys_list = Func() + + @classmethod + def tearDownClass(cls): + page = cls.page + del page.set_keys_type, page.load_keys_list def setUp(self): + d = self.page + # The following is needed for test_load_key_cfg, _delete_custom_keys. + # This may indicate a defect in some test or function. + for section in idleConf.GetSectionList('user', 'keys'): + idleConf.userCfg['keys'].remove_section(section) + changes.clear() + d.set_keys_type.called = 0 + d.load_keys_list.called = 0 + + def test_load_key_cfg(self): + tracers.detach() + d = self.page + eq = self.assertEqual + + # Use builtin keyset with no user keysets created. + idleConf.CurrentKeys = mock.Mock(return_value='IDLE Classic OSX') + d.load_key_cfg() + self.assertTrue(d.keyset_source.get()) + # builtinlist sets variable builtin_name to the CurrentKeys default. + eq(d.builtin_name.get(), 'IDLE Classic OSX') + eq(d.custom_name.get(), '- no custom keys -') + eq(d.custom_keyset_on.state(), ('disabled',)) + eq(d.set_keys_type.called, 1) + eq(d.load_keys_list.called, 1) + eq(d.load_keys_list.args, ('IDLE Classic OSX', )) + + # Builtin keyset with non-empty user keyset list. + idleConf.SetOption('keys', 'test1', 'option', 'value') + idleConf.SetOption('keys', 'test2', 'option2', 'value2') + d.load_key_cfg() + eq(d.builtin_name.get(), 'IDLE Classic OSX') + eq(d.custom_name.get(), 'test1') + eq(d.set_keys_type.called, 2) + eq(d.load_keys_list.called, 2) + eq(d.load_keys_list.args, ('IDLE Classic OSX', )) + + # Use custom keyset. + idleConf.CurrentKeys = mock.Mock(return_value='test2') + idleConf.default_keys = mock.Mock(return_value='IDLE Modern Unix') + idleConf.SetOption('main', 'Keys', 'default', '0') + d.load_key_cfg() + self.assertFalse(d.keyset_source.get()) + eq(d.builtin_name.get(), 'IDLE Modern Unix') + eq(d.custom_name.get(), 'test2') + eq(d.set_keys_type.called, 3) + eq(d.load_keys_list.called, 3) + eq(d.load_keys_list.args, ('test2', )) + + del idleConf.CurrentKeys, idleConf.default_keys + tracers.attach() + + def test_keyset_source(self): + eq = self.assertEqual + d = self.page + # Test these separately. + d.var_changed_builtin_name = Func() + d.var_changed_custom_name = Func() + # Builtin selected. + d.builtin_keyset_on.invoke() + eq(mainpage, {'Keys': {'default': 'True'}}) + eq(d.var_changed_builtin_name.called, 1) + eq(d.var_changed_custom_name.called, 0) + changes.clear() + + # Custom selected. + d.custom_keyset_on.state(('!disabled',)) + d.custom_keyset_on.invoke() + self.assertEqual(mainpage, {'Keys': {'default': 'False'}}) + eq(d.var_changed_builtin_name.called, 1) + eq(d.var_changed_custom_name.called, 1) + del d.var_changed_builtin_name, d.var_changed_custom_name + + def test_builtin_name(self): + eq = self.assertEqual + d = self.page + idleConf.userCfg['main'].remove_section('Keys') + item_list = ['IDLE Classic Windows', 'IDLE Classic OSX', + 'IDLE Modern UNIX'] + + # Not in old_keys, defaults name to first item. + d.builtinlist.SetMenu(item_list, 'IDLE Modern UNIX') + eq(mainpage, {'Keys': {'name': 'IDLE Classic Windows', + 'name2': 'IDLE Modern UNIX'}}) + eq(d.keys_message['text'], 'New key set, see Help') + eq(d.load_keys_list.called, 1) + eq(d.load_keys_list.args, ('IDLE Modern UNIX', )) + + # Not in old keys - uses name2. + changes.clear() + idleConf.SetOption('main', 'Keys', 'name', 'IDLE Classic Unix') + d.builtinlist.SetMenu(item_list, 'IDLE Modern UNIX') + eq(mainpage, {'Keys': {'name2': 'IDLE Modern UNIX'}}) + eq(d.keys_message['text'], 'New key set, see Help') + eq(d.load_keys_list.called, 2) + eq(d.load_keys_list.args, ('IDLE Modern UNIX', )) + + # Builtin name in old_keys. changes.clear() + d.builtinlist.SetMenu(item_list, 'IDLE Classic OSX') + eq(mainpage, {'Keys': {'name': 'IDLE Classic OSX', 'name2': ''}}) + eq(d.keys_message['text'], '') + eq(d.load_keys_list.called, 3) + eq(d.load_keys_list.args, ('IDLE Classic OSX', )) + + def test_custom_name(self): + d = self.page + + # If no selections, doesn't get added. + d.customlist.SetMenu([], '- no custom keys -') + self.assertNotIn('Keys', mainpage) + self.assertEqual(d.load_keys_list.called, 0) + + # Custom name selected. + changes.clear() + d.customlist.SetMenu(['a', 'b', 'c'], 'c') + self.assertEqual(mainpage, {'Keys': {'name': 'c'}}) + self.assertEqual(d.load_keys_list.called, 1) + + def test_keybinding(self): + idleConf.SetOption('extensions', 'ZzDummy', 'enable', 'True') + d = self.page + d.custom_name.set('my custom keys') + d.bindingslist.delete(0, 'end') + d.bindingslist.insert(0, 'copy') + d.bindingslist.insert(1, 'z-in') + d.bindingslist.selection_set(0) + d.bindingslist.selection_anchor(0) + # Core binding - adds to keys. + d.keybinding.set('') + self.assertEqual(keyspage, + {'my custom keys': {'copy': ''}}) + + # Not a core binding - adds to extensions. + d.bindingslist.selection_set(1) + d.bindingslist.selection_anchor(1) + d.keybinding.set('') + self.assertEqual(extpage, + {'ZzDummy_cfgBindings': {'z-in': ''}}) + + def test_set_keys_type(self): + eq = self.assertEqual + d = self.page + del d.set_keys_type + + # Builtin keyset selected. + d.keyset_source.set(True) + d.set_keys_type() + eq(d.builtinlist['state'], NORMAL) + eq(d.customlist['state'], DISABLED) + eq(d.button_delete_custom_keys.state(), ('disabled',)) + + # Custom keyset selected. + d.keyset_source.set(False) + d.set_keys_type() + eq(d.builtinlist['state'], DISABLED) + eq(d.custom_keyset_on.state(), ('selected',)) + eq(d.customlist['state'], NORMAL) + eq(d.button_delete_custom_keys.state(), ()) + d.set_keys_type = Func() + + def test_get_new_keys(self): + eq = self.assertEqual + d = self.page + orig_getkeysdialog = configdialog.GetKeysDialog + gkd = configdialog.GetKeysDialog = Func(return_self=True) + gnkn = d.get_new_keys_name = Func() + + d.button_new_keys.state(('!disabled',)) + d.bindingslist.delete(0, 'end') + d.bindingslist.insert(0, 'copy - ') + d.bindingslist.selection_set(0) + d.bindingslist.selection_anchor(0) + d.keybinding.set('Key-a') + d.keyset_source.set(True) # Default keyset. + + # Default keyset; no change to binding. + gkd.result = '' + d.button_new_keys.invoke() + eq(d.bindingslist.get('anchor'), 'copy - ') + # Keybinding isn't changed when there isn't a change entered. + eq(d.keybinding.get(), 'Key-a') + + # Default keyset; binding changed. + gkd.result = '' + # No keyset name selected therefore binding not saved. + gnkn.result = '' + d.button_new_keys.invoke() + eq(gnkn.called, 1) + eq(d.bindingslist.get('anchor'), 'copy - ') + # Keyset name selected. + gnkn.result = 'My New Key Set' + d.button_new_keys.invoke() + eq(d.custom_name.get(), gnkn.result) + eq(d.bindingslist.get('anchor'), 'copy - ') + eq(d.keybinding.get(), '') + + # User keyset; binding changed. + d.keyset_source.set(False) # Custom keyset. + gnkn.called = 0 + gkd.result = '' + d.button_new_keys.invoke() + eq(gnkn.called, 0) + eq(d.bindingslist.get('anchor'), 'copy - ') + eq(d.keybinding.get(), '') + + del d.get_new_keys_name + configdialog.GetKeysDialog = orig_getkeysdialog + + def test_get_new_keys_name(self): + orig_sectionname = configdialog.SectionName + sn = configdialog.SectionName = Func(return_self=True) + d = self.page + + sn.result = 'New Keys' + self.assertEqual(d.get_new_keys_name(''), 'New Keys') + + configdialog.SectionName = orig_sectionname + + def test_save_as_new_key_set(self): + d = self.page + gnkn = d.get_new_keys_name = Func() + d.keyset_source.set(True) + + # No name entered. + gnkn.result = '' + d.button_save_custom_keys.invoke() + + # Name entered. + gnkn.result = 'my new key set' + gnkn.called = 0 + self.assertNotIn(gnkn.result, idleConf.userCfg['keys']) + d.button_save_custom_keys.invoke() + self.assertIn(gnkn.result, idleConf.userCfg['keys']) + + del d.get_new_keys_name + + def test_on_bindingslist_select(self): + d = self.page + b = d.bindingslist + b.delete(0, 'end') + b.insert(0, 'copy') + b.insert(1, 'find') + b.activate(0) + + b.focus_force() + b.see(1) + b.update() + x, y, dx, dy = b.bbox(1) + x += dx // 2 + y += dy // 2 + b.event_generate('', x=0, y=0) + b.event_generate('', x=x, y=y) + b.event_generate('', x=x, y=y) + b.event_generate('', x=x, y=y) + self.assertEqual(b.get('anchor'), 'find') + self.assertEqual(d.button_new_keys.state(), ()) + + def test_create_new_key_set_and_save_new_key_set(self): + eq = self.assertEqual + d = self.page + + # Use default as previously active keyset. + d.keyset_source.set(True) + d.builtin_name.set('IDLE Classic Windows') + first_new = 'my new custom key set' + second_new = 'my second custom keyset' + + # No changes, so keysets are an exact copy. + self.assertNotIn(first_new, idleConf.userCfg) + d.create_new_key_set(first_new) + eq(idleConf.GetSectionList('user', 'keys'), [first_new]) + eq(idleConf.GetKeySet('IDLE Classic Windows'), + idleConf.GetKeySet(first_new)) + eq(d.custom_name.get(), first_new) + self.assertFalse(d.keyset_source.get()) # Use custom set. + eq(d.set_keys_type.called, 1) + + # Test that changed keybindings are in new keyset. + changes.add_option('keys', first_new, 'copy', '') + self.assertNotIn(second_new, idleConf.userCfg) + d.create_new_key_set(second_new) + eq(idleConf.GetSectionList('user', 'keys'), [first_new, second_new]) + self.assertNotEqual(idleConf.GetKeySet(first_new), + idleConf.GetKeySet(second_new)) + # Check that difference in keysets was in option `copy` from `changes`. + idleConf.SetOption('keys', first_new, 'copy', '') + eq(idleConf.GetKeySet(first_new), idleConf.GetKeySet(second_new)) + + def test_load_keys_list(self): + eq = self.assertEqual + d = self.page + gks = idleConf.GetKeySet = Func() + del d.load_keys_list + b = d.bindingslist + + b.delete(0, 'end') + b.insert(0, '<>') + b.insert(1, '<>') + gks.result = {'<>': ['', ''], + '<>': [''], + '<>': ['']} + changes.add_option('keys', 'my keys', 'spam', '') + expected = ('copy - ', + 'force-open-completions - ', + 'spam - ') + + # No current selection. + d.load_keys_list('my keys') + eq(b.get(0, 'end'), expected) + eq(b.get('anchor'), '') + eq(b.curselection(), ()) + + # Check selection. + b.selection_set(1) + b.selection_anchor(1) + d.load_keys_list('my keys') + eq(b.get(0, 'end'), expected) + eq(b.get('anchor'), 'force-open-completions - ') + eq(b.curselection(), (1, )) + + # Change selection. + b.selection_set(2) + b.selection_anchor(2) + d.load_keys_list('my keys') + eq(b.get(0, 'end'), expected) + eq(b.get('anchor'), 'spam - ') + eq(b.curselection(), (2, )) + d.load_keys_list = Func() + + del idleConf.GetKeySet + + def test_delete_custom_keys(self): + eq = self.assertEqual + d = self.page + d.button_delete_custom_keys.state(('!disabled',)) + yesno = d.askyesno = Func() + dialog.deactivate_current_config = Func() + dialog.activate_config_changes = Func() + + keyset_name = 'spam key set' + idleConf.userCfg['keys'].SetOption(keyset_name, 'name', 'value') + keyspage[keyset_name] = {'option': 'True'} + + # Force custom keyset. + d.keyset_source.set(False) + d.custom_name.set(keyset_name) + + # Cancel deletion. + yesno.result = False + d.button_delete_custom_keys.invoke() + eq(yesno.called, 1) + eq(keyspage[keyset_name], {'option': 'True'}) + eq(idleConf.GetSectionList('user', 'keys'), ['spam key set']) + eq(dialog.deactivate_current_config.called, 0) + eq(dialog.activate_config_changes.called, 0) + eq(d.set_keys_type.called, 0) + + # Confirm deletion. + yesno.result = True + d.button_delete_custom_keys.invoke() + eq(yesno.called, 2) + self.assertNotIn(keyset_name, keyspage) + eq(idleConf.GetSectionList('user', 'keys'), []) + eq(d.custom_keyset_on.state(), ('disabled',)) + eq(d.custom_name.get(), '- no custom keys -') + eq(dialog.deactivate_current_config.called, 1) + eq(dialog.activate_config_changes.called, 1) + eq(d.set_keys_type.called, 1) + del dialog.activate_config_changes, dialog.deactivate_current_config + del d.askyesno -class GeneralTest(unittest.TestCase): + +class GenPageTest(unittest.TestCase): + """Test that general tab widgets enable users to make changes. + + Test that widget actions set vars, that var changes add + options to changes and that helplist works correctly. + """ + @classmethod + def setUpClass(cls): + page = cls.page = dialog.genpage + dialog.note.select(page) + page.set = page.set_add_delete_state = Func() + page.upc = page.update_help_changes = Func() + + @classmethod + def tearDownClass(cls): + page = cls.page + del page.set, page.set_add_delete_state + del page.upc, page.update_help_changes + page.helplist.delete(0, 'end') + page.user_helplist.clear() def setUp(self): changes.clear() + def test_load_general_cfg(self): + # Set to wrong values, load, check right values. + eq = self.assertEqual + d = self.page + d.startup_edit.set(1) + d.autosave.set(1) + d.win_width.set(1) + d.win_height.set(1) + d.helplist.insert('end', 'bad') + d.user_helplist = ['bad', 'worse'] + idleConf.SetOption('main', 'HelpFiles', '1', 'name;file') + d.load_general_cfg() + eq(d.startup_edit.get(), 0) + eq(d.autosave.get(), 0) + eq(d.win_width.get(), '80') + eq(d.win_height.get(), '40') + eq(d.helplist.get(0, 'end'), ('name',)) + eq(d.user_helplist, [('name', 'file', '1')]) + def test_startup(self): - configure.radioStartupEdit.invoke() - self.assertEqual(changes, - [('main', 'General', 'editor-on-startup', 1)]) + d = self.page + d.startup_editor_on.invoke() + self.assertEqual(mainpage, + {'General': {'editor-on-startup': '1'}}) + changes.clear() + d.startup_shell_on.invoke() + self.assertEqual(mainpage, + {'General': {'editor-on-startup': '0'}}) + + def test_editor_size(self): + d = self.page + d.win_height_int.delete(0, 'end') + d.win_height_int.insert(0, '11') + self.assertEqual(mainpage, {'EditorWindow': {'height': '11'}}) + changes.clear() + d.win_width_int.delete(0, 'end') + d.win_width_int.insert(0, '11') + self.assertEqual(mainpage, {'EditorWindow': {'width': '11'}}) + + def test_autocomplete_wait(self): + self.page.auto_wait_int.delete(0, 'end') + self.page.auto_wait_int.insert(0, '11') + self.assertEqual(extpage, {'AutoComplete': {'popupwait': '11'}}) + + def test_parenmatch(self): + d = self.page + eq = self.assertEqual + d.paren_style_type['menu'].invoke(0) + eq(extpage, {'ParenMatch': {'style': 'opener'}}) + changes.clear() + d.paren_flash_time.delete(0, 'end') + d.paren_flash_time.insert(0, '11') + eq(extpage, {'ParenMatch': {'flash-delay': '11'}}) + changes.clear() + d.bell_on.invoke() + eq(extpage, {'ParenMatch': {'bell': 'False'}}) def test_autosave(self): - configure.radioSaveAuto.invoke() - self.assertEqual(changes, [('main', 'General', 'autosave', 1)]) + d = self.page + d.save_auto_on.invoke() + self.assertEqual(mainpage, {'General': {'autosave': '1'}}) + d.save_ask_on.invoke() + self.assertEqual(mainpage, {'General': {'autosave': '0'}}) - def test_editor_size(self): - configure.entryWinHeight.insert(0, '1') - self.assertEqual(changes, [('main', 'EditorWindow', 'height', '140')]) + def test_paragraph(self): + self.page.format_width_int.delete(0, 'end') + self.page.format_width_int.insert(0, '11') + self.assertEqual(extpage, {'FormatParagraph': {'max-width': '11'}}) + + def test_context(self): + self.page.context_int.delete(0, 'end') + self.page.context_int.insert(0, '1') + self.assertEqual(extpage, {'CodeContext': {'numlines': '1'}}) + + def test_source_selected(self): + d = self.page + d.set = d.set_add_delete_state + d.upc = d.update_help_changes + helplist = d.helplist + dex = 'end' + helplist.insert(dex, 'source') + helplist.activate(dex) + + helplist.focus_force() + helplist.see(dex) + helplist.update() + x, y, dx, dy = helplist.bbox(dex) + x += dx // 2 + y += dy // 2 + d.set.called = d.upc.called = 0 + helplist.event_generate('', x=0, y=0) + helplist.event_generate('', x=x, y=y) + helplist.event_generate('', x=x, y=y) + helplist.event_generate('', x=x, y=y) + self.assertEqual(helplist.get('anchor'), 'source') + self.assertTrue(d.set.called) + self.assertFalse(d.upc.called) + + def test_set_add_delete_state(self): + # Call with 0 items, 1 unselected item, 1 selected item. + eq = self.assertEqual + d = self.page + del d.set_add_delete_state # Unmask method. + sad = d.set_add_delete_state + h = d.helplist + + h.delete(0, 'end') + sad() + eq(d.button_helplist_edit.state(), ('disabled',)) + eq(d.button_helplist_remove.state(), ('disabled',)) + + h.insert(0, 'source') + sad() + eq(d.button_helplist_edit.state(), ('disabled',)) + eq(d.button_helplist_remove.state(), ('disabled',)) + + h.selection_set(0) + sad() + eq(d.button_helplist_edit.state(), ()) + eq(d.button_helplist_remove.state(), ()) + d.set_add_delete_state = Func() # Mask method. + + def test_helplist_item_add(self): + # Call without and twice with HelpSource result. + # Double call enables check on order. + eq = self.assertEqual + orig_helpsource = configdialog.HelpSource + hs = configdialog.HelpSource = Func(return_self=True) + d = self.page + d.helplist.delete(0, 'end') + d.user_helplist.clear() + d.set.called = d.upc.called = 0 + + hs.result = '' + d.helplist_item_add() + self.assertTrue(list(d.helplist.get(0, 'end')) == + d.user_helplist == []) + self.assertFalse(d.upc.called) + + hs.result = ('name1', 'file1') + d.helplist_item_add() + hs.result = ('name2', 'file2') + d.helplist_item_add() + eq(d.helplist.get(0, 'end'), ('name1', 'name2')) + eq(d.user_helplist, [('name1', 'file1'), ('name2', 'file2')]) + eq(d.upc.called, 2) + self.assertFalse(d.set.called) + + configdialog.HelpSource = orig_helpsource + + def test_helplist_item_edit(self): + # Call without and with HelpSource change. + eq = self.assertEqual + orig_helpsource = configdialog.HelpSource + hs = configdialog.HelpSource = Func(return_self=True) + d = self.page + d.helplist.delete(0, 'end') + d.helplist.insert(0, 'name1') + d.helplist.selection_set(0) + d.helplist.selection_anchor(0) + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.set.called = d.upc.called = 0 + + hs.result = '' + d.helplist_item_edit() + hs.result = ('name1', 'file1') + d.helplist_item_edit() + eq(d.helplist.get(0, 'end'), ('name1',)) + eq(d.user_helplist, [('name1', 'file1')]) + self.assertFalse(d.upc.called) + + hs.result = ('name2', 'file2') + d.helplist_item_edit() + eq(d.helplist.get(0, 'end'), ('name2',)) + eq(d.user_helplist, [('name2', 'file2')]) + self.assertTrue(d.upc.called == d.set.called == 1) + + configdialog.HelpSource = orig_helpsource + + def test_helplist_item_remove(self): + eq = self.assertEqual + d = self.page + d.helplist.delete(0, 'end') + d.helplist.insert(0, 'name1') + d.helplist.selection_set(0) + d.helplist.selection_anchor(0) + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.set.called = d.upc.called = 0 + + d.helplist_item_remove() + eq(d.helplist.get(0, 'end'), ()) + eq(d.user_helplist, []) + self.assertTrue(d.upc.called == d.set.called == 1) + + def test_update_help_changes(self): + d = self.page + del d.update_help_changes + d.user_helplist.clear() + d.user_helplist.append(('name1', 'file1')) + d.user_helplist.append(('name2', 'file2')) + + d.update_help_changes() + self.assertEqual(mainpage['HelpFiles'], + {'1': 'name1;file1', '2': 'name2;file2'}) + d.update_help_changes = Func() + + +class VarTraceTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tracers = configdialog.VarTrace() + cls.iv = IntVar(root) + cls.bv = BooleanVar(root) + + @classmethod + def tearDownClass(cls): + del cls.tracers, cls.iv, cls.bv + + def setUp(self): + self.tracers.clear() + self.called = 0 + + def var_changed_increment(self, *params): + self.called += 13 + + def var_changed_boolean(self, *params): + pass + + def test_init(self): + tr = self.tracers + tr.__init__() + self.assertEqual(tr.untraced, []) + self.assertEqual(tr.traced, []) + + def test_clear(self): + tr = self.tracers + tr.untraced.append(0) + tr.traced.append(1) + tr.clear() + self.assertEqual(tr.untraced, []) + self.assertEqual(tr.traced, []) + + def test_add(self): + tr = self.tracers + func = Func() + cb = tr.make_callback = mock.Mock(return_value=func) + + iv = tr.add(self.iv, self.var_changed_increment) + self.assertIs(iv, self.iv) + bv = tr.add(self.bv, self.var_changed_boolean) + self.assertIs(bv, self.bv) + + sv = StringVar(root) + sv2 = tr.add(sv, ('main', 'section', 'option')) + self.assertIs(sv2, sv) + cb.assert_called_once() + cb.assert_called_with(sv, ('main', 'section', 'option')) + + expected = [(iv, self.var_changed_increment), + (bv, self.var_changed_boolean), + (sv, func)] + self.assertEqual(tr.traced, []) + self.assertEqual(tr.untraced, expected) + + del tr.make_callback + + def test_make_callback(self): + cb = self.tracers.make_callback(self.iv, ('main', 'section', 'option')) + self.assertTrue(callable(cb)) + self.iv.set(42) + # Not attached, so set didn't invoke the callback. + self.assertNotIn('section', changes['main']) + # Invoke callback manually. + cb() + self.assertIn('section', changes['main']) + self.assertEqual(changes['main']['section']['option'], '42') changes.clear() - configure.entryWinWidth.insert(0, '1') - self.assertEqual(changes, [('main', 'EditorWindow', 'width', '180')]) - #def test_help_sources(self): pass # TODO + def test_attach_detach(self): + tr = self.tracers + iv = tr.add(self.iv, self.var_changed_increment) + bv = tr.add(self.bv, self.var_changed_boolean) + expected = [(iv, self.var_changed_increment), + (bv, self.var_changed_boolean)] + + # Attach callbacks and test call increment. + tr.attach() + self.assertEqual(tr.untraced, []) + self.assertCountEqual(tr.traced, expected) + iv.set(1) + self.assertEqual(iv.get(), 1) + self.assertEqual(self.called, 13) + + # Check that only one callback is attached to a variable. + # If more than one callback were attached, then var_changed_increment + # would be called twice and the counter would be 2. + self.called = 0 + tr.attach() + iv.set(1) + self.assertEqual(self.called, 13) + + # Detach callbacks. + self.called = 0 + tr.detach() + self.assertEqual(tr.traced, []) + self.assertCountEqual(tr.untraced, expected) + iv.set(1) + self.assertEqual(self.called, 0) if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index e9d29d45b82345..64a2a88b7e3765 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -1,7 +1,5 @@ import unittest -from tkinter import Tk, Text from idlelib.editor import EditorWindow -from test.support import requires class Editor_func_test(unittest.TestCase): def test_filename_to_unicode(self): diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 843efb9ad2458c..1f67aaddb3b411 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -1,24 +1,150 @@ '''Test idlelib.help_about. -Coverage: +Coverage: 100% ''' -from idlelib import help_about -from idlelib import textview +from test.support import requires, findfile +from tkinter import Tk, TclError +import unittest +from unittest import mock from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func -import unittest +from idlelib.help_about import AboutDialog as About +from idlelib import help_about +from idlelib import textview +import os.path +from platform import python_version, architecture + + +class LiveDialogTest(unittest.TestCase): + """Simulate user clicking buttons other than [Close]. + + Test that invoked textview has text from source. + """ + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_build_bits(self): + self.assertIn(help_about.build_bits(), ('32', '64')) + + def test_dialog_title(self): + """Test about dialog title""" + self.assertEqual(self.dialog.title(), 'About IDLE') + + def test_dialog_logo(self): + """Test about dialog logo.""" + path, file = os.path.split(self.dialog.icon_image['file']) + fn, ext = os.path.splitext(file) + self.assertEqual(fn, 'idle_48') + + def test_printer_buttons(self): + """Test buttons whose commands use printer function.""" + dialog = self.dialog + button_sources = [(dialog.py_license, license), + (dialog.py_copyright, copyright), + (dialog.py_credits, credits)] + + for button, printer in button_sources: + printer._Printer__setup() + button.invoke() + get = dialog._current_textview.viewframe.textframe.text.get + self.assertEqual(printer._Printer__lines[0], get('1.0', '1.end')) + self.assertEqual( + printer._Printer__lines[1], get('2.0', '2.end')) + dialog._current_textview.destroy() + + def test_file_buttons(self): + """Test buttons that display files.""" + dialog = self.dialog + button_sources = [(self.dialog.readme, 'README.txt'), + (self.dialog.idle_news, 'NEWS.txt'), + (self.dialog.idle_credits, 'CREDITS.txt')] + + for button, filename in button_sources: + button.invoke() + fn = findfile(filename, subdir='idlelib') + get = dialog._current_textview.viewframe.textframe.text.get + with open(fn) as f: + self.assertEqual(f.readline().strip(), get('1.0', '1.end')) + f.readline() + self.assertEqual(f.readline().strip(), get('3.0', '3.end')) + dialog._current_textview.destroy() + + +class DefaultTitleTest(unittest.TestCase): + "Test default title." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_dialog_title(self): + """Test about dialog title""" + self.assertEqual(self.dialog.title(), + f'About IDLE {python_version()}' + f' ({help_about.build_bits()} bit)') + + +class CloseTest(unittest.TestCase): + """Simulate user clicking [Close] button""" + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_close(self): + self.assertEqual(self.dialog.winfo_class(), 'Toplevel') + self.dialog.button_ok.invoke() + with self.assertRaises(TclError): + self.dialog.winfo_class() + -About = help_about.AboutDialog class Dummy_about_dialog(): # Dummy class for testing file display functions. - idle_credits = About.ShowIDLECredits - idle_readme = About.ShowIDLEAbout - idle_news = About.ShowIDLENEWS + idle_credits = About.show_idle_credits + idle_readme = About.show_readme + idle_news = About.show_idle_news # Called by the above display_file_text = About.display_file_text + _utest = True class DisplayFileTest(unittest.TestCase): + """Test functions that display files. + + While somewhat redundant with gui-based test_file_dialog, + these unit tests run on all buildbots, not just a few. + """ dialog = Dummy_about_dialog() @classmethod @@ -29,14 +155,13 @@ def setUpClass(cls): cls.view = Func() textview.showerror = cls.error textview.view_text = cls.view - cls.About = Dummy_about_dialog() @classmethod def tearDownClass(cls): textview.showerror = cls.orig_error textview.view_text = cls.orig_view - def test_file_isplay(self): + def test_file_display(self): for handler in (self.dialog.idle_credits, self.dialog.idle_readme, self.dialog.idle_news): diff --git a/Lib/idlelib/idle_test/test_macosx.py b/Lib/idlelib/idle_test/test_macosx.py index fae75d8a497320..3d85f3ca72254c 100644 --- a/Lib/idlelib/idle_test/test_macosx.py +++ b/Lib/idlelib/idle_test/test_macosx.py @@ -4,7 +4,6 @@ ''' from idlelib import macosx from test.support import requires -import sys import tkinter as tk import unittest import unittest.mock as mock @@ -78,6 +77,10 @@ def setUpClass(cls): requires('gui') cls.root = tk.Tk() cls.root.withdraw() + def cmd(tkpath, func): + assert isinstance(tkpath, str) + assert isinstance(func, type(cmd)) + cls.root.createcommand = cmd @classmethod def tearDownClass(cls): diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py new file mode 100644 index 00000000000000..231c7bf9cfb620 --- /dev/null +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -0,0 +1,172 @@ +""" Test idlelib.outwin. +""" + +import unittest +from tkinter import Tk, Text +from idlelib.idle_test.mock_tk import Mbox_func +from idlelib.idle_test.mock_idle import Func +from idlelib import outwin +from test.support import requires +from unittest import mock + + +class OutputWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + root = cls.root = Tk() + root.withdraw() + w = cls.window = outwin.OutputWindow(None, None, None, root) + cls.text = w.text = Text(root) + + @classmethod + def tearDownClass(cls): + cls.window.close() + del cls.text, cls.window + cls.root.destroy() + del cls.root + + def setUp(self): + self.text.delete('1.0', 'end') + + def test_ispythonsource(self): + # OutputWindow overrides ispythonsource to always return False. + w = self.window + self.assertFalse(w.ispythonsource('test.txt')) + self.assertFalse(w.ispythonsource(__file__)) + + def test_window_title(self): + self.assertEqual(self.window.top.title(), 'Output') + + def test_maybesave(self): + w = self.window + eq = self.assertEqual + w.get_saved = Func() + + w.get_saved.result = False + eq(w.maybesave(), 'no') + eq(w.get_saved.called, 1) + + w.get_saved.result = True + eq(w.maybesave(), 'yes') + eq(w.get_saved.called, 2) + del w.get_saved + + def test_write(self): + eq = self.assertEqual + delete = self.text.delete + get = self.text.get + write = self.window.write + + # Test bytes. + b = b'Test bytes.' + eq(write(b), len(b)) + eq(get('1.0', '1.end'), b.decode()) + + # No new line - insert stays on same line. + delete('1.0', 'end') + test_text = 'test text' + eq(write(test_text), len(test_text)) + eq(get('1.0', '1.end'), 'test text') + eq(get('insert linestart', 'insert lineend'), 'test text') + + # New line - insert moves to next line. + delete('1.0', 'end') + test_text = 'test text\n' + eq(write(test_text), len(test_text)) + eq(get('1.0', '1.end'), 'test text') + eq(get('insert linestart', 'insert lineend'), '') + + # Text after new line is tagged for second line of Text widget. + delete('1.0', 'end') + test_text = 'test text\nLine 2' + eq(write(test_text), len(test_text)) + eq(get('1.0', '1.end'), 'test text') + eq(get('2.0', '2.end'), 'Line 2') + eq(get('insert linestart', 'insert lineend'), 'Line 2') + + # Test tags. + delete('1.0', 'end') + test_text = 'test text\n' + test_text2 = 'Line 2\n' + eq(write(test_text, tags='mytag'), len(test_text)) + eq(write(test_text2, tags='secondtag'), len(test_text2)) + eq(get('mytag.first', 'mytag.last'), test_text) + eq(get('secondtag.first', 'secondtag.last'), test_text2) + eq(get('1.0', '1.end'), test_text.rstrip('\n')) + eq(get('2.0', '2.end'), test_text2.rstrip('\n')) + + def test_writelines(self): + eq = self.assertEqual + get = self.text.get + writelines = self.window.writelines + + writelines(('Line 1\n', 'Line 2\n', 'Line 3\n')) + eq(get('1.0', '1.end'), 'Line 1') + eq(get('2.0', '2.end'), 'Line 2') + eq(get('3.0', '3.end'), 'Line 3') + eq(get('insert linestart', 'insert lineend'), '') + + def test_goto_file_line(self): + eq = self.assertEqual + w = self.window + text = self.text + + w.flist = mock.Mock() + gfl = w.flist.gotofileline = Func() + showerror = w.showerror = Mbox_func() + + # No file/line number. + w.write('Not a file line') + self.assertIsNone(w.goto_file_line()) + eq(gfl.called, 0) + eq(showerror.title, 'No special line') + + # Current file/line number. + w.write(f'{str(__file__)}: 42: spam\n') + w.write(f'{str(__file__)}: 21: spam') + self.assertIsNone(w.goto_file_line()) + eq(gfl.args, (str(__file__), 21)) + + # Previous line has file/line number. + text.delete('1.0', 'end') + w.write(f'{str(__file__)}: 42: spam\n') + w.write('Not a file line') + self.assertIsNone(w.goto_file_line()) + eq(gfl.args, (str(__file__), 42)) + + del w.flist.gotofileline, w.showerror + + +class ModuleFunctionTest(unittest.TestCase): + + @classmethod + def setUp(cls): + outwin.file_line_progs = None + + def test_compile_progs(self): + outwin.compile_progs() + for pat, regex in zip(outwin.file_line_pats, outwin.file_line_progs): + self.assertEqual(regex.pattern, pat) + + @mock.patch('builtins.open') + def test_file_line_helper(self, mock_open): + flh = outwin.file_line_helper + test_lines = ( + (r'foo file "testfile1", line 42, bar', ('testfile1', 42)), + (r'foo testfile2(21) bar', ('testfile2', 21)), + (r' testfile3 : 42: foo bar\n', (' testfile3 ', 42)), + (r'foo testfile4.py :1: ', ('foo testfile4.py ', 1)), + ('testfile5: \u19D4\u19D2: ', ('testfile5', 42)), + (r'testfile6: 42', None), # only one `:` + (r'testfile7 42 text', None) # no separators + ) + for line, expected_output in test_lines: + self.assertEqual(flh(line), expected_output) + if expected_output: + mock_open.assert_called_with(expected_output[0], 'r') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index 051f7eac2dce06..3caa2754a6d8a2 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -3,13 +3,14 @@ This must currently be a gui test because ParenMatch methods use several text methods not defined on idlelib.idle_test.mock_tk.Text. ''' +from idlelib.parenmatch import ParenMatch from test.support import requires requires('gui') import unittest from unittest.mock import Mock from tkinter import Tk, Text -from idlelib.parenmatch import ParenMatch + class DummyEditwin: def __init__(self, text): @@ -24,6 +25,7 @@ class ParenMatchTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.root = Tk() + cls.root.withdraw() cls.text = Text(cls.root) cls.editwin = DummyEditwin(cls.text) cls.editwin.text_frame = Mock() @@ -43,46 +45,39 @@ def get_parenmatch(self): pm.bell = lambda: None return pm - def test_paren_expression(self): + def test_paren_styles(self): """ - Test ParenMatch with 'expression' style. + Test ParenMatch with each style. """ text = self.text pm = self.get_parenmatch() - pm.set_style('expression') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.15')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) - - # paren_closed_event can only be tested as below - pm.paren_closed_event('event') - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.16')) - - def test_paren_default(self): - """ - Test ParenMatch with 'default' style. - """ - text = self.text - pm = self.get_parenmatch() - pm.set_style('default') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.11')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) + for style, range1, range2 in ( + ('opener', ('1.10', '1.11'), ('1.10', '1.11')), + ('default',('1.10', '1.11'),('1.10', '1.11')), + ('parens', ('1.14', '1.15'), ('1.15', '1.16')), + ('expression', ('1.10', '1.15'), ('1.10', '1.16'))): + with self.subTest(style=style): + text.delete('1.0', 'end') + pm.STYLE = style + text.insert('insert', 'def foobar(a, b') + + pm.flash_paren_event('event') + self.assertIn('<>', text.event_info()) + if style == 'parens': + self.assertTupleEqual(text.tag_nextrange('paren', '1.0'), + ('1.10', '1.11')) + self.assertTupleEqual( + text.tag_prevrange('paren', 'end'), range1) + + text.insert('insert', ')') + pm.restore_event() + self.assertNotIn('<>', + text.event_info()) + self.assertEqual(text.tag_prevrange('paren', 'end'), ()) + + pm.paren_closed_event('event') + self.assertTupleEqual( + text.tag_prevrange('paren', 'end'), range2) def test_paren_corner(self): """ @@ -94,14 +89,14 @@ def test_paren_corner(self): pm = self.get_parenmatch() text.insert('insert', '# this is a commen)') - self.assertIsNone(pm.paren_closed_event('event')) + pm.paren_closed_event('event') text.insert('insert', '\ndef') - self.assertIsNone(pm.flash_paren_event('event')) - self.assertIsNone(pm.paren_closed_event('event')) + pm.flash_paren_event('event') + pm.paren_closed_event('event') text.insert('insert', ' a, *arg)') - self.assertIsNone(pm.paren_closed_event('event')) + pm.paren_closed_event('event') def test_handle_restore_timer(self): pm = self.get_parenmatch() diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index 66af8eb85b2598..1210afe70df956 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -258,6 +258,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + del cls.dialog.destroy del cls.dialog cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index 9913ed2b7c81e1..2ecbd34168c54e 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -221,8 +221,8 @@ def test_replace_regex(self): self.assertIn('Invalid Replace Expression', showerror.message) # test access method - self.engine.setcookedpat("\'") - equal(pv.get(), "\\'") + self.engine.setcookedpat("?") + equal(pv.get(), "\\?") def test_replace_backwards(self): equal = self.assertEqual diff --git a/Lib/idlelib/idle_test/test_searchbase.py b/Lib/idlelib/idle_test/test_searchbase.py index d769fa2fc2f0de..27b02fbe54602c 100644 --- a/Lib/idlelib/idle_test/test_searchbase.py +++ b/Lib/idlelib/idle_test/test_searchbase.py @@ -5,7 +5,7 @@ ''' import unittest from test.support import requires -from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar +from tkinter import Tk, Frame ##, BooleanVar, StringVar from idlelib import searchengine as se from idlelib import searchbase as sdb from idlelib.idle_test.mock_idle import Func diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index f018f5ef1975ba..c129c2f0819a2f 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -1,11 +1,11 @@ '''Test idlelib.textview. -Since all methods and functions create (or destroy) a TextViewer, which -is a widget containing multiple widgets, all tests must be gui tests. +Since all methods and functions create (or destroy) a ViewWindow, which +is a widget containing a widget, etcetera, all tests must be gui tests. Using mock Text would not change this. Other mocks are used to retrieve information about calls. -Coverage: 94%. +Coverage: 100%. ''' from idlelib import textview as tv from test.support import requires @@ -14,6 +14,7 @@ import unittest import os from tkinter import Tk +from tkinter.ttk import Button from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func @@ -25,45 +26,73 @@ def setUpModule(): def tearDownModule(): global root root.update_idletasks() - root.destroy() # Pyflakes falsely sees root as undefined. + root.destroy() del root +# If we call ViewWindow or wrapper functions with defaults +# modal=True, _utest=False, test hangs on call to wait_window. +# Have also gotten tk error 'can't invoke "event" command'. -class TV(tv.TextViewer): # Used in TextViewTest. + +class VW(tv.ViewWindow): # Used in ViewWindowTest. transient = Func() grab_set = Func() wait_window = Func() -class TextViewTest(unittest.TestCase): + +# Call wrapper class VW with mock wait_window. +class ViewWindowTest(unittest.TestCase): def setUp(self): - TV.transient.__init__() - TV.grab_set.__init__() - TV.wait_window.__init__() + VW.transient.__init__() + VW.grab_set.__init__() + VW.wait_window.__init__() def test_init_modal(self): - view = TV(root, 'Title', 'test text') - self.assertTrue(TV.transient.called) - self.assertTrue(TV.grab_set.called) - self.assertTrue(TV.wait_window.called) - view.Ok() + view = VW(root, 'Title', 'test text') + self.assertTrue(VW.transient.called) + self.assertTrue(VW.grab_set.called) + self.assertTrue(VW.wait_window.called) + view.ok() def test_init_nonmodal(self): - view = TV(root, 'Title', 'test text', modal=False) - self.assertFalse(TV.transient.called) - self.assertFalse(TV.grab_set.called) - self.assertFalse(TV.wait_window.called) - view.Ok() + view = VW(root, 'Title', 'test text', modal=False) + self.assertFalse(VW.transient.called) + self.assertFalse(VW.grab_set.called) + self.assertFalse(VW.wait_window.called) + view.ok() def test_ok(self): - view = TV(root, 'Title', 'test text', modal=False) + view = VW(root, 'Title', 'test text', modal=False) view.destroy = Func() - view.Ok() + view.ok() self.assertTrue(view.destroy.called) del view.destroy # Unmask real function. view.destroy() +class TextFrameTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + "By itself, this tests that file parsed without exception." + cls.root = root = Tk() + root.withdraw() + cls.frame = tv.TextFrame(root, 'test text') + + @classmethod + def tearDownClass(cls): + del cls.frame + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_line1(self): + get = self.frame.text.get + self.assertEqual(get('1.0', '1.end'), 'test text') + + +# Call ViewWindow with modal=False. class ViewFunctionTest(unittest.TestCase): @classmethod @@ -77,23 +106,73 @@ def tearDownClass(cls): del cls.orig_error def test_view_text(self): - # If modal True, get tk error 'can't invoke "event" command'. view = tv.view_text(root, 'Title', 'test text', modal=False) - self.assertIsInstance(view, tv.TextViewer) - view.Ok() + self.assertIsInstance(view, tv.ViewWindow) + self.assertIsInstance(view.viewframe, tv.ViewFrame) + view.ok() def test_view_file(self): - test_dir = os.path.dirname(__file__) - testfile = os.path.join(test_dir, 'test_textview.py') - view = tv.view_file(root, 'Title', testfile, modal=False) - self.assertIsInstance(view, tv.TextViewer) - self.assertIn('Test', view.textView.get('1.0', '1.end')) - view.Ok() - + view = tv.view_file(root, 'Title', __file__, modal=False) + self.assertIsInstance(view, tv.ViewWindow) + self.assertIsInstance(view.viewframe, tv.ViewFrame) + get = view.viewframe.textframe.text.get + self.assertIn('Test', get('1.0', '1.end')) + view.ok() + + def test_bad_file(self): # Mock showerror will be used; view_file will return None. - testfile = os.path.join(test_dir, '../notthere.py') - view = tv.view_file(root, 'Title', testfile, modal=False) + view = tv.view_file(root, 'Title', 'abc.xyz', modal=False) + self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'File Load Error') + + def test_bad_encoding(self): + p = os.path + fn = p.abspath(p.join(p.dirname(__file__), '..', 'CREDITS.txt')) + tv.showerror.title = None + view = tv.view_file(root, 'Title', fn, 'ascii', modal=False) self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'Unicode Decode Error') + + +# Call ViewWindow with _utest=True. +class ButtonClickTest(unittest.TestCase): + + def setUp(self): + self.view = None + self.called = False + + def tearDown(self): + if self.view: + self.view.destroy() + + def test_view_text_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_text(root, 'TITLE_TEXT', 'COMMAND', _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_TEXT') + self.assertEqual(self.view.viewframe.textframe.text.get('1.0', '1.end'), + 'COMMAND') + + def test_view_file_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_file(root, 'TITLE_FILE', __file__, _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_FILE') + get = self.view.viewframe.textframe.text.get + with open(__file__) as f: + self.assertEqual(get('1.0', '1.end'), f.readline().strip()) + f.readline() + self.assertEqual(get('3.0', '3.end'), f.readline().strip()) if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py index 09ba9641af552b..bb597d87ffd104 100644 --- a/Lib/idlelib/idle_test/test_tree.py +++ b/Lib/idlelib/idle_test/test_tree.py @@ -5,7 +5,6 @@ from idlelib import tree from test.support import requires requires('gui') -import os import unittest from tkinter import Tk diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 3414c7b3aff4b3..f9b6907b40cecc 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -487,11 +487,11 @@ def print_window(self, event): opendialog = None savedialog = None - filetypes = [ + filetypes = ( ("Python files", "*.py *.pyw", "TEXT"), ("Text files", "*.txt", "TEXT"), ("All files", "*"), - ] + ) defaultextension = '.py' if sys.platform == 'darwin' else '' diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index c225dd9e1a115d..d85278a0b765ae 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -2,7 +2,6 @@ A number of functions that enhance IDLE on Mac OSX. """ from sys import platform # Used in _init_tk_type, changed by test. -import warnings import tkinter @@ -166,7 +165,7 @@ def about_dialog(event=None): "Handle Help 'About IDLE' event." # Synchronize with editor.EditorWindow.about_dialog. from idlelib import help_about - help_about.AboutDialog(root, 'About IDLE') + help_about.AboutDialog(root) def config_dialog(event=None): "Handle Options 'Configure IDLE' event." diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py index 65345cd1f16188..143570d6b11c41 100644 --- a/Lib/idlelib/mainmenu.py +++ b/Lib/idlelib/mainmenu.py @@ -25,7 +25,7 @@ ('_New File', '<>'), ('_Open...', '<>'), ('Open _Module...', '<>'), - ('Class _Browser', '<>'), + ('Module _Browser', '<>'), ('_Path Browser', '<>'), None, ('_Save', '<>'), @@ -52,6 +52,11 @@ ('Find in Files...', '<>'), ('R_eplace...', '<>'), ('Go to _Line', '<>'), + ('S_how Completions', '<>'), + ('E_xpand Word', '<>'), + ('Show C_all Tip', '<>'), + ('Show Surrounding P_arens', '<>'), + ]), ('format', [ ('_Indent Region', '<>'), @@ -62,9 +67,13 @@ ('Untabify Region', '<>'), ('Toggle Tabs', '<>'), ('New Indent Width', '<>'), + ('F_ormat Paragraph', '<>'), + ('S_trip Trailing Whitespace', '<>'), ]), ('run', [ ('Python Shell', '<>'), + ('C_heck Module', '<>'), + ('R_un Module', '<>'), ]), ('shell', [ ('_View Last Restart', '<>'), @@ -80,7 +89,10 @@ ]), ('options', [ ('Configure _IDLE', '<>'), - None, + ('_Code Context', '<>'), + ]), + ('windows', [ + ('Zoom Height', '<>'), ]), ('help', [ ('_About IDLE', '<>'), diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index f6d2915c62443e..6c2a792d86b99a 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -1,43 +1,114 @@ +"""Editor window that can serve as an output file. +""" + import re -from tkinter import * -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox from idlelib.editor import EditorWindow from idlelib import iomenu -class OutputWindow(EditorWindow): +file_line_pats = [ + # order of patterns matters + r'file "([^"]*)", line (\d+)', + r'([^\s]+)\((\d+)\)', + r'^(\s*\S.*?):\s*(\d+):', # Win filename, maybe starting with spaces + r'([^\s]+):\s*(\d+):', # filename or path, ltrim + r'^\s*(\S.*?):\s*(\d+):', # Win abs path with embedded spaces, ltrim +] + +file_line_progs = None + + +def compile_progs(): + "Compile the patterns for matching to file name and line number." + global file_line_progs + file_line_progs = [re.compile(pat, re.IGNORECASE) + for pat in file_line_pats] + +def file_line_helper(line): + """Extract file name and line number from line of text. + + Check if line of text contains one of the file/line patterns. + If it does and if the file and line are valid, return + a tuple of the file name and line number. If it doesn't match + or if the file or line is invalid, return None. + """ + if not file_line_progs: + compile_progs() + for prog in file_line_progs: + match = prog.search(line) + if match: + filename, lineno = match.group(1, 2) + try: + f = open(filename, "r") + f.close() + break + except OSError: + continue + else: + return None + try: + return filename, int(lineno) + except TypeError: + return None + + +class OutputWindow(EditorWindow): """An editor window that can serve as an output file. Also the future base class for the Python shell window. This class has no input facilities. + + Adds binding to open a file at a line to the text widget. """ + # Our own right-button menu + rmenu_specs = [ + ("Cut", "<>", "rmenu_check_cut"), + ("Copy", "<>", "rmenu_check_copy"), + ("Paste", "<>", "rmenu_check_paste"), + (None, None, None), + ("Go to file/line", "<>", None), + ] + def __init__(self, *args): EditorWindow.__init__(self, *args) self.text.bind("<>", self.goto_file_line) + self.text.unbind("<>") # Customize EditorWindow - def ispythonsource(self, filename): - # No colorization needed - return 0 + "Python source is only part of output: do not colorize." + return False def short_title(self): + "Customize EditorWindow title." return "Output" def maybesave(self): - # Override base class method -- don't ask any questions - if self.get_saved(): - return "yes" - else: - return "no" + "Customize EditorWindow to not display save file messagebox." + return 'yes' if self.get_saved() else 'no' # Act as output file - def write(self, s, tags=(), mark="insert"): + """Write text to text widget. + + The text is inserted at the given index with the provided + tags. The text widget is then scrolled to make it visible + and updated to display it, giving the effect of seeing each + line as it is added. + + Args: + s: Text to insert into text widget. + tags: Tuple of tag strings to apply on the insert. + mark: Index for the insert. + + Return: + Length of text inserted. + """ if isinstance(s, (bytes, bytes)): s = s.decode(iomenu.encoding, "replace") self.text.insert(mark, s, tags) @@ -46,80 +117,46 @@ def write(self, s, tags=(), mark="insert"): return len(s) def writelines(self, lines): + "Write each item in lines iterable." for line in lines: self.write(line) def flush(self): + "No flushing needed as write() directly writes to widget." pass - # Our own right-button menu - - rmenu_specs = [ - ("Cut", "<>", "rmenu_check_cut"), - ("Copy", "<>", "rmenu_check_copy"), - ("Paste", "<>", "rmenu_check_paste"), - (None, None, None), - ("Go to file/line", "<>", None), - ] + def showerror(self, *args, **kwargs): + messagebox.showerror(*args, **kwargs) - file_line_pats = [ - # order of patterns matters - r'file "([^"]*)", line (\d+)', - r'([^\s]+)\((\d+)\)', - r'^(\s*\S.*?):\s*(\d+):', # Win filename, maybe starting with spaces - r'([^\s]+):\s*(\d+):', # filename or path, ltrim - r'^\s*(\S.*?):\s*(\d+):', # Win abs path with embedded spaces, ltrim - ] + def goto_file_line(self, event=None): + """Handle request to open file/line. - file_line_progs = None + If the selected or previous line in the output window + contains a file name and line number, then open that file + name in a new window and position on the line number. - def goto_file_line(self, event=None): - if self.file_line_progs is None: - l = [] - for pat in self.file_line_pats: - l.append(re.compile(pat, re.IGNORECASE)) - self.file_line_progs = l - # x, y = self.event.x, self.event.y - # self.text.mark_set("insert", "@%d,%d" % (x, y)) + Otherwise, display an error messagebox. + """ line = self.text.get("insert linestart", "insert lineend") - result = self._file_line_helper(line) + result = file_line_helper(line) if not result: # Try the previous line. This is handy e.g. in tracebacks, # where you tend to right-click on the displayed source line line = self.text.get("insert -1line linestart", "insert -1line lineend") - result = self._file_line_helper(line) + result = file_line_helper(line) if not result: - tkMessageBox.showerror( + self.showerror( "No special line", "The line you point at doesn't look like " "a valid file name followed by a line number.", parent=self.text) return filename, lineno = result - edit = self.flist.open(filename) - edit.gotoline(lineno) - - def _file_line_helper(self, line): - for prog in self.file_line_progs: - match = prog.search(line) - if match: - filename, lineno = match.group(1, 2) - try: - f = open(filename, "r") - f.close() - break - except OSError: - continue - else: - return None - try: - return filename, int(lineno) - except TypeError: - return None + self.flist.gotofileline(filename, lineno) -# These classes are currently not used but might come in handy +# These classes are currently not used but might come in handy class OnDemandOutputWindow: tagdefs = { @@ -145,3 +182,7 @@ def setup(self): text.tag_configure(tag, **cnf) text.tag_raise('sel') self.write = self.owin.write + +if __name__ == '__main__': + import unittest + unittest.main('idlelib.idle_test.test_outwin', verbosity=2, exit=False) diff --git a/Lib/idlelib/paragraph.py b/Lib/idlelib/paragraph.py index f11bdaeb77ac38..cf8dfdb641f69d 100644 --- a/Lib/idlelib/paragraph.py +++ b/Lib/idlelib/paragraph.py @@ -1,4 +1,4 @@ -"""Extension to format a paragraph or selection to a max width. +"""Format a paragraph, comment block, or selection to a max width. Does basic, standard text formatting, and also understands Python comment blocks. Thus, for editing Python source code, this @@ -21,15 +21,14 @@ class FormatParagraph: - menudefs = [ - ('format', [ # /s/edit/format dscherer@cmu.edu - ('Format Paragraph', '<>'), - ]) - ] - def __init__(self, editwin): self.editwin = editwin + @classmethod + def reload(cls): + cls.max_width = idleConf.GetOption('extensions', 'FormatParagraph', + 'max-width', type='int', default=72) + def close(self): self.editwin = None @@ -45,11 +44,7 @@ def format_paragraph_event(self, event, limit=None): The length limit parameter is for testing with a known value. """ - if limit is None: - # The default length limit is that defined by pep8 - limit = idleConf.GetOption( - 'extensions', 'FormatParagraph', 'max-width', - type='int', default=72) + limit = self.max_width if limit is None else limit text = self.editwin.text first, last = self.editwin.get_selection_indices() if first and last: @@ -75,6 +70,9 @@ def format_paragraph_event(self, event, limit=None): text.see("insert") return "break" + +FormatParagraph.reload() + def find_paragraph(text, mark): """Returns the start/stop indices enclosing the paragraph that mark is in. diff --git a/Lib/idlelib/parenmatch.py b/Lib/idlelib/parenmatch.py index ccec708f31f436..983ca20675af1d 100644 --- a/Lib/idlelib/parenmatch.py +++ b/Lib/idlelib/parenmatch.py @@ -1,4 +1,4 @@ -"""ParenMatch -- An IDLE extension for parenthesis matching. +"""ParenMatch -- for parenthesis matching. When you hit a right paren, the cursor should move briefly to the left paren. Paren here is used generically; the matching applies to @@ -11,43 +11,25 @@ CHECK_DELAY = 100 # milliseconds class ParenMatch: - """Highlight matching parentheses + """Highlight matching openers and closers, (), [], and {}. - There are three supported style of paren matching, based loosely - on the Emacs options. The style is select based on the - HILITE_STYLE attribute; it can be changed used the set_style - method. + There are three supported styles of paren matching. When a right + paren (opener) is typed: - The supported styles are: + opener -- highlight the matching left paren (closer); + parens -- highlight the left and right parens (opener and closer); + expression -- highlight the entire expression from opener to closer. + (For back compatibility, 'default' is a synonym for 'opener'). - default -- When a right paren is typed, highlight the matching - left paren for 1/2 sec. - - expression -- When a right paren is typed, highlight the entire - expression from the left paren to the right paren. + Flash-delay is the maximum milliseconds the highlighting remains. + Any cursor movement (key press or click) before that removes the + highlight. If flash-delay is 0, there is no maximum. TODO: - - extend IDLE with configuration dialog to change options - - implement rest of Emacs highlight styles (see below) - - print mismatch warning in IDLE status window - - Note: In Emacs, there are several styles of highlight where the - matching paren is highlighted whenever the cursor is immediately - to the right of a right paren. I don't know how to do that in Tk, - so I haven't bothered. + - Augment bell() with mismatch warning in status window. + - Highlight when cursor is moved to the right of a closer. + This might be too expensive to check. """ - menudefs = [ - ('edit', [ - ("Show surrounding parens", "<>"), - ]) - ] - STYLE = idleConf.GetOption('extensions','ParenMatch','style', - default='expression') - FLASH_DELAY = idleConf.GetOption('extensions','ParenMatch','flash-delay', - type='int',default=500) - HILITE_CONFIG = idleConf.GetHighlight(idleConf.CurrentTheme(),'hilite') - BELL = idleConf.GetOption('extensions','ParenMatch','bell', - type='bool',default=1) RESTORE_VIRTUAL_EVENT_NAME = "<>" # We want the restore event be called before the usual return and @@ -63,44 +45,44 @@ def __init__(self, editwin): # and deactivate_restore (which calls event_delete). editwin.text.bind(self.RESTORE_VIRTUAL_EVENT_NAME, self.restore_event) - self.bell = self.text.bell if self.BELL else lambda: None self.counter = 0 self.is_restore_active = 0 - self.set_style(self.STYLE) + + @classmethod + def reload(cls): + cls.STYLE = idleConf.GetOption( + 'extensions','ParenMatch','style', default='opener') + cls.FLASH_DELAY = idleConf.GetOption( + 'extensions','ParenMatch','flash-delay', type='int',default=500) + cls.BELL = idleConf.GetOption( + 'extensions','ParenMatch','bell', type='bool', default=1) + cls.HILITE_CONFIG = idleConf.GetHighlight(idleConf.CurrentTheme(), + 'hilite') def activate_restore(self): + "Activate mechanism to restore text from highlighting." if not self.is_restore_active: for seq in self.RESTORE_SEQUENCES: self.text.event_add(self.RESTORE_VIRTUAL_EVENT_NAME, seq) self.is_restore_active = True def deactivate_restore(self): + "Remove restore event bindings." if self.is_restore_active: for seq in self.RESTORE_SEQUENCES: self.text.event_delete(self.RESTORE_VIRTUAL_EVENT_NAME, seq) self.is_restore_active = False - def set_style(self, style): - self.STYLE = style - if style == "default": - self.create_tag = self.create_tag_default - self.set_timeout = self.set_timeout_last - elif style == "expression": - self.create_tag = self.create_tag_expression - self.set_timeout = self.set_timeout_none - def flash_paren_event(self, event): + "Handle editor 'show surrounding parens' event (menu or shortcut)." indices = (HyperParser(self.editwin, "insert") .get_surrounding_brackets()) - if indices is None: - self.bell() - return - self.activate_restore() - self.create_tag(indices) - self.set_timeout_last() + self.finish_paren_event(indices) + return "break" def paren_closed_event(self, event): - # If it was a shortcut and not really a closing paren, quit. + "Handle user input of closer." + # If user bound non-closer to <>, quit. closer = self.text.get("insert-1c") if closer not in _openers: return @@ -108,14 +90,22 @@ def paren_closed_event(self, event): if not hp.is_in_code(): return indices = hp.get_surrounding_brackets(_openers[closer], True) - if indices is None: - self.bell() + self.finish_paren_event(indices) + return # Allow calltips to see ')' + + def finish_paren_event(self, indices): + if indices is None and self.BELL: + self.text.bell() return self.activate_restore() - self.create_tag(indices) - self.set_timeout() + # self.create_tag(indices) + self.tagfuncs.get(self.STYLE, self.create_tag_expression)(self, indices) + # self.set_timeout() + (self.set_timeout_last if self.FLASH_DELAY else + self.set_timeout_none)() def restore_event(self, event=None): + "Remove effect of doing match." self.text.tag_delete("paren") self.deactivate_restore() self.counter += 1 # disable the last timer, if there is one. @@ -127,11 +117,20 @@ def handle_restore_timer(self, timer_count): # any one of the create_tag_XXX methods can be used depending on # the style - def create_tag_default(self, indices): + def create_tag_opener(self, indices): """Highlight the single paren that matches""" self.text.tag_add("paren", indices[0]) self.text.tag_config("paren", self.HILITE_CONFIG) + def create_tag_parens(self, indices): + """Highlight the left and right parens""" + if self.text.get(indices[1]) in (')', ']', '}'): + rightindex = indices[1]+"+1c" + else: + rightindex = indices[1] + self.text.tag_add("paren", indices[0], indices[0]+"+1c", rightindex+"-1c", rightindex) + self.text.tag_config("paren", self.HILITE_CONFIG) + def create_tag_expression(self, indices): """Highlight the entire expression""" if self.text.get(indices[1]) in (')', ']', '}'): @@ -141,6 +140,13 @@ def create_tag_expression(self, indices): self.text.tag_add("paren", indices[0], rightindex) self.text.tag_config("paren", self.HILITE_CONFIG) + tagfuncs = { + 'opener': create_tag_opener, + 'default': create_tag_opener, + 'parens': create_tag_parens, + 'expression': create_tag_expression, + } + # any one of the set_timeout_XXX methods can be used depending on # the style @@ -160,7 +166,7 @@ def callme(callme, self=self, c=self.counter, self.editwin.text_frame.after(CHECK_DELAY, callme, callme) def set_timeout_last(self): - """The last highlight created will be removed after .5 sec""" + """The last highlight created will be removed after FLASH_DELAY millisecs""" # associate a counter with an event; only disable the "paren" # tag if the event is for the most recent timer. self.counter += 1 @@ -169,6 +175,9 @@ def set_timeout_last(self): lambda self=self, c=self.counter: self.handle_restore_timer(c)) +ParenMatch.reload() + + if __name__ == '__main__': import unittest unittest.main('idlelib.idle_test.test_parenmatch', verbosity=2) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index 6c19508d314d8f..c0aa2a1590916c 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -2,18 +2,19 @@ import os import sys -from idlelib.browser import ClassBrowser, ModuleBrowserTreeItem +from idlelib.browser import ModuleBrowser, ModuleBrowserTreeItem from idlelib.pyshell import PyShellFileList from idlelib.tree import TreeItem -class PathBrowser(ClassBrowser): +class PathBrowser(ModuleBrowser): - def __init__(self, flist, _htest=False): + def __init__(self, flist, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ self._htest = _htest + self._utest = _utest self.init(flist) def settitle(self): diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py index 6739dfd1a07a6b..536b2d7f5fef73 100644 --- a/Lib/idlelib/pyparse.py +++ b/Lib/idlelib/pyparse.py @@ -1,4 +1,4 @@ -from collections import Mapping +from collections.abc import Mapping import re import sys diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391ca006fc..168eeae9ad8e9c 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -10,35 +12,32 @@ if TkVersion < 8.5: root = Tk() # otherwise create root in main root.withdraw() + from idlelib.run import fix_scaling + fix_scaling(root) tkMessageBox.showerror("Idle Cannot Start", "Idle requires tcl/tk 8.5+, not %s." % TkVersion, parent=root) raise SystemExit(1) from code import InteractiveInterpreter -import getopt -import io import linecache import os import os.path -from platform import python_version, system +from platform import python_version import re import socket import subprocess -import sys import threading import time import tokenize import warnings -from idlelib import testing # bool value from idlelib.colorizer import ColorDelegator from idlelib.config import idleConf from idlelib import debugger from idlelib import debugger_r from idlelib.editor import EditorWindow, fixwordbreaks from idlelib.filelist import FileList -from idlelib import macosx from idlelib.outwin import OutputWindow from idlelib import rpc from idlelib.run import idle_formatwarning, PseudoInputFile, PseudoOutputFile @@ -120,8 +119,8 @@ def __init__(self, *args): self.text.bind("<>", self.clear_breakpoint_here) self.text.bind("<>", self.flist.open_shell) - self.breakpointPath = os.path.join(idleConf.GetUserCfgDir(), - 'breakpoints.lst') + self.breakpointPath = os.path.join( + idleConf.userdir, 'breakpoints.lst') # whenever a file is changed, restore breakpoints def filename_changed_hook(old_hook=self.io.filename_change_hook, self=self): @@ -895,7 +894,7 @@ def __init__(self, flist=None): try: # page help() text to shell. import pydoc # import must be done here to capture i/o rebinding. - # XXX KBK 27Dec07 use TextViewer someday, but must work w/o subproc + # XXX KBK 27Dec07 use text viewer someday, but must work w/o subproc pydoc.pager = pydoc.plainpager except: sys.stderr = sys.__stderr__ @@ -1371,6 +1370,11 @@ def fix_x11_paste(root): """ def main(): + import getopt + from platform import system + from idlelib import testing # bool value + from idlelib import macosx + global flist, root, use_subprocess capture_warnings(True) @@ -1455,6 +1459,8 @@ def main(): NoDefaultRoot() root = Tk(className="Idle") root.withdraw() + from idlelib.run import fix_scaling + fix_scaling(root) # set application icon icondir = os.path.join(os.path.dirname(__file__), 'Icons') diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index 3b1f1e25be7f86..593506383c41ab 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -24,7 +24,7 @@ import os from sys import executable, platform # Platform is set for one test. -from tkinter import Toplevel, StringVar, W, E, N, S +from tkinter import Toplevel, StringVar, W, E, S from tkinter.ttk import Frame, Button, Entry, Label from tkinter import filedialog from tkinter.font import Font diff --git a/Lib/idlelib/rstrip.py b/Lib/idlelib/rstrip.py index 2ce3c7eafe5a81..18c86f9b2c8896 100644 --- a/Lib/idlelib/rstrip.py +++ b/Lib/idlelib/rstrip.py @@ -2,12 +2,8 @@ class RstripExtension: - menudefs = [ - ('format', [None, ('Strip trailing whitespace', '<>'), ] ), ] - def __init__(self, editwin): self.editwin = editwin - self.editwin.text.bind("<>", self.do_rstrip) def do_rstrip(self, event=None): diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index afa9744a346c92..39e0c116f9bc41 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -52,7 +52,7 @@ def idle_showwarning_subproc( try: file.write(idle_formatwarning( message, category, filename, lineno, line)) - except IOError: + except OSError: pass # the file (probably stderr) is invalid - this warning gets lost. _warnings_showwarning = None @@ -180,19 +180,17 @@ def manage_socket(address): server.handle_request() # A single request only def show_socket_error(err, address): + "Display socket error from manage_socket." import tkinter - import tkinter.messagebox as tkMessageBox + from tkinter.messagebox import showerror root = tkinter.Tk() + fix_scaling(root) root.withdraw() - if err.args[0] == 61: # connection refused - msg = "IDLE's subprocess can't connect to %s:%d. This may be due "\ - "to your personal firewall configuration. It is safe to "\ - "allow this internal connection because no data is visible on "\ - "external ports." % address - tkMessageBox.showerror("IDLE Subprocess Error", msg, parent=root) - else: - tkMessageBox.showerror("IDLE Subprocess Error", - "Socket Error: %s" % err.args[1], parent=root) + msg = f"IDLE's subprocess can't connect to {address[0]}:{address[1]}.\n"\ + f"Fatal OSError #{err.errno}: {err.strerror}.\n"\ + f"See the 'Startup failure' section of the IDLE doc, online at\n"\ + f"https://docs.python.org/3/library/idle.html#startup-failure" + showerror("IDLE Subprocess Error", msg, parent=root) root.destroy() def print_exception(): @@ -280,6 +278,18 @@ def exit(): sys.exit(0) +def fix_scaling(root): + """Scale fonts on HiDPI displays.""" + import tkinter.font + scaling = float(root.tk.call('tk', 'scaling')) + if scaling > 1.4: + for name in tkinter.font.names(root): + font = tkinter.font.Font(root=root, name=name, exists=True) + size = int(font['size']) + if size < 0: + font['size'] = round(-0.75*size) + + class MyRPCServer(rpc.RPCServer): def handle_error(self, request, client_address): diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py index 79d86adabc85a0..45bf56345825a1 100644 --- a/Lib/idlelib/runscript.py +++ b/Lib/idlelib/runscript.py @@ -1,22 +1,14 @@ -"""Extension to execute code outside the Python shell window. +"""Execute code from an editor. -This adds the following commands: +Check module: do a full syntax check of the current module. +Also run the tabnanny to catch any inconsistent tabs. -- Check module does a full syntax check of the current module. - It also runs the tabnanny to catch any inconsistent tabs. - -- Run module executes the module's code in the __main__ namespace. The window - must have been saved previously. The module is added to sys.modules, and is - also added to the __main__ namespace. - -XXX GvR Redesign this interface (yet again) as follows: - -- Present a dialog box for ``Run Module'' - -- Allow specify command line arguments in the dialog box +Run module: also execute the module's code in the __main__ namespace. +The window must have been saved previously. The module is added to +sys.modules, and is also added to the __main__ namespace. +TODO: Specify command line arguments in a dialog box. """ - import os import tabnanny import tokenize @@ -40,11 +32,6 @@ class ScriptBinding: - menudefs = [ - ('run', [None, - ('Check Module', '<>'), - ('Run Module', '<>'), ]), ] - def __init__(self, editwin): self.editwin = editwin # Provide instance variables referenced by debugger @@ -63,6 +50,7 @@ def check_module_event(self, event): return 'break' if not self.tabnanny(filename): return 'break' + return "break" def tabnanny(self, filename): # XXX: tabnanny should work on binary files as well diff --git a/Lib/idlelib/scrolledlist.py b/Lib/idlelib/scrolledlist.py index cc08c26e5eefaa..cdf658404ab643 100644 --- a/Lib/idlelib/scrolledlist.py +++ b/Lib/idlelib/scrolledlist.py @@ -76,6 +76,7 @@ def popup_event(self, event): index = self.listbox.index("active") self.select(index) menu.tk_popup(event.x_root, event.y_root) + return "break" def make_menu(self): menu = Menu(self.listbox, tearoff=0) diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 0698def5d4c7f2..400fa632a098cf 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -1,6 +1,5 @@ import linecache import os -import re import sys import tkinter as tk diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index adee326e1deb91..e3b55065c6d9cc 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -1,90 +1,136 @@ """Simple text browser for IDLE """ -from tkinter import * -from tkinter.ttk import Scrollbar +from tkinter import Toplevel, Text +from tkinter.ttk import Frame, Scrollbar, Button from tkinter.messagebox import showerror -class TextViewer(Toplevel): +class TextFrame(Frame): + "Display text with scrollbar." + + def __init__(self, parent, rawtext): + """Create a frame for Textview. + + parent - parent widget for this frame + rawtext - text to display + """ + super().__init__(parent) + self['relief'] = 'sunken' + self['height'] = 700 + # TODO: get fg/bg from theme. + self.bg = '#ffffff' + self.fg = '#000000' + + self.text = text = Text(self, wrap='word', highlightthickness=0, + fg=self.fg, bg=self.bg) + self.scroll = scroll = Scrollbar(self, orient='vertical', + takefocus=False, command=text.yview) + text['yscrollcommand'] = scroll.set + text.insert(0.0, rawtext) + text['state'] = 'disabled' + text.focus_set() + + scroll.pack(side='right', fill='y') + text.pack(side='left', expand=True, fill='both') + + +class ViewFrame(Frame): + "Display TextFrame and Close button." + def __init__(self, parent, text): + super().__init__(parent) + self.parent = parent + self.bind('', self.ok) + self.bind('', self.ok) + self.textframe = TextFrame(self, text) + self.button_ok = button_ok = Button( + self, text='Close', command=self.ok, takefocus=False) + self.textframe.pack(side='top', expand=True, fill='both') + button_ok.pack(side='bottom') + + def ok(self, event=None): + """Dismiss text viewer dialog.""" + self.parent.destroy() + + +class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." - def __init__(self, parent, title, text, modal=True, _htest=False): - """Show the given text in a scrollable window with a 'close' button + def __init__(self, parent, title, text, modal=True, + *, _htest=False, _utest=False): + """Show the given text in a scrollable window with a 'close' button. - If modal option set to False, user can interact with other windows, - otherwise they will be unable to interact with other windows until - the textview window is closed. + If modal is left True, users cannot interact with other windows + until the textview window is closed. + parent - parent of this dialog + title - string which is title of popup dialog + text - text to display in dialog _htest - bool; change box location when running htest. + _utest - bool; don't wait_window when running unittest. """ - Toplevel.__init__(self, parent) - self.configure(borderwidth=5) + super().__init__(parent) + self['borderwidth'] = 5 # Place dialog below parent if running htest. - self.geometry("=%dx%d+%d+%d" % (750, 500, - parent.winfo_rootx() + 10, - parent.winfo_rooty() + (10 if not _htest else 100))) - # TODO: get fg/bg from theme. - self.bg = '#ffffff' - self.fg = '#000000' + x = parent.winfo_rootx() + 10 + y = parent.winfo_rooty() + (10 if not _htest else 100) + self.geometry(f'=750x500+{x}+{y}') - self.CreateWidgets() self.title(title) - self.protocol("WM_DELETE_WINDOW", self.Ok) - self.parent = parent - self.textView.focus_set() - # Bind keys for closing this dialog. - self.bind('',self.Ok) - self.bind('',self.Ok) - self.textView.insert(0.0, text) - self.textView.config(state=DISABLED) + self.viewframe = ViewFrame(self, text) + self.protocol("WM_DELETE_WINDOW", self.ok) + self.button_ok = button_ok = Button(self, text='Close', + command=self.ok, takefocus=False) + self.viewframe.pack(side='top', expand=True, fill='both') if modal: self.transient(parent) self.grab_set() - self.wait_window() - - def CreateWidgets(self): - frameText = Frame(self, relief=SUNKEN, height=700) - frameButtons = Frame(self) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok, takefocus=FALSE) - self.scrollbarView = Scrollbar(frameText, orient=VERTICAL, - takefocus=FALSE) - self.textView = Text(frameText, wrap=WORD, highlightthickness=0, - fg=self.fg, bg=self.bg) - self.scrollbarView.config(command=self.textView.yview) - self.textView.config(yscrollcommand=self.scrollbarView.set) - self.buttonOk.pack() - self.scrollbarView.pack(side=RIGHT,fill=Y) - self.textView.pack(side=LEFT,expand=TRUE,fill=BOTH) - frameButtons.pack(side=BOTTOM,fill=X) - frameText.pack(side=TOP,expand=TRUE,fill=BOTH) - - def Ok(self, event=None): + if not _utest: + self.wait_window() + + def ok(self, event=None): + """Dismiss text viewer dialog.""" self.destroy() -def view_text(parent, title, text, modal=True): - return TextViewer(parent, title, text, modal) +def view_text(parent, title, text, modal=True, _utest=False): + """Create text viewer for given text. + + parent - parent of this dialog + title - string which is the title of popup dialog + text - text to display in this dialog + modal - controls if users can interact with other windows while this + dialog is displayed + _utest - bool; controls wait_window on unittest + """ + return ViewWindow(parent, title, text, modal, _utest=_utest) + -def view_file(parent, title, filename, encoding=None, modal=True): +def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): + """Create text viewer for text in filename. + + Return error message if file cannot be read. Otherwise calls view_text + with contents of the file. + """ try: with open(filename, 'r', encoding=encoding) as file: contents = file.read() except OSError: showerror(title='File Load Error', - message='Unable to load file %r .' % filename, + message=f'Unable to load file {filename!r} .', parent=parent) except UnicodeDecodeError as err: showerror(title='Unicode Decode Error', message=str(err), parent=parent) else: - return view_text(parent, title, contents, modal) + return view_text(parent, title, contents, modal, _utest=_utest) + return None + if __name__ == '__main__': import unittest unittest.main('idlelib.idle_test.test_textview', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(TextViewer) + run(ViewWindow) diff --git a/Lib/idlelib/zoomheight.py b/Lib/idlelib/zoomheight.py index aa4a427eab737f..74fbc888a80d3f 100644 --- a/Lib/idlelib/zoomheight.py +++ b/Lib/idlelib/zoomheight.py @@ -1,4 +1,4 @@ -# Sample extension: zoom a window to maximum height +"Zoom a window to maximum height." import re import sys @@ -8,18 +8,13 @@ class ZoomHeight: - menudefs = [ - ('windows', [ - ('_Zoom Height', '<>'), - ]) - ] - def __init__(self, editwin): self.editwin = editwin def zoom_height_event(self, event): top = self.editwin.top zoom_height(top) + return "break" def zoom_height(top): diff --git a/Lib/idlelib/zzdummy.py b/Lib/idlelib/zzdummy.py new file mode 100644 index 00000000000000..8084499646653d --- /dev/null +++ b/Lib/idlelib/zzdummy.py @@ -0,0 +1,42 @@ +"Example extension, also used for testing." + +from idlelib.config import idleConf + +ztext = idleConf.GetOption('extensions', 'ZzDummy', 'z-text') + + +class ZzDummy: + +## menudefs = [ +## ('format', [ +## ('Z in', '<>'), +## ('Z out', '<>'), +## ] ) +## ] + + def __init__(self, editwin): + self.text = editwin.text + z_in = False + + @classmethod + def reload(cls): + cls.ztext = idleConf.GetOption('extensions', 'ZzDummy', 'z-text') + + def z_in_event(self, event): + """ + """ + text = self.text + text.undo_block_start() + for line in range(1, text.index('end')): + text.insert('%d.0', ztest) + text.undo_block_stop() + return "break" + + def z_out_event(self, event): pass + +ZzDummy.reload() + +##if __name__ == "__main__": +## import unittest +## unittest.main('idlelib.idle_test.test_zzdummy', +## verbosity=2, exit=False) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2fa90120e7decc..1c0b03bff8a379 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -318,9 +318,12 @@ def shutdown(self): self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: self.sock.close() diff --git a/Lib/imp.py b/Lib/imp.py index 781ff23d252161..866464b245b24c 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -203,8 +203,9 @@ def load_package(name, path): extensions = (machinery.SOURCE_SUFFIXES[:] + machinery.BYTECODE_SUFFIXES[:]) for extension in extensions: - path = os.path.join(path, '__init__'+extension) - if os.path.exists(path): + init_path = os.path.join(path, '__init__' + extension) + if os.path.exists(init_path): + path = init_path break else: raise ValueError('{!r} is not a package'.format(path)) diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82e05f268..8b11d22b024ffa 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index a531a0351dfb06..4cf8aec1b2a4aa 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -39,6 +39,7 @@ def _new_module(name): # Module-level locking ######################################################## # A dict mapping module names to weakrefs of _ModuleLock instances +# Dictionary protected by the global import lock _module_locks = {} # A dict mapping thread ids to _ModuleLock instances _blocking_on = {} @@ -144,10 +145,7 @@ def __init__(self, name): self._lock = None def __enter__(self): - try: - self._lock = _get_module_lock(self._name) - finally: - _imp.release_lock() + self._lock = _get_module_lock(self._name) self._lock.acquire() def __exit__(self, *args, **kwargs): @@ -159,31 +157,47 @@ def __exit__(self, *args, **kwargs): def _get_module_lock(name): """Get or create the module lock for a given module name. - Should only be called with the import lock taken.""" - lock = None + Acquire/release internally the global import lock to protect + _module_locks.""" + + _imp.acquire_lock() try: - lock = _module_locks[name]() - except KeyError: - pass - if lock is None: - if _thread is None: - lock = _DummyModuleLock(name) - else: - lock = _ModuleLock(name) - def cb(_): - del _module_locks[name] - _module_locks[name] = _weakref.ref(lock, cb) + try: + lock = _module_locks[name]() + except KeyError: + lock = None + + if lock is None: + if _thread is None: + lock = _DummyModuleLock(name) + else: + lock = _ModuleLock(name) + + def cb(ref, name=name): + _imp.acquire_lock() + try: + # bpo-31070: Check if another thread created a new lock + # after the previous lock was destroyed + # but before the weakref callback was called. + if _module_locks.get(name) is ref: + del _module_locks[name] + finally: + _imp.release_lock() + + _module_locks[name] = _weakref.ref(lock, cb) + finally: + _imp.release_lock() + return lock + def _lock_unlock_module(name): - """Release the global import lock, and acquires then release the - module lock for a given module name. + """Acquires then releases the module lock for a given module name. + This is used to ensure a module is completely initialized, in the event it is being imported by another thread. - - Should only be called with the import lock taken.""" + """ lock = _get_module_lock(name) - _imp.release_lock() try: lock.acquire() except _DeadlockError: @@ -442,9 +456,6 @@ def spec_from_loader(name, loader, *, origin=None, is_package=None): return ModuleSpec(name, loader, origin=origin, is_package=is_package) -_POPULATE = object() - - def _spec_from_module(module, loader=None, origin=None): # This function is meant for use in _setup(). try: @@ -587,7 +598,6 @@ def _module_repr_from_spec(spec): def _exec(spec, module): """Execute the spec's specified module in an existing module's namespace.""" name = spec.name - _imp.acquire_lock() with _ModuleLockManager(name): if sys.modules.get(name) is not module: msg = 'module {!r} not in sys.modules'.format(name) @@ -670,7 +680,6 @@ def _load(spec): clobbered. """ - _imp.acquire_lock() with _ModuleLockManager(spec.name): return _load_unlocked(spec) @@ -917,10 +926,6 @@ def _sanity_check(name, package, level): elif not package: raise ImportError('attempted relative import with no known parent ' 'package') - elif package not in sys.modules: - msg = ('Parent module {!r} not loaded, cannot perform relative ' - 'import') - raise SystemError(msg.format(package)) if not name and level == 0: raise ValueError('Empty module name') @@ -955,10 +960,23 @@ def _find_and_load_unlocked(name, import_): return module +_NEEDS_LOADING = object() + + def _find_and_load(name, import_): - """Find and load the module, and release the import lock.""" + """Find and load the module.""" with _ModuleLockManager(name): - return _find_and_load_unlocked(name, import_) + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return _find_and_load_unlocked(name, import_) + + if module is None: + message = ('import of {} halted; ' + 'None in sys.modules'.format(name)) + raise ModuleNotFoundError(message, name=name) + + _lock_unlock_module(name) + return module def _gcd_import(name, package=None, level=0): @@ -973,17 +991,7 @@ def _gcd_import(name, package=None, level=0): _sanity_check(name, package, level) if level > 0: name = _resolve_name(name, package, level) - _imp.acquire_lock() - if name not in sys.modules: - return _find_and_load(name, _gcd_import) - module = sys.modules[name] - if module is None: - _imp.release_lock() - message = ('import of {} halted; ' - 'None in sys.modules'.format(name)) - raise ModuleNotFoundError(message, name=name) - _lock_unlock_module(name) - return module + return _find_and_load(name, _gcd_import) def _handle_fromlist(module, fromlist, import_): diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5d41b47f..e9c2dbd5c88d99 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -31,6 +31,7 @@ __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') +import abc import ast import dis import collections.abc @@ -253,18 +254,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -285,7 +292,27 @@ def isroutine(object): def isabstract(object): """Return true if the object is an abstract base class (ABC).""" - return bool(isinstance(object, type) and object.__flags__ & TPFLAGS_IS_ABSTRACT) + if not isinstance(object, type): + return False + if object.__flags__ & TPFLAGS_IS_ABSTRACT: + return True + if not issubclass(type(object), abc.ABCMeta): + return False + if hasattr(object, '__abstractmethods__'): + # It looks like ABCMeta.__new__ has finished running; + # TPFLAGS_IS_ABSTRACT should have been accurate. + return False + # It looks like ABCMeta.__new__ has not finished running yet; we're + # probably in __init_subclass__. We'll look for abstractmethods manually. + for name, value in object.__dict__.items(): + if getattr(value, "__isabstractmethod__", False): + return True + for base in object.__bases__: + for name in getattr(base, "__abstractmethods__", ()): + value = getattr(object, name, None) + if getattr(value, "__isabstractmethod__", False): + return True + return False def getmembers(object, predicate=None): """Return all members of an object as (name, value) pairs sorted by name. @@ -478,13 +505,16 @@ def _is_wrapper(f): def _is_wrapper(f): return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting - memo = {id(f)} # Memoise by id to tolerate non-hashable objects + # Memoise by id to tolerate non-hashable objects, but store objects to + # ensure they aren't destroyed, which would allow their IDs to be reused. + memo = {id(f): f} + recursion_limit = sys.getrecursionlimit() while _is_wrapper(func): func = func.__wrapped__ id_func = id(func) - if id_func in memo: + if (id_func in memo) or (len(memo) >= recursion_limit): raise ValueError('wrapper loop when unwrapping {!r}'.format(f)) - memo.add(id_func) + memo[id_func] = func return func # -------------------------------------------------- source code extraction @@ -2214,11 +2244,16 @@ def _signature_from_callable(obj, *, sigcls=sigcls) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) - - return sig.replace(parameters=new_params) + if first_wrapped_param.kind is Parameter.VAR_POSITIONAL: + # First argument of the wrapped callable is `*args`, as in + # `partialmethod(lambda *args)`. + return sig + else: + sig_params = tuple(sig.parameters.values()) + assert first_wrapped_param is not sig_params[0] + new_params = (first_wrapped_param,) + sig_params + return sig.replace(parameters=new_params) if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 20f33cbdeb67d0..70746f8de85c09 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1410,7 +1410,8 @@ def __lt__(self, other): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. @@ -2100,7 +2101,8 @@ def __lt__(self, other): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index b8d5e6cff8cb21..6d0511ebfe90cf 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,7 @@ >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") + ... raise TypeError(repr(obj) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py index bb6718cbf770f5..a7a5a154f6fb11 100644 --- a/Lib/lib2to3/fixes/fix_filter.py +++ b/Lib/lib2to3/fixes/fix_filter.py @@ -15,7 +15,10 @@ # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, ListComp, in_special_context + class FixFilter(fixer_base.ConditionalFix): BM_compatible = True @@ -34,16 +37,19 @@ class FixFilter(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< 'filter' trailer< '(' arglist< none='None' ',' seq=any > ')' > + [extra_trailers=trailer*] > | power< 'filter' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -53,23 +59,32 @@ def transform(self, node, results): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if "filter_lambda" in results: new = ListComp(results.get("fp").clone(), results.get("fp").clone(), results.get("it").clone(), results.get("xp").clone()) + new = Node(syms.power, [new] + trailers, prefix="") elif "none" in results: new = ListComp(Name("_f"), Name("_f"), results["seq"].clone(), Name("_f")) + new = Node(syms.power, [new] + trailers, prefix="") else: if in_special_context(node): return None - new = node.clone() + + args = results['args'].clone() + new = Node(syms.power, [Name("filter"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py index 9f966feedea5aa..78cf81c6f94098 100644 --- a/Lib/lib2to3/fixes/fix_map.py +++ b/Lib/lib2to3/fixes/fix_map.py @@ -22,8 +22,10 @@ # Local imports from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context from ..pygram import python_symbols as syms +from ..pytree import Node + class FixMap(fixer_base.ConditionalFix): BM_compatible = True @@ -32,6 +34,7 @@ class FixMap(fixer_base.ConditionalFix): map_none=power< 'map' trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > + [extra_trailers=trailer*] > | map_lambda=power< @@ -47,10 +50,12 @@ class FixMap(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< - 'map' trailer< '(' [arglist=any] ')' > + 'map' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -60,6 +65,11 @@ def transform(self, node, results): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if node.parent.type == syms.simple_stmt: self.warning(node, "You should use a for loop here") new = node.clone() @@ -69,23 +79,32 @@ def transform(self, node, results): new = ListComp(results["xp"].clone(), results["fp"].clone(), results["it"].clone()) + new = Node(syms.power, [new] + trailers, prefix="") + else: if "map_none" in results: new = results["arg"].clone() + new.prefix = "" else: - if "arglist" in results: - args = results["arglist"] - if args.type == syms.arglist and \ - args.children[0].type == token.NAME and \ - args.children[0].value == "None": + if "args" in results: + args = results["args"] + if args.type == syms.trailer and \ + args.children[1].type == syms.arglist and \ + args.children[1].children[0].type == token.NAME and \ + args.children[1].children[0].value == "None": self.warning(node, "cannot convert map(None, ...) " "with multiple arguments because map() " "now truncates to the shortest sequence") return + + new = Node(syms.power, [Name("map"), args.clone()]) + new.prefix = "" + if in_special_context(node): return None - new = node.clone() + + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) + new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py index 8f36a94fb4f550..52c28df6aab411 100644 --- a/Lib/lib2to3/fixes/fix_zip.py +++ b/Lib/lib2to3/fixes/fix_zip.py @@ -9,13 +9,16 @@ # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, in_special_context + class FixZip(fixer_base.ConditionalFix): BM_compatible = True PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > + power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] > """ @@ -28,8 +31,16 @@ def transform(self, node, results): if in_special_context(node): return None - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) + args = results['args'].clone() + args.prefix = "" + + trailers = [] + if 'trailers' in results: + trailers = [n.clone() for n in results['trailers']] + for n in trailers: + n.prefix = "" + + new = Node(syms.power, [Name("zip"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60f7da89e..45afc5f4e53fcf 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. @@ -74,10 +74,11 @@ def maybe(*choices): return group(*choices) + '?' Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""') +_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?" +Triple = group(_litprefix + "'''", _litprefix + '"""') # Single-line ' or " string. -String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get @@ -95,9 +96,9 @@ def maybe(*choices): return group(*choices) + '?' Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) @@ -109,19 +110,26 @@ def maybe(*choices): return group(*choices) + '?' "r'''": single3prog, 'r"""': double3prog, "u'''": single3prog, 'u"""': double3prog, "b'''": single3prog, 'b"""': double3prog, + "f'''": single3prog, 'f"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, + "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog, + "F'''": single3prog, 'F"""': double3prog, "uR'''": single3prog, 'uR"""': double3prog, "Ur'''": single3prog, 'Ur"""': double3prog, "UR'''": single3prog, 'UR"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "rB'''": single3prog, 'rB"""': double3prog, + "Rb'''": single3prog, 'Rb"""': double3prog, + "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, + 'f': None, 'F': None, 'b': None, 'B': None} triple_quoted = {} @@ -129,20 +137,26 @@ def maybe(*choices): return group(*choices) + '?' "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""', + "f'''", 'f"""', "F'''", 'F"""', "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""',): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "rb'''", 'rb"""', "Rb'''", 'Rb"""', + "rB'''", 'rB"""', "RB'''", 'RB"""',): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', ): + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',): single_quoted[t] = t tabsize = 8 diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a5411a855a..0b9bee0ab42ea6 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ def testLongIntegers(self): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index b3f2680725961e..3e1a255737ec4b 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2954,10 +2954,23 @@ def test_filter_basic(self): a = """x = [x for x in range(10) if x%2 == 0]""" self.check(b, a) - # XXX This (rare) case is not supported -## b = """x = filter(f, 'abc')[0]""" -## a = """x = list(filter(f, 'abc'))[0]""" -## self.check(b, a) + def test_filter_trailers(self): + b = """x = filter(None, 'abc')[0]""" + a = """x = [_f for _f in 'abc' if _f][0]""" + self.check(b, a) + + b = """x = len(filter(f, 'abc')[0])""" + a = """x = len(list(filter(f, 'abc'))[0])""" + self.check(b, a) + + b = """x = filter(lambda x: x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) + + # Note the parens around x + b = """x = filter(lambda (x): x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) def test_filter_nochange(self): a = """b.join(filter(f, 'abc'))""" @@ -3022,6 +3035,23 @@ def test_prefix_preservation(self): a = """x = list(map( f, 'abc' ))""" self.check(b, a) + def test_map_trailers(self): + b = """x = map(f, 'abc')[0]""" + a = """x = list(map(f, 'abc'))[0]""" + self.check(b, a) + + b = """x = map(None, l)[0]""" + a = """x = list(l)[0]""" + self.check(b, a) + + b = """x = map(lambda x:x, l)[0]""" + a = """x = [x for x in l][0]""" + self.check(b, a) + + b = """x = map(f, 'abc')[0][1]""" + a = """x = list(map(f, 'abc'))[0][1]""" + self.check(b, a) + def test_trailing_comment(self): b = """x = map(f, 'abc') # foo""" a = """x = list(map(f, 'abc')) # foo""" @@ -3066,11 +3096,6 @@ def test_map_basic(self): """ self.warns(b, a, "You should use a for loop here") - # XXX This (rare) case is not supported -## b = """x = map(f, 'abc')[0]""" -## a = """x = list(map(f, 'abc'))[0]""" -## self.check(b, a) - def test_map_nochange(self): a = """b.join(map(f, 'abc'))""" self.unchanged(a) @@ -3130,6 +3155,10 @@ def check(self, b, a): super(Test_zip, self).check(b, a) def test_zip_basic(self): + b = """x = zip()""" + a = """x = list(zip())""" + self.check(b, a) + b = """x = zip(a, b, c)""" a = """x = list(zip(a, b, c))""" self.check(b, a) @@ -3138,6 +3167,15 @@ def test_zip_basic(self): a = """x = len(list(zip(a, b)))""" self.check(b, a) + def test_zip_trailers(self): + b = """x = zip(a, b, c)[0]""" + a = """x = list(zip(a, b, c))[0]""" + self.check(b, a) + + b = """x = zip(a, b, c)[0][1]""" + a = """x = list(zip(a, b, c))[0][1]""" + self.check(b, a) + def test_zip_nochange(self): a = """b.join(zip(a, b))""" self.unchanged(a) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 9a969e8ec04741..3f7ab9714e38f9 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -320,6 +320,7 @@ def test_5(self): def test_6(self): self.validate("lst: List[int] = []") + class TestExcept(GrammarTest): def test_new(self): s = """ @@ -338,6 +339,27 @@ def test_old(self): self.validate(s) +class TestStringLiterals(GrammarTest): + prefixes = ("'", '"', + "r'", 'r"', "R'", 'R"', + "u'", 'u"', "U'", 'U"', + "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',) + + def test_lit(self): + for pre in self.prefixes: + single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) + self.validate(single) + triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) + self.validate(triple) + + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms class TestSetLiteral(GrammarTest): def test_1(self): diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index b44a3b22288cee..0965536797d7c6 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1697,6 +1697,27 @@ def hasHandlers(self): """ return self.logger.hasHandlers() + def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + """ + Low-level log implementation, proxied to allow nested logger adapters. + """ + return self.logger._log( + level, + msg, + args, + exc_info=exc_info, + extra=extra, + stack_info=stack_info, + ) + + @property + def manager(self): + return self.logger.manager + + @manager.setter + def set_manager(self, value): + self.logger.manager = value + def __repr__(self): logger = self.logger level = getLevelName(logger.getEffectiveLevel()) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d779734f37b0d..2f934b33071563 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,16 +815,38 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_INET, socktype) - if socktype == socket.SOCK_STREAM: - self.socket.connect(address) + host, port = address + ress = socket.getaddrinfo(host, port, 0, socktype) + if not ress: + raise OSError("getaddrinfo returns an empty list") + for res in ress: + af, socktype, proto, _, sa = res + err = sock = None + try: + sock = socket.socket(af, socktype, proto) + if socktype == socket.SOCK_STREAM: + sock.connect(sa) + break + except OSError as exc: + err = exc + if sock is not None: + sock.close() + if err is not None: + raise err + self.socket = sock self.socktype = socktype - self.formatter = None def _connect_unixsocket(self, address): use_socktype = self.socktype @@ -863,7 +885,7 @@ def encodePriority(self, facility, priority): priority = self.priority_names[priority] return (facility << 3) | priority - def close (self): + def close(self): """ Closes the socket. """ diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 9a886803dc1e2a..5a7e6493bf51ff 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -439,6 +439,7 @@ def _default_mime_types(): '.jpeg' : 'image/jpeg', '.jpg' : 'image/jpeg', '.js' : 'application/javascript', + '.json' : 'application/json', '.ksh' : 'text/plain', '.latex' : 'application/x-latex', '.m1v' : 'video/mpeg', diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index f2c179e4e0afaa..d5ce6257456766 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -149,8 +149,15 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): util._close_stdin() - # ignoring SIGCHLD means no need to reap zombie processes - handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) + # ignoring SIGCHLD means no need to reap zombie processes; + # letting SIGINT through avoids KeyboardInterrupt tracebacks + handlers = { + signal.SIGCHLD: signal.SIG_IGN, + signal.SIGINT: signal.SIG_DFL, + } + old_handlers = {sig: signal.signal(sig, val) + for (sig, val) in handlers.items()} + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() @@ -175,7 +182,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): code = 1 if os.fork() == 0: try: - _serve_one(s, listener, alive_r, handler) + _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() @@ -186,11 +193,12 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): if e.errno != errno.ECONNABORTED: raise -def _serve_one(s, listener, alive_r, handler): - # close unnecessary stuff and reset SIGCHLD handler +def _serve_one(s, listener, alive_r, handlers): + # close unnecessary stuff and reset signal handlers listener.close() os.close(alive_r) - signal.signal(signal.SIGCHLD, handler) + for sig, val in handlers.items(): + signal.signal(sig, val) # receive fds from parent process fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf42614d59eb..a545f3c1a18961 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -128,9 +128,15 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -275,6 +281,17 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -283,15 +300,23 @@ def imap(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -302,15 +327,23 @@ def imap_unordered(self, func, iterable, chunksize=1): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -321,7 +354,7 @@ def apply_async(self, func, args=(), kwds={}, callback=None, if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -352,8 +385,14 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -375,37 +414,32 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +479,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +496,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index bca8b7a0047070..1d26b5e521e7e7 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -104,6 +104,9 @@ def start(self): _cleanup() self._popen = self._Popen(self) self._sentinel = self._popen.sentinel + # Avoid a refcycle if the target function holds an indirect + # reference to the process object (see bpo-30775) + del self._target, self._args, self._kwargs _children.add(self) def terminate(self): @@ -129,10 +132,16 @@ def is_alive(self): if self is _current_process: return True assert self._parent_pid == os.getpid(), 'can only test a child process' + if self._popen is None: return False - self._popen.poll() - return self._popen.returncode is None + + returncode = self._popen.poll() + if returncode is None: + return True + else: + _children.discard(self) + return False @property def name(self): diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index dda03ddf5425ce..513807cafecb6d 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -101,7 +101,7 @@ def get(self, block=True, timeout=None): try: if block: timeout = deadline - time.time() - if timeout < 0 or not self._poll(timeout): + if not self._poll(timeout): raise Empty elif not self._poll(): raise Empty @@ -169,14 +169,7 @@ def _start_thread(self): self._thread.start() debug('... done self._thread.start()') - # On process exit we will wait for data to be flushed to pipe. - # - # However, if this process created the queue then all - # processes which use the queue will be descendants of this - # process. Therefore waiting for the queue to be flushed - # is pointless once all the child processes have been joined. - created_by_this_process = (self._opid == os.getpid()) - if not self._joincancelled and not created_by_this_process: + if not self._joincancelled: self._jointhread = Finalize( self._thread, Queue._finalize_join, [weakref.ref(self._thread)], @@ -221,8 +214,8 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): else: wacquire = None - try: - while 1: + while 1: + try: nacquire() try: if not buffer: @@ -249,21 +242,19 @@ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): wrelease() except IndexError: pass - except Exception as e: - if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - return - # Since this runs in a daemon thread the resources it uses - # may be become unusable while the process is cleaning up. - # We ignore errors which happen after the process has - # started to cleanup. - try: + except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return + # Since this runs in a daemon thread the resources it uses + # may be become unusable while the process is cleaning up. + # We ignore errors which happen after the process has + # started to cleanup. if is_exiting(): info('error in queue thread: %s', e) + return else: import traceback traceback.print_exc() - except Exception: - pass _sentinel = object() @@ -337,6 +328,7 @@ def __getstate__(self): def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state + self._poll = self._reader.poll def get(self): with self._rlock: diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 1a2c0db40b9cc6..b490caa7e64333 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -241,20 +241,28 @@ def _run_finalizers(minpriority=None): return if minpriority is None: - f = lambda p : p[0][0] is not None + f = lambda p : p[0] is not None else: - f = lambda p : p[0][0] is not None and p[0][0] >= minpriority - - items = [x for x in list(_finalizer_registry.items()) if f(x)] - items.sort(reverse=True) - - for key, finalizer in items: - sub_debug('calling %s', finalizer) - try: - finalizer() - except Exception: - import traceback - traceback.print_exc() + f = lambda p : p[0] is not None and p[0] >= minpriority + + # Careful: _finalizer_registry may be mutated while this function + # is running (either by a GC run or by another thread). + + # list(_finalizer_registry) should be atomic, while + # list(_finalizer_registry.items()) is not. + keys = [key for key in list(_finalizer_registry) if f(key)] + keys.sort(reverse=True) + + for key in keys: + finalizer = _finalizer_registry.get(key) + # key may have been removed from the registry + if finalizer is not None: + sub_debug('calling %s', finalizer) + try: + finalizer() + except Exception: + import traceback + traceback.print_exc() if minpriority is None: _finalizer_registry.clear() @@ -386,7 +394,7 @@ def _close_stdin(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - passfds = sorted(passfds) + passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( diff --git a/Lib/os.py b/Lib/os.py index fa06f3937ba505..9fa8acb4b131ef 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -685,7 +685,9 @@ def __delitem__(self, key): raise KeyError(key) from None def __iter__(self): - for key in self._data: + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: yield self.decodekey(key) def __len__(self): diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f347216b183a9..48b566d9295f8f 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -187,19 +187,18 @@ def resolve(self, path, strict=False): if strict: return self._ext_to_normal(_getfinalpathname(s)) else: + tail_parts = [] # End of the path after the first one not found while True: try: s = self._ext_to_normal(_getfinalpathname(s)) except FileNotFoundError: previous_s = s - s = os.path.dirname(s) + s, tail = os.path.split(s) + tail_parts.append(tail) if previous_s == s: return path else: - if previous_s is None: - return s - else: - return s + os.path.sep + os.path.basename(previous_s) + return os.path.join(s, *reversed(tail_parts)) # Means fallback on absolute return None @@ -330,12 +329,10 @@ def _resolve(path, rest): try: target = accessor.readlink(newpath) except OSError as e: - if e.errno != EINVAL: - if strict: - raise - else: - return newpath - # Not a symlink + if e.errno != EINVAL and strict: + raise + # Not a symlink, or non-strict mode. We just leave the path + # untouched. path = newpath else: seen[newpath] = None # not resolved symlink @@ -1220,25 +1217,23 @@ def touch(self, mode=0o666, exist_ok=True): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/platform.py b/Lib/platform.py index e48ad0b6e7bdea..cc2db9870d84b6 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -110,7 +110,7 @@ """ -__version__ = '1.0.7' +__version__ = '1.0.8' import collections import sys, os, re, subprocess @@ -1198,7 +1198,9 @@ def _sys_version(sys_version=None): elif buildtime: builddate = builddate + ' ' + buildtime - if hasattr(sys, '_mercurial'): + if hasattr(sys, '_git'): + _, branch, revision = sys._git + elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial elif hasattr(sys, 'subversion'): # sys.subversion was added in Python 2.5 diff --git a/Lib/poplib.py b/Lib/poplib.py index cae6950eb6d2d6..6bcfa5cfeba37b 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -288,9 +288,12 @@ def close(self): if sock is not None: try: sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: sock.close() diff --git a/Lib/pstats.py b/Lib/pstats.py index d861413d4195f7..2c5bf981b85cf8 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -48,11 +48,14 @@ class Stats: printed. The sort_stats() method now processes some additional options (i.e., in - addition to the old -1, 0, 1, or 2). It takes an arbitrary number of - quoted strings to select the sort order. For example sort_stats('time', - 'name') sorts on the major key of 'internal function time', and on the - minor key of 'the name of the function'. Look at the two tables in - sort_stats() and get_sort_arg_defs(self) for more examples. + addition to the old -1, 0, 1, or 2 that are respectively interpreted as + 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number + of quoted strings to select the sort order. + + For example sort_stats('time', 'name') sorts on the major key of 'internal + function time', and on the minor key of 'the name of the function'. Look at + the two tables in sort_stats() and get_sort_arg_defs(self) for more + examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index c7fac3395b3be0..deb540d526ef7d 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Dec 6 18:51:51 2016 -topics = {'assert': '\n' - 'The "assert" statement\n' +# Autogenerated by Sphinx on Mon Sep 18 23:00:11 2017 +topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' 'Assert statements are a convenient way to insert debugging ' @@ -39,8 +38,7 @@ 'Assignments to "__debug__" are illegal. The value for the ' 'built-in\n' 'variable is determined when the interpreter starts.\n', - 'assignment': '\n' - 'Assignment statements\n' + 'assignment': 'Assignment statements\n' '*********************\n' '\n' 'Assignment statements are used to (re)bind names to values and ' @@ -405,8 +403,7 @@ 'See also: **PEP 526** - Variable and attribute annotation ' 'syntax\n' ' **PEP 484** - Type hints\n', - 'atom-identifiers': '\n' - 'Identifiers (Names)\n' + 'atom-identifiers': 'Identifiers (Names)\n' '*******************\n' '\n' 'An identifier occurring as an atom is a name. See ' @@ -446,8 +443,7 @@ 'happen. If the class name consists only of underscores, ' 'no\n' 'transformation is done.\n', - 'atom-literals': '\n' - 'Literals\n' + 'atom-literals': 'Literals\n' '********\n' '\n' 'Python supports string and bytes literals and various ' @@ -476,8 +472,7 @@ 'may obtain\n' 'the same object or a different object with the same ' 'value.\n', - 'attribute-access': '\n' - 'Customizing attribute access\n' + 'attribute-access': 'Customizing attribute access\n' '****************************\n' '\n' 'The following methods can be defined to customize the ' @@ -851,8 +846,7 @@ '* *__class__* assignment works only if both classes have ' 'the same\n' ' *__slots__*.\n', - 'attribute-references': '\n' - 'Attribute references\n' + 'attribute-references': 'Attribute references\n' '********************\n' '\n' 'An attribute reference is a primary followed by a ' @@ -875,8 +869,7 @@ 'determined by the object. Multiple evaluations of ' 'the same attribute\n' 'reference may yield different objects.\n', - 'augassign': '\n' - 'Augmented assignment statements\n' + 'augassign': 'Augmented assignment statements\n' '*******************************\n' '\n' 'Augmented assignment is the combination, in a single statement, ' @@ -940,8 +933,7 @@ 'about\n' 'class and instance attributes applies as for regular ' 'assignments.\n', - 'binary': '\n' - 'Binary arithmetic operations\n' + 'binary': 'Binary arithmetic operations\n' '****************************\n' '\n' 'The binary arithmetic operations have the conventional priority\n' @@ -1029,8 +1021,7 @@ 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' 'The numeric arguments are first converted to a common type.\n', - 'bitwise': '\n' - 'Binary bitwise operations\n' + 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' 'Each of the three bitwise operations has a different priority ' @@ -1050,8 +1041,7 @@ 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' 'which must be integers.\n', - 'bltin-code-objects': '\n' - 'Code Objects\n' + 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' 'Code objects are used by the implementation to ' @@ -1074,8 +1064,7 @@ '\n' 'See The standard type hierarchy for more ' 'information.\n', - 'bltin-ellipsis-object': '\n' - 'The Ellipsis Object\n' + 'bltin-ellipsis-object': 'The Ellipsis Object\n' '*******************\n' '\n' 'This object is commonly used by slicing (see ' @@ -1087,8 +1076,7 @@ '"Ellipsis" singleton.\n' '\n' 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': '\n' - 'The Null Object\n' + 'bltin-null-object': 'The Null Object\n' '***************\n' '\n' "This object is returned by functions that don't " @@ -1100,8 +1088,7 @@ 'same singleton.\n' '\n' 'It is written as "None".\n', - 'bltin-type-objects': '\n' - 'Type Objects\n' + 'bltin-type-objects': 'Type Objects\n' '************\n' '\n' 'Type objects represent the various object types. An ' @@ -1113,8 +1100,7 @@ 'all standard built-in types.\n' '\n' 'Types are written like this: "".\n', - 'booleans': '\n' - 'Boolean operations\n' + 'booleans': 'Boolean operations\n' '******************\n' '\n' ' or_test ::= and_test | or_test "or" and_test\n' @@ -1163,8 +1149,7 @@ 'its\n' 'argument (for example, "not \'foo\'" produces "False" rather ' 'than "\'\'".)\n', - 'break': '\n' - 'The "break" statement\n' + 'break': 'The "break" statement\n' '*********************\n' '\n' ' break_stmt ::= "break"\n' @@ -1185,8 +1170,7 @@ 'clause, that "finally" clause is executed before really leaving ' 'the\n' 'loop.\n', - 'callable-types': '\n' - 'Emulating callable objects\n' + 'callable-types': 'Emulating callable objects\n' '**************************\n' '\n' 'object.__call__(self[, args...])\n' @@ -1195,8 +1179,7 @@ 'this method\n' ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': '\n' - 'Calls\n' + 'calls': 'Calls\n' '*****\n' '\n' 'A call calls a callable object (e.g., a *function*) with a ' @@ -1217,7 +1200,8 @@ ' ("," "*" expression | "," ' 'keyword_item)*\n' ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "**" expression)*\n' + ' ("," keyword_item | "," "**" ' + 'expression)*\n' ' keyword_item ::= identifier "=" expression\n' '\n' 'An optional trailing comma may be present after the positional and\n' @@ -1316,7 +1300,7 @@ ' 2 1\n' ' >>> f(a=1, *(2,))\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' " TypeError: f() got multiple values for keyword argument 'a'\n" ' >>> f(1, *(2,))\n' ' 1 2\n' @@ -1382,8 +1366,7 @@ ' The class must define a "__call__()" method; the effect is then ' 'the\n' ' same as if that method was called.\n', - 'class': '\n' - 'Class definitions\n' + 'class': 'Class definitions\n' '*****************\n' '\n' 'A class definition defines a class object (see section The ' @@ -1469,8 +1452,7 @@ '\n' 'See also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n' ' Class Decorators\n', - 'comparisons': '\n' - 'Comparisons\n' + 'comparisons': 'Comparisons\n' '***********\n' '\n' 'Unlike C, all comparison operations in Python have the same ' @@ -1623,7 +1605,7 @@ 'restriction that\n' ' ranges do not support order comparison. Equality ' 'comparison across\n' - ' these types results in unequality, and ordering comparison ' + ' these types results in inequality, and ordering comparison ' 'across\n' ' these types raises "TypeError".\n' '\n' @@ -1687,7 +1669,7 @@ 'they\n' ' have equal *(key, value)* pairs. Equality comparison of the ' 'keys and\n' - ' elements enforces reflexivity.\n' + ' values enforces reflexivity.\n' '\n' ' Order comparisons ("<", ">", "<=", and ">=") raise ' '"TypeError".\n' @@ -1762,6 +1744,12 @@ ' to sequences, but not to sets or mappings). See also the\n' ' "total_ordering()" decorator.\n' '\n' + '* The "hash()" result should be consistent with equality. ' + 'Objects\n' + ' that are equal should either have the same hash value, or ' + 'be marked\n' + ' as unhashable.\n' + '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' 'not-a-number values are an example for not following these ' @@ -1773,9 +1761,9 @@ '\n' 'The operators "in" and "not in" test for membership. "x in ' 's"\n' - 'evaluates to true if *x* is a member of *s*, and false ' - 'otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in ' + 'evaluates to "True" if *x* is a member of *s*, and "False" ' + 'otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' 'sequences\n' 'and set types support this as well as dictionary, for which ' '"in" tests\n' @@ -1786,30 +1774,32 @@ 'for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only ' - 'if *x* is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != ' - '-1". Empty\n' - 'strings are always considered to be a substring of any other ' - 'string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and ' + 'only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != ' + '-1".\n' + 'Empty strings are always considered to be a substring of any ' + 'other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" ' 'method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true ' + 'value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" ' 'but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with ' - '"x == z"\n' - 'is produced while iterating over "y". If an exception is ' + 'define "__iter__()", "x in y" is "True" if some value "z" ' + 'with "x ==\n' + 'z" is produced while iterating over "y". If an exception is ' 'raised\n' 'during the iteration, it is as if "in" raised that ' 'exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class ' 'defines\n' - '"__getitem__()", "x in y" is true if and only if there is a ' + '"__getitem__()", "x in y" is "True" if and only if there is a ' 'non-\n' 'negative integer index *i* such that "x == y[i]", and all ' 'lower\n' @@ -1833,8 +1823,7 @@ 'is determined using the "id()" function. "x is not y" yields ' 'the\n' 'inverse truth value. [4]\n', - 'compound': '\n' - 'Compound statements\n' + 'compound': 'Compound statements\n' '*******************\n' '\n' 'Compound statements contain (groups of) other statements; they ' @@ -2613,7 +2602,8 @@ 'functions, even if they do not contain "await" or "async" ' 'keywords.\n' '\n' - 'It is a "SyntaxError" to use "yield" expressions in "async def"\n' + 'It is a "SyntaxError" to use "yield from" expressions in "async ' + 'def"\n' 'coroutines.\n' '\n' 'An example of a coroutine function:\n' @@ -2724,8 +2714,7 @@ ' body is transformed into the namespace\'s "__doc__" item ' 'and\n' " therefore the class's *docstring*.\n", - 'context-managers': '\n' - 'With Statement Context Managers\n' + 'context-managers': 'With Statement Context Managers\n' '*******************************\n' '\n' 'A *context manager* is an object that defines the ' @@ -2787,8 +2776,7 @@ ' The specification, background, and examples for the ' 'Python "with"\n' ' statement.\n', - 'continue': '\n' - 'The "continue" statement\n' + 'continue': 'The "continue" statement\n' '************************\n' '\n' ' continue_stmt ::= "continue"\n' @@ -2805,8 +2793,7 @@ '"finally" clause, that "finally" clause is executed before ' 'really\n' 'starting the next loop cycle.\n', - 'conversions': '\n' - 'Arithmetic conversions\n' + 'conversions': 'Arithmetic conversions\n' '**********************\n' '\n' 'When a description of an arithmetic operator below uses the ' @@ -2832,8 +2819,7 @@ "left argument to the '%' operator). Extensions must define " 'their own\n' 'conversion behavior.\n', - 'customization': '\n' - 'Basic customization\n' + 'customization': 'Basic customization\n' '*******************\n' '\n' 'object.__new__(cls[, ...])\n' @@ -2855,11 +2841,11 @@ ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -2894,7 +2880,7 @@ ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -3043,8 +3029,8 @@ '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -3152,15 +3138,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only ' - 'required property\n' + ' "__hash__()" should return an integer. The only required ' + 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive ' - 'or) the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the ' + 'components of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -3272,8 +3261,7 @@ ' neither "__len__()" nor "__bool__()", all its instances ' 'are\n' ' considered true.\n', - 'debugger': '\n' - '"pdb" --- The Python Debugger\n' + 'debugger': '"pdb" --- The Python Debugger\n' '*****************************\n' '\n' '**Source code:** Lib/pdb.py\n' @@ -3360,7 +3348,7 @@ ' >>> import mymodule\n' ' >>> mymodule.test()\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' ' File "./mymodule.py", line 4, in test\n' ' test2()\n' ' File "./mymodule.py", line 3, in test2\n' @@ -3938,8 +3926,7 @@ '[1] Whether a frame is considered to originate in a certain ' 'module\n' ' is determined by the "__name__" in the frame globals.\n', - 'del': '\n' - 'The "del" statement\n' + 'del': 'The "del" statement\n' '*******************\n' '\n' ' del_stmt ::= "del" target_list\n' @@ -3968,8 +3955,7 @@ 'Changed in version 3.2: Previously it was illegal to delete a name\n' 'from the local namespace if it occurs as a free variable in a nested\n' 'block.\n', - 'dict': '\n' - 'Dictionary displays\n' + 'dict': 'Dictionary displays\n' '*******************\n' '\n' 'A dictionary display is a possibly empty series of key/datum pairs\n' @@ -4013,8 +3999,7 @@ 'should be *hashable*, which excludes all mutable objects.) Clashes\n' 'between duplicate keys are not detected; the last datum (textually\n' 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': '\n' - 'Interaction with dynamic features\n' + 'dynamic-features': 'Interaction with dynamic features\n' '*********************************\n' '\n' 'Name resolution of free variables occurs at runtime, not ' @@ -4027,16 +4012,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are ' - 'illegal when used\n' - 'in conjunction with nested scopes that contain free ' - 'variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile ' - 'time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access ' 'to the full\n' 'environment for resolving names. Names may be resolved ' @@ -4050,8 +4025,7 @@ 'override the global and local namespace. If only one ' 'namespace is\n' 'specified, it is used for both.\n', - 'else': '\n' - 'The "if" statement\n' + 'else': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -4068,8 +4042,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'exceptions': '\n' - 'Exceptions\n' + 'exceptions': 'Exceptions\n' '**********\n' '\n' 'Exceptions are a means of breaking out of the normal flow of ' @@ -4145,8 +4118,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'execmodel': '\n' - 'Execution model\n' + 'execmodel': 'Execution model\n' '***************\n' '\n' '\n' @@ -4343,6 +4315,13 @@ 'Builtins and restricted execution\n' '---------------------------------\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. ' + 'Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -4355,16 +4334,7 @@ 'in any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -4380,14 +4350,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal ' - 'when used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -4477,8 +4439,7 @@ ' these operations is not available at the time the module ' 'is\n' ' compiled.\n', - 'exprlists': '\n' - 'Expression lists\n' + 'exprlists': 'Expression lists\n' '****************\n' '\n' ' expression_list ::= expression ( "," expression )* [","]\n' @@ -4515,8 +4476,7 @@ 'value of that expression. (To create an empty tuple, use an ' 'empty pair\n' 'of parentheses: "()".)\n', - 'floating': '\n' - 'Floating point literals\n' + 'floating': 'Floating point literals\n' '***********************\n' '\n' 'Floating point literals are described by the following lexical\n' @@ -4552,8 +4512,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'for': '\n' - 'The "for" statement\n' + 'for': 'The "for" statement\n' '*******************\n' '\n' 'The "for" statement is used to iterate over the elements of a ' @@ -4625,8 +4584,7 @@ '\n' ' for x in a[:]:\n' ' if x < 0: a.remove(x)\n', - 'formatstrings': '\n' - 'Format String Syntax\n' + 'formatstrings': 'Format String Syntax\n' '********************\n' '\n' 'The "str.format()" method and the "Formatter" class share ' @@ -5345,8 +5303,7 @@ ' 9 9 11 1001\n' ' 10 A 12 1010\n' ' 11 B 13 1011\n', - 'function': '\n' - 'Function definitions\n' + 'function': 'Function definitions\n' '********************\n' '\n' 'A function definition defines a user-defined function object ' @@ -5515,8 +5472,7 @@ '\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n', - 'global': '\n' - 'The "global" statement\n' + 'global': 'The "global" statement\n' '**********************\n' '\n' ' global_stmt ::= "global" identifier ("," identifier)*\n' @@ -5547,8 +5503,8 @@ 'change\n' 'the meaning of the program.\n' '\n' - '**Programmer\'s note:** the "global" is a directive to the ' - 'parser. It\n' + '**Programmer\'s note:** "global" is a directive to the parser. ' + 'It\n' 'applies only to code parsed at the same time as the "global"\n' 'statement. In particular, a "global" statement contained in a ' 'string\n' @@ -5560,8 +5516,7 @@ 'code containing the function call. The same applies to the ' '"eval()"\n' 'and "compile()" functions.\n', - 'id-classes': '\n' - 'Reserved classes of identifiers\n' + 'id-classes': 'Reserved classes of identifiers\n' '*******************************\n' '\n' 'Certain classes of identifiers (besides keywords) have ' @@ -5609,8 +5564,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'identifiers': '\n' - 'Identifiers and keywords\n' + 'identifiers': 'Identifiers and keywords\n' '************************\n' '\n' 'Identifiers (also referred to as *names*) are described by ' @@ -5758,8 +5712,7 @@ ' to help avoid name clashes between "private" attributes of ' 'base and\n' ' derived classes. See section Identifiers (Names).\n', - 'if': '\n' - 'The "if" statement\n' + 'if': 'The "if" statement\n' '******************\n' '\n' 'The "if" statement is used for conditional execution:\n' @@ -5775,8 +5728,7 @@ '(and no other part of the "if" statement is executed or evaluated).\n' 'If all expressions are false, the suite of the "else" clause, if\n' 'present, is executed.\n', - 'imaginary': '\n' - 'Imaginary literals\n' + 'imaginary': 'Imaginary literals\n' '******************\n' '\n' 'Imaginary literals are described by the following lexical ' @@ -5796,8 +5748,7 @@ '\n' ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' '3.14_15_93j\n', - 'import': '\n' - 'The "import" statement\n' + 'import': 'The "import" statement\n' '**********************\n' '\n' ' import_stmt ::= "import" module ["as" name] ( "," module ' @@ -6058,13 +6009,13 @@ '\n' ' **PEP 236** - Back to the __future__\n' ' The original proposal for the __future__ mechanism.\n', - 'in': '\n' - 'Membership test operations\n' + 'in': 'Membership test operations\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to true if *x* is a member of *s*, and false otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in sequences\n' + 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' + 'sequences\n' 'and set types support this as well as dictionary, for which "in" ' 'tests\n' 'whether the dictionary has a given key. For container types such as\n' @@ -6072,30 +6023,29 @@ 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only if *x* ' - 'is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != -1". Empty\n' - 'strings are always considered to be a substring of any other string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' + 'Empty strings are always considered to be a substring of any other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with "x == z"\n' - 'is produced while iterating over "y". If an exception is raised\n' + 'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n' + 'z" is produced while iterating over "y". If an exception is raised\n' 'during the iteration, it is as if "in" raised that exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is true if and only if there is a non-\n' + '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' 'negative integer index *i* such that "x == y[i]", and all lower\n' 'integer indices do not raise "IndexError" exception. (If any other\n' 'exception is raised, it is as if "in" raised that exception).\n' '\n' 'The operator "not in" is defined to have the inverse true value of\n' '"in".\n', - 'integers': '\n' - 'Integer literals\n' + 'integers': 'Integer literals\n' '****************\n' '\n' 'Integer literals are described by the following lexical ' @@ -6141,8 +6091,7 @@ 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', - 'lambda': '\n' - 'Lambdas\n' + 'lambda': 'Lambdas\n' '*******\n' '\n' ' lambda_expr ::= "lambda" [parameter_list]: expression\n' @@ -6165,8 +6114,7 @@ 'Note that functions created with lambda expressions cannot ' 'contain\n' 'statements or annotations.\n', - 'lists': '\n' - 'List displays\n' + 'lists': 'List displays\n' '*************\n' '\n' 'A list display is a possibly empty series of expressions enclosed ' @@ -6183,8 +6131,7 @@ 'from left to right and placed into the list object in that order.\n' 'When a comprehension is supplied, the list is constructed from the\n' 'elements resulting from the comprehension.\n', - 'naming': '\n' - 'Naming and binding\n' + 'naming': 'Naming and binding\n' '******************\n' '\n' '\n' @@ -6341,6 +6288,12 @@ 'Builtins and restricted execution\n' '=================================\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -6353,15 +6306,7 @@ 'any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -6377,14 +6322,6 @@ ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal when ' - 'used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is illegal ' - 'to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -6397,8 +6334,7 @@ 'override the global and local namespace. If only one namespace ' 'is\n' 'specified, it is used for both.\n', - 'nonlocal': '\n' - 'The "nonlocal" statement\n' + 'nonlocal': 'The "nonlocal" statement\n' '************************\n' '\n' ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' @@ -6429,8 +6365,7 @@ '\n' ' **PEP 3104** - Access to Names in Outer Scopes\n' ' The specification for the "nonlocal" statement.\n', - 'numbers': '\n' - 'Numeric literals\n' + 'numbers': 'Numeric literals\n' '****************\n' '\n' 'There are three types of numeric literals: integers, floating ' @@ -6444,8 +6379,7 @@ 'is actually an expression composed of the unary operator \'"-"\' ' 'and the\n' 'literal "1".\n', - 'numeric-types': '\n' - 'Emulating numeric types\n' + 'numeric-types': 'Emulating numeric types\n' '***********************\n' '\n' 'The following methods can be defined to emulate numeric ' @@ -6621,8 +6555,7 @@ ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n', - 'objects': '\n' - 'Objects, values and types\n' + 'objects': 'Objects, values and types\n' '*************************\n' '\n' "*Objects* are Python's abstraction for data. All data in a " @@ -6750,8 +6683,7 @@ 'created empty lists. (Note that "c = d = []" assigns the same ' 'object\n' 'to both "c" and "d".)\n', - 'operator-summary': '\n' - 'Operator precedence\n' + 'operator-summary': 'Operator precedence\n' '*******************\n' '\n' 'The following table summarizes the operator precedence ' @@ -6924,8 +6856,7 @@ 'arithmetic\n' ' or bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', - 'pass': '\n' - 'The "pass" statement\n' + 'pass': 'The "pass" statement\n' '********************\n' '\n' ' pass_stmt ::= "pass"\n' @@ -6938,8 +6869,7 @@ ' def f(arg): pass # a function that does nothing (yet)\n' '\n' ' class C: pass # a class with no methods (yet)\n', - 'power': '\n' - 'The power operator\n' + 'power': 'The power operator\n' '******************\n' '\n' 'The power operator binds more tightly than unary operators on its\n' @@ -6973,8 +6903,7 @@ 'Raising a negative number to a fractional power results in a ' '"complex"\n' 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': '\n' - 'The "raise" statement\n' + 'raise': 'The "raise" statement\n' '*********************\n' '\n' ' raise_stmt ::= "raise" [expression ["from" expression]]\n' @@ -7024,7 +6953,7 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' The above exception was the direct cause of the following ' 'exception:\n' @@ -7046,7 +6975,7 @@ ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' During handling of the above exception, another exception ' 'occurred:\n' @@ -7055,12 +6984,31 @@ ' File "", line 4, in \n' ' RuntimeError: Something bad happened\n' '\n' + 'Exception chaining can be explicitly suppressed by specifying ' + '"None"\n' + 'in the "from" clause:\n' + '\n' + ' >>> try:\n' + ' ... print(1 / 0)\n' + ' ... except:\n' + ' ... raise RuntimeError("Something bad happened") from None\n' + ' ...\n' + ' Traceback (most recent call last):\n' + ' File "", line 4, in \n' + ' RuntimeError: Something bad happened\n' + '\n' 'Additional information on exceptions can be found in section\n' 'Exceptions, and information about handling exceptions is in ' 'section\n' - 'The try statement.\n', - 'return': '\n' - 'The "return" statement\n' + 'The try statement.\n' + '\n' + 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' + 'from Y".\n' + '\n' + 'New in version 3.3: The "__suppress_context__" attribute to ' + 'suppress\n' + 'automatic display of the exception context.\n', + 'return': 'The "return" statement\n' '**********************\n' '\n' ' return_stmt ::= "return" [expression_list]\n' @@ -7087,9 +7035,15 @@ 'generator is done and will cause "StopIteration" to be raised. ' 'The\n' 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n', - 'sequence-types': '\n' - 'Emulating container types\n' + '"StopIteration" and becomes the "StopIteration.value" attribute.\n' + '\n' + 'In an asynchronous generator function, an empty "return" ' + 'statement\n' + 'indicates that the asynchronous generator is done and will cause\n' + '"StopAsyncIteration" to be raised. A non-empty "return" statement ' + 'is\n' + 'a syntax error in an asynchronous generator function.\n', + 'sequence-types': 'Emulating container types\n' '*************************\n' '\n' 'The following methods can be defined to implement ' @@ -7161,6 +7115,16 @@ ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the ' + 'length is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may ' + 'raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -7310,8 +7274,7 @@ ' iteration protocol via "__getitem__()", see this ' 'section in the\n' ' language reference.\n', - 'shifting': '\n' - 'Shifting operations\n' + 'shifting': 'Shifting operations\n' '*******************\n' '\n' 'The shifting operations have lower priority than the arithmetic\n' @@ -7335,8 +7298,7 @@ 'operand is\n' ' larger than "sys.maxsize" an "OverflowError" exception is ' 'raised.\n', - 'slicings': '\n' - 'Slicings\n' + 'slicings': 'Slicings\n' '********\n' '\n' 'A slicing selects a range of items in a sequence object (e.g., ' @@ -7387,8 +7349,7 @@ 'as lower bound, upper bound and stride, respectively, ' 'substituting\n' '"None" for missing expressions.\n', - 'specialattrs': '\n' - 'Special Attributes\n' + 'specialattrs': 'Special Attributes\n' '******************\n' '\n' 'The implementation adds a few special read-only attributes ' @@ -7473,8 +7434,7 @@ '[5] To format only a tuple you should therefore provide a\n' ' singleton tuple whose only element is the tuple to be ' 'formatted.\n', - 'specialnames': '\n' - 'Special method names\n' + 'specialnames': 'Special method names\n' '********************\n' '\n' 'A class can implement certain operations that are invoked by ' @@ -7540,11 +7500,11 @@ ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -7579,7 +7539,7 @@ ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -7726,8 +7686,8 @@ '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -7835,15 +7795,18 @@ 'on members\n' ' of hashed collections including "set", "frozenset", and ' '"dict".\n' - ' "__hash__()" should return an integer. The only required ' + ' "__hash__()" should return an integer. The only required ' 'property\n' ' is that objects which compare equal have the same hash ' 'value; it is\n' - ' advised to somehow mix together (e.g. using exclusive or) ' - 'the hash\n' - ' values for the components of the object that also play a ' - 'part in\n' - ' comparison of objects.\n' + ' advised to mix together the hash values of the components ' + 'of the\n' + ' object that also play a part in comparison of objects by ' + 'packing\n' + ' them into a tuple and hashing the tuple. Example:\n' + '\n' + ' def __hash__(self):\n' + ' return hash((self.name, self.nick, self.color))\n' '\n' ' Note: "hash()" truncates the value returned from an ' "object's\n" @@ -8595,7 +8558,7 @@ '\n' 'The potential uses for metaclasses are boundless. Some ideas ' 'that have\n' - 'been explored include logging, interface checking, ' + 'been explored include enum, logging, interface checking, ' 'automatic\n' 'delegation, automatic property creation, proxies, ' 'frameworks, and\n' @@ -8780,6 +8743,15 @@ ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the length ' + 'is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -9262,8 +9234,7 @@ 'special method *must* be set on the class object itself in ' 'order to be\n' 'consistently invoked by the interpreter).\n', - 'string-methods': '\n' - 'String Methods\n' + 'string-methods': 'String Methods\n' '**************\n' '\n' 'Strings implement all of the common sequence operations, ' @@ -9500,12 +9471,11 @@ 'characters\n' ' and there is at least one character, false otherwise. ' 'Decimal\n' - ' characters are those from general category "Nd". This ' - 'category\n' - ' includes digit characters, and all characters that can ' - 'be used to\n' - ' form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC ' - 'DIGIT ZERO.\n' + ' characters are those that can be used to form numbers ' + 'in base 10,\n' + ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' + 'decimal character\n' + ' is a character in the Unicode General Category "Nd".\n' '\n' 'str.isdigit()\n' '\n' @@ -9515,10 +9485,13 @@ 'include decimal\n' ' characters and digits that need special handling, such ' 'as the\n' - ' compatibility superscript digits. Formally, a digit is ' - 'a character\n' - ' that has the property value Numeric_Type=Digit or\n' - ' Numeric_Type=Decimal.\n' + ' compatibility superscript digits. This covers digits ' + 'which cannot\n' + ' be used to form numbers in base 10, like the Kharosthi ' + 'numbers.\n' + ' Formally, a digit is a character that has the property ' + 'value\n' + ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' @@ -9603,13 +9576,13 @@ 'str.join(iterable)\n' '\n' ' Return a string which is the concatenation of the ' - 'strings in the\n' - ' *iterable* *iterable*. A "TypeError" will be raised if ' - 'there are\n' - ' any non-string values in *iterable*, including "bytes" ' - 'objects.\n' - ' The separator between elements is the string providing ' - 'this method.\n' + 'strings in\n' + ' *iterable*. A "TypeError" will be raised if there are ' + 'any non-\n' + ' string values in *iterable*, including "bytes" ' + 'objects. The\n' + ' separator between elements is the string providing this ' + 'method.\n' '\n' 'str.ljust(width[, fillchar])\n' '\n' @@ -10064,8 +10037,7 @@ " '00042'\n" ' >>> "-42".zfill(5)\n' " '-0042'\n", - 'strings': '\n' - 'String and Bytes literals\n' + 'strings': 'String and Bytes literals\n' '*************************\n' '\n' 'String literals are described by the following lexical ' @@ -10299,8 +10271,7 @@ 'followed by a newline is interpreted as those two characters as ' 'part\n' 'of the literal, *not* as a line continuation.\n', - 'subscriptions': '\n' - 'Subscriptions\n' + 'subscriptions': 'Subscriptions\n' '*************\n' '\n' 'A subscription selects an item of a sequence (string, tuple ' @@ -10357,32 +10328,26 @@ "A string's items are characters. A character is not a " 'separate data\n' 'type but a string of exactly one character.\n', - 'truth': '\n' - 'Truth Value Testing\n' + 'truth': 'Truth Value Testing\n' '*******************\n' '\n' 'Any object can be tested for truth value, for use in an "if" or\n' - '"while" condition or as operand of the Boolean operations below. ' - 'The\n' - 'following values are considered false:\n' - '\n' - '* "None"\n' - '\n' - '* "False"\n' - '\n' - '* zero of any numeric type, for example, "0", "0.0", "0j".\n' + '"while" condition or as operand of the Boolean operations below.\n' '\n' - '* any empty sequence, for example, "\'\'", "()", "[]".\n' + 'By default, an object is considered true unless its class defines\n' + 'either a "__bool__()" method that returns "False" or a "__len__()"\n' + 'method that returns zero, when called with the object. [1] Here ' + 'are\n' + 'most of the built-in objects considered false:\n' '\n' - '* any empty mapping, for example, "{}".\n' + '* constants defined to be false: "None" and "False".\n' '\n' - '* instances of user-defined classes, if the class defines a\n' - ' "__bool__()" or "__len__()" method, when that method returns the\n' - ' integer zero or "bool" value "False". [1]\n' + '* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)",\n' + ' "Fraction(0, 1)"\n' '\n' - 'All other values are considered true --- so objects of many types ' - 'are\n' - 'always true.\n' + '* empty sequences and collections: "\'\'", "()", "[]", "{}", ' + '"set()",\n' + ' "range(0)"\n' '\n' 'Operations and built-in functions that have a Boolean result ' 'always\n' @@ -10390,8 +10355,7 @@ 'otherwise stated. (Important exception: the Boolean operations ' '"or"\n' 'and "and" always return one of their operands.)\n', - 'try': '\n' - 'The "try" statement\n' + 'try': 'The "try" statement\n' '*******************\n' '\n' 'The "try" statement specifies exception handlers and/or cleanup code\n' @@ -10538,8 +10502,7 @@ 'Exceptions, and information on using the "raise" statement to ' 'generate\n' 'exceptions may be found in section The raise statement.\n', - 'types': '\n' - 'The standard type hierarchy\n' + 'types': 'The standard type hierarchy\n' '***************************\n' '\n' 'Below is a list of the types that are built into Python. ' @@ -10739,11 +10702,11 @@ '8-bit\n' ' bytes, represented by integers in the range 0 <= x < 256.\n' ' Bytes literals (like "b\'abc\'") and the built-in ' - 'function\n' - ' "bytes()" can be used to construct bytes objects. Also,\n' - ' bytes objects can be decoded to strings via the ' - '"decode()"\n' - ' method.\n' + '"bytes()"\n' + ' constructor can be used to create bytes objects. Also, ' + 'bytes\n' + ' objects can be decoded to strings via the "decode()" ' + 'method.\n' '\n' ' Mutable sequences\n' ' Mutable sequences can be changed after they are created. ' @@ -10769,7 +10732,7 @@ ' the built-in "bytearray()" constructor. Aside from being\n' ' mutable (and hence unhashable), byte arrays otherwise ' 'provide\n' - ' the same interface and functionality as immutable bytes\n' + ' the same interface and functionality as immutable "bytes"\n' ' objects.\n' '\n' ' The extension module "array" provides an additional example ' @@ -11097,6 +11060,27 @@ 'statements.\n' ' See also the Coroutine Objects section.\n' '\n' + ' Asynchronous generator functions\n' + ' A function or method which is defined using "async def" and\n' + ' which uses the "yield" statement is called a *asynchronous\n' + ' generator function*. Such a function, when called, returns ' + 'an\n' + ' asynchronous iterator object which can be used in an "async ' + 'for"\n' + ' statement to execute the body of the function.\n' + '\n' + ' Calling the asynchronous iterator\'s "aiterator.__anext__()"\n' + ' method will return an *awaitable* which when awaited will\n' + ' execute until it provides a value using the "yield" ' + 'expression.\n' + ' When the function executes an empty "return" statement or ' + 'falls\n' + ' off the end, a "StopAsyncIteration" exception is raised and ' + 'the\n' + ' asynchronous iterator will have reached the end of the set ' + 'of\n' + ' values to be yielded.\n' + '\n' ' Built-in functions\n' ' A built-in function object is a wrapper around a C function.\n' ' Examples of built-in functions are "len()" and "math.sin()"\n' @@ -11233,14 +11217,14 @@ 'the\n' ' dictionary containing the class\'s namespace; "__bases__" is a ' 'tuple\n' - ' (possibly empty or a singleton) containing the base classes, in ' - 'the\n' - ' order of their occurrence in the base class list; "__doc__" is ' - 'the\n' - ' class\'s documentation string, or "None" if undefined;\n' - ' "__annotations__" (optional) is a dictionary containing ' - '*variable\n' - ' annotations* collected during class body execution.\n' + ' containing the base classes, in the order of their occurrence ' + 'in\n' + ' the base class list; "__doc__" is the class\'s documentation ' + 'string,\n' + ' or "None" if undefined; "__annotations__" (optional) is a\n' + ' dictionary containing *variable annotations* collected during ' + 'class\n' + ' body execution.\n' '\n' 'Class instances\n' ' A class instance is created by calling a class object (see ' @@ -11520,8 +11504,7 @@ ' under "User-defined methods". Class method objects are ' 'created\n' ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': '\n' - 'Functions\n' + 'typesfunctions': 'Functions\n' '*********\n' '\n' 'Function objects are created by function definitions. The ' @@ -11538,8 +11521,7 @@ 'different object types.\n' '\n' 'See Function definitions for more information.\n', - 'typesmapping': '\n' - 'Mapping Types --- "dict"\n' + 'typesmapping': 'Mapping Types --- "dict"\n' '************************\n' '\n' 'A *mapping* object maps *hashable* values to arbitrary ' @@ -11896,8 +11878,7 @@ " {'bacon'}\n" " >>> keys ^ {'sausage', 'juice'}\n" " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': '\n' - 'Methods\n' + 'typesmethods': 'Methods\n' '*******\n' '\n' 'Methods are functions that are called using the attribute ' @@ -11954,8 +11935,7 @@ " 'my name is method'\n" '\n' 'See The standard type hierarchy for more information.\n', - 'typesmodules': '\n' - 'Modules\n' + 'typesmodules': 'Modules\n' '*******\n' '\n' 'The only special operation on a module is attribute access: ' @@ -11992,8 +11972,7 @@ 'written as\n' '"".\n', - 'typesseq': '\n' - 'Sequence Types --- "list", "tuple", "range"\n' + 'typesseq': 'Sequence Types --- "list", "tuple", "range"\n' '*******************************************\n' '\n' 'There are three basic sequence types: lists, tuples, and range\n' @@ -12026,7 +12005,7 @@ 'comparison operations. The "+" (concatenation) and "*" ' '(repetition)\n' 'operations have the same priority as the corresponding numeric\n' - 'operations.\n' + 'operations. [3]\n' '\n' '+----------------------------+----------------------------------+------------+\n' '| Operation | Result ' @@ -12141,9 +12120,9 @@ '\n' '3. If *i* or *j* is negative, the index is relative to the end ' 'of\n' - ' the string: "len(s) + i" or "len(s) + j" is substituted. But ' - 'note\n' - ' that "-0" is still "0".\n' + ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' + 'But\n' + ' note that "-0" is still "0".\n' '\n' '4. The slice of *s* from *i* to *j* is defined as the sequence ' 'of\n' @@ -12162,12 +12141,17 @@ ' (j-i)/k". In other words, the indices are "i", "i+k", ' '"i+2*k",\n' ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). If *i* or *j* is greater than "len(s)", use\n' - ' "len(s)". If *i* or *j* are omitted or "None", they become ' - '"end"\n' - ' values (which end depends on the sign of *k*). Note, *k* ' - 'cannot be\n' - ' zero. If *k* is "None", it is treated like "1".\n' + ' including *j*). When *k* is positive, *i* and *j* are ' + 'reduced to\n' + ' "len(s)" if they are greater. When *k* is negative, *i* and ' + '*j* are\n' + ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' + 'are\n' + ' omitted or "None", they become "end" values (which end ' + 'depends on\n' + ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' + '"None", it\n' + ' is treated like "1".\n' '\n' '6. Concatenating immutable sequences always results in a new\n' ' object. This means that building up a sequence by repeated\n' @@ -12412,7 +12396,7 @@ 'operations.\n' ' Lists also provide the following additional method:\n' '\n' - ' sort(*, key=None, reverse=None)\n' + ' sort(*, key=None, reverse=False)\n' '\n' ' This method sorts the list in place, using only "<" ' 'comparisons\n' @@ -12685,8 +12669,7 @@ ' * The linspace recipe shows how to implement a lazy version ' 'of\n' ' range that suitable for floating point applications.\n', - 'typesseq-mutable': '\n' - 'Mutable Sequence Types\n' + 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' 'The operations in the following table are defined on ' @@ -12826,8 +12809,7 @@ 'referenced multiple\n' ' times, as explained for "s * n" under Common Sequence ' 'Operations.\n', - 'unary': '\n' - 'Unary arithmetic and bitwise operations\n' + 'unary': 'Unary arithmetic and bitwise operations\n' '***************************************\n' '\n' 'All unary arithmetic and bitwise operations have the same ' @@ -12849,8 +12831,7 @@ 'In all three cases, if the argument does not have the proper type, ' 'a\n' '"TypeError" exception is raised.\n', - 'while': '\n' - 'The "while" statement\n' + 'while': 'The "while" statement\n' '*********************\n' '\n' 'The "while" statement is used for repeated execution as long as an\n' @@ -12874,8 +12855,7 @@ 'executed in the first suite skips the rest of the suite and goes ' 'back\n' 'to testing the expression.\n', - 'with': '\n' - 'The "with" statement\n' + 'with': 'The "with" statement\n' '********************\n' '\n' 'The "with" statement is used to wrap the execution of a block with\n' @@ -12948,8 +12928,7 @@ ' The specification, background, and examples for the Python ' '"with"\n' ' statement.\n', - 'yield': '\n' - 'The "yield" statement\n' + 'yield': 'The "yield" statement\n' '*********************\n' '\n' ' yield_stmt ::= yield_expression\n' diff --git a/Lib/random.py b/Lib/random.py index ad1c9167b02a65..0152e5ea12bf79 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -109,9 +109,10 @@ def seed(self, a=None, version=2): """ if version == 1 and isinstance(a, (str, bytes)): + a = a.decode('latin-1') if isinstance(a, bytes) else a x = ord(a[0]) << 7 if a else 0 - for c in a: - x = ((1000003 * x) ^ ord(c)) & 0xFFFFFFFFFFFFFFFF + for c in map(ord, a): + x = ((1000003 * x) ^ c) & 0xFFFFFFFFFFFFFFFF x ^= len(a) a = -2 if x == -1 else x diff --git a/Lib/smtplib.py b/Lib/smtplib.py index f7c2c77ab42075..5e422b704ad4dc 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -250,6 +250,7 @@ def __init__(self, host='', port=0, local_hostname=None, if host: (code, msg) = self.connect(host, port) if code != 220: + self.close() raise SMTPConnectError(code, msg) if local_hostname is not None: self.local_hostname = local_hostname diff --git a/Lib/socket.py b/Lib/socket.py index 740e71782af2c3..1ada24d3326422 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -711,6 +711,8 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, if source_address: sock.bind(source_address) sock.connect(sa) + # Break explicitly a reference cycle + err = None return sock except error as _: diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py index 45f1b04c69648f..b8a13de55bc720 100644 --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -179,6 +179,15 @@ def CheckDdlDoesNotAutostartTransaction(self): result = self.con.execute("select * from test").fetchall() self.assertEqual(result, []) + def CheckImmediateTransactionalDDL(self): + # You can achieve transactional DDL by issuing a BEGIN + # statement manually. + self.con.execute("begin immediate") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + def CheckTransactionalDDL(self): # You can achieve transactional DDL by issuing a BEGIN # statement manually. diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index fc684ae96fd30a..a6e8a1f08d68a8 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -21,6 +21,17 @@ # should this really be here? class error(Exception): + """Exception raised for invalid regular expressions. + + Attributes: + + msg: The unformatted error message + pattern: The regular expression pattern + pos: The index in the pattern where compilation failed (may be None) + lineno: The line corresponding to pos (may be None) + colno: The column corresponding to pos (may be None) + """ + def __init__(self, msg, pattern=None, pos=None): self.msg = msg self.pattern = pattern diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 6aa49c3bf6f8ed..b505e11a7eea91 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -404,7 +404,7 @@ def _escape(source, escape, state): pass raise source.error("bad escape %s" % escape, len(escape)) -def _parse_sub(source, state, verbose, nested=True): +def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c items = [] @@ -412,7 +412,8 @@ def _parse_sub(source, state, verbose, nested=True): sourcematch = source.match start = source.tell() while True: - itemsappend(_parse(source, state, verbose)) + itemsappend(_parse(source, state, verbose, nested + 1, + not nested and not items)) if not sourcematch("|"): break @@ -454,10 +455,10 @@ def _parse_sub(source, state, verbose, nested=True): subpattern.append((BRANCH, (None, items))) return subpattern -def _parse_sub_cond(source, state, condgroup, verbose): - item_yes = _parse(source, state, verbose) +def _parse_sub_cond(source, state, condgroup, verbose, nested): + item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): - item_no = _parse(source, state, verbose) + item_no = _parse(source, state, verbose, nested + 1) if source.next == "|": raise source.error("conditional backref with more than two branches") else: @@ -466,7 +467,7 @@ def _parse_sub_cond(source, state, condgroup, verbose): subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -def _parse(source, state, verbose): +def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -692,7 +693,7 @@ def _parse(source, state, verbose): lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups - p = _parse_sub(source, state, verbose) + p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None @@ -730,18 +731,19 @@ def _parse(source, state, verbose): state.checklookbehindgroup(condgroup, source) elif char in FLAGS or char == "-": # flags - pos = source.pos flags = _parse_flags(source, state, char) if flags is None: # global flags - if pos != 3: # "(?x" + if not first or subpattern: import warnings warnings.warn( - 'Flags not at the start of the expression %s%s' % ( + 'Flags not at the start of the expression %r%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), - DeprecationWarning, stacklevel=7 + DeprecationWarning, stacklevel=nested + 6 ) + if (state.flags & SRE_FLAG_VERBOSE) and not verbose: + raise Verbose continue add_flags, del_flags = flags group = None @@ -756,11 +758,11 @@ def _parse(source, state, verbose): except error as err: raise source.error(err.msg, len(name) + 1) from None if condgroup: - p = _parse_sub_cond(source, state, condgroup, verbose) + p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1) else: sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) - p = _parse_sub(source, state, sub_verbose) + p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) @@ -795,9 +797,6 @@ def _parse_flags(source, state, char): msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": - if ((add_flags & SRE_FLAG_VERBOSE) and - not (state.flags & SRE_FLAG_VERBOSE)): - raise Verbose state.flags |= add_flags return None if add_flags & GLOBAL_FLAGS: @@ -853,7 +852,7 @@ def parse(str, flags=0, pattern=None): pattern.str = str try: - p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, False) + p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... @@ -861,7 +860,7 @@ def parse(str, flags=0, pattern=None): pattern.flags = flags | SRE_FLAG_VERBOSE pattern.str = str source.seek(0) - p = _parse_sub(source, pattern, True, False) + p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) diff --git a/Lib/ssl.py b/Lib/ssl.py index 8ad4a339a933d7..1f3a31a9b79687 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -115,7 +115,7 @@ pass -from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN +from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_TLSv1_3 from _ssl import _OPENSSL_API_VERSION @@ -178,6 +178,7 @@ # (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL') # Enable a better set of ciphers by default # This list has been explicitly chosen to: +# * TLS 1.3 ChaCha20 and AES-GCM cipher suites # * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) # * Prefer ECDHE over DHE for better performance # * Prefer AEAD over CBC for better performance and security @@ -189,6 +190,8 @@ # * Disable NULL authentication, NULL encryption, 3DES and MD5 MACs # for security reasons _DEFAULT_CIPHERS = ( + 'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:' + 'TLS13-AES-128-GCM-SHA256:' 'ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:' 'ECDH+AES128:DH+AES:ECDH+HIGH:DH+HIGH:RSA+AESGCM:RSA+AES:RSA+HIGH:' '!aNULL:!eNULL:!MD5:!3DES' @@ -196,6 +199,7 @@ # Restricted and more secure ciphers for the server side # This list has been explicitly chosen to: +# * TLS 1.3 ChaCha20 and AES-GCM cipher suites # * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) # * Prefer ECDHE over DHE for better performance # * Prefer AEAD over CBC for better performance and security @@ -206,6 +210,8 @@ # * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, RC4, and # 3DES for security reasons _RESTRICTED_SERVER_CIPHERS = ( + 'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:' + 'TLS13-AES-128-GCM-SHA256:' 'ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:' 'ECDH+AES128:DH+AES:ECDH+HIGH:DH+HIGH:RSA+AESGCM:RSA+AES:RSA+HIGH:' '!aNULL:!eNULL:!MD5:!DSS:!RC4:!3DES' @@ -959,11 +965,12 @@ def sendall(self, data, flags=0): raise ValueError( "non-zero flags not allowed in calls to sendall() on %s" % self.__class__) - amount = len(data) count = 0 - while (count < amount): - v = self.send(data[count:]) - count += v + with memoryview(data) as view, view.cast("B") as byte_view: + amount = len(byte_view) + while count < amount: + v = self.send(byte_view[count:]) + count += v else: return socket.sendall(self, data, flags) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb459e2615..290ae44f03d761 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -38,7 +38,7 @@ getoutput(...): Runs a command in the shell, waits for it to complete, then returns the output getstatusoutput(...): Runs a command in the shell, waits for it to complete, - then returns a (status, output) tuple + then returns a (exitcode, output) tuple """ import sys @@ -493,7 +493,7 @@ def list2cmdline(seq): # def getstatusoutput(cmd): - """ Return (status, output) of executing cmd in a shell. + """Return (exitcode, output) of executing cmd in a shell. Execute the string 'cmd' in a shell with 'check_output' and return a 2-tuple (status, output). The locale encoding is used @@ -507,19 +507,21 @@ def getstatusoutput(cmd): >>> subprocess.getstatusoutput('ls /bin/ls') (0, '/bin/ls') >>> subprocess.getstatusoutput('cat /bin/junk') - (256, 'cat: /bin/junk: No such file or directory') + (1, 'cat: /bin/junk: No such file or directory') >>> subprocess.getstatusoutput('/bin/junk') - (256, 'sh: /bin/junk: not found') + (127, 'sh: /bin/junk: not found') + >>> subprocess.getstatusoutput('/bin/kill $$') + (-15, '') """ try: data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT) - status = 0 + exitcode = 0 except CalledProcessError as ex: data = ex.output - status = ex.returncode + exitcode = ex.returncode if data[-1:] == '\n': data = data[:-1] - return status, data + return exitcode, data def getoutput(cmd): """Return output (stdout or stderr) of executing cmd in a shell. @@ -725,7 +727,10 @@ def __init__(self, args, bufsize=-1, executable=None, to_close.append(self._devnull) for fd in to_close: try: - os.close(fd) + if _mswindows and isinstance(fd, Handle): + fd.Close() + else: + os.close(fd) except OSError: pass @@ -776,19 +781,21 @@ def _stdin_write(self, input): self.stdin.write(input) except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: - # Issue #19612: On Windows, stdin.write() fails with EINVAL - # if the process already exited before the write + except OSError as exc: + if exc.errno == errno.EINVAL: + # bpo-19612, bpo-30418: On Windows, stdin.write() fails + # with EINVAL if the child process exited or if the child + # process is still running but closed the pipe. pass else: raise + try: self.stdin.close() except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: + except OSError as exc: + if exc.errno == errno.EINVAL: pass else: raise @@ -986,7 +993,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe @@ -1003,6 +1010,9 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errwrite.Close() if hasattr(self, '_devnull'): os.close(self._devnull) + # Prevent a double close of these handles/fds from __init__ + # on error. + self._closed_child_pipe_fds = True # Retain the process handle, but close the thread handle self._child_created = True @@ -1237,8 +1247,12 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # and pass it to fork_exec() if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) @@ -1253,7 +1267,8 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, + close_fds, tuple(sorted(map(int, fds_to_keep))), + cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, @@ -1301,29 +1316,32 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) + # The encoding here should match the encoding + # written in by the subprocess implementations + # like _posixsubprocess + err_msg = err_msg.decode() except ValueError: exception_name = b'SubprocessError' hex_errno = b'0' - err_msg = (b'Bad exception data from child: ' + - repr(errpipe_data)) + err_msg = 'Bad exception data from child: {!r}'.format( + bytes(errpipe_data)) child_exception_type = getattr( builtins, exception_name.decode('ascii'), SubprocessError) - err_msg = err_msg.decode(errors="surrogatepass") if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) child_exec_never_called = (err_msg == "noexec") if child_exec_never_called: err_msg = "" + # The error must be from chdir(cwd). + err_filename = cwd + else: + err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) if errno_num == errno.ENOENT: - if child_exec_never_called: - # The error must be from chdir(cwd). - err_msg += ': ' + repr(cwd) - else: - err_msg += ': ' + repr(orig_executable) - raise child_exception_type(errno_num, err_msg) + err_msg += ': ' + repr(err_filename) + raise child_exception_type(errno_num, err_msg, err_filename) raise child_exception_type(err_msg) diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56a392ae6..bfb670c9027d76 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc096..00000000000000 Binary files a/Lib/test/185test.db and /dev/null differ diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b5f47825466de6..db30e6b9d22a0c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -31,11 +32,12 @@ # without thread support. import threading -import multiprocessing.dummy import multiprocessing.connection -import multiprocessing.managers +import multiprocessing.dummy import multiprocessing.heap +import multiprocessing.managers import multiprocessing.pool +import multiprocessing.queues from multiprocessing import util @@ -63,6 +65,13 @@ def latin(s): return s.encode('latin') + +def close_queue(queue): + if isinstance(queue, multiprocessing.queues.Queue): + queue.close() + queue.join_thread() + + # # Constants # @@ -190,6 +199,12 @@ def get_value(self): # Testcases # +class DummyCallable: + def __call__(self, q, c): + assert isinstance(c, DummyCallable) + q.put(5) + + class _TestProcess(BaseTestCase): ALLOWED_TYPES = ('processes', 'threads') @@ -268,6 +283,7 @@ def test_process(self): self.assertEqual(p.exitcode, 0) self.assertEqual(p.is_alive(), False) self.assertNotIn(p, self.active_children()) + close_queue(q) @classmethod def _test_terminate(cls): @@ -397,6 +413,19 @@ def test_sentinel(self): p.join() self.assertTrue(wait_for_handle(sentinel, timeout=1)) + def test_lose_target_ref(self): + c = DummyCallable() + wr = weakref.ref(c) + q = self.Queue() + p = self.Process(target=c, args=(q, c)) + del c + p.start() + p.join() + self.assertIs(wr(), None) + self.assertEqual(q.get(), 5) + close_queue(q) + + # # # @@ -581,6 +610,7 @@ def test_put(self): self.assertEqual(queue_full(queue, MAXSIZE), False) proc.join() + close_queue(queue) @classmethod def _test_get(cls, queue, child_can_start, parent_can_continue): @@ -643,6 +673,7 @@ def test_get(self): self.assertTimingAlmostEqual(get.elapsed, TIMEOUT3) proc.join() + close_queue(queue) @classmethod def _test_fork(cls, queue): @@ -678,6 +709,7 @@ def test_fork(self): self.assertRaises(pyqueue.Empty, queue.get, False) p.join() + close_queue(queue) def test_qsize(self): q = self.Queue() @@ -693,6 +725,7 @@ def test_qsize(self): self.assertEqual(q.qsize(), 1) q.get() self.assertEqual(q.qsize(), 0) + close_queue(q) @classmethod def _test_task_done(cls, q): @@ -720,6 +753,7 @@ def test_task_done(self): for p in workers: p.join() + close_queue(queue) def test_no_import_lock_contention(self): with test.support.temp_cwd(): @@ -750,6 +784,23 @@ def test_timeout(self): # Tolerate a delta of 30 ms because of the bad clock resolution on # Windows (usually 15.6 ms) self.assertGreaterEqual(delta, 0.170) + close_queue(q) + + def test_queue_feeder_donot_stop_onexc(self): + # bpo-30414: verify feeder handles exceptions correctly + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + def __reduce__(self): + raise AttributeError + with test.support.captured_stderr(): + q = self.Queue() + q.put(NotSerializable()) + q.put(True) + # bpo-30595: use a timeout of 1 second for slow buildbots + self.assertTrue(q.get(timeout=1.0)) + close_queue(q) # # @@ -862,10 +913,12 @@ def test_notify(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) p = threading.Thread(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) # wait for both children to start sleeping sleeping.acquire() @@ -908,11 +961,13 @@ def test_notify_all(self): args=(cond, sleeping, woken, TIMEOUT1)) p.daemon = True p.start() + self.addCleanup(p.join) t = threading.Thread(target=self.f, args=(cond, sleeping, woken, TIMEOUT1)) t.daemon = True t.start() + self.addCleanup(t.join) # wait for them all to sleep for i in range(6): @@ -931,10 +986,12 @@ def test_notify_all(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() + self.addCleanup(p.join) t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) t.daemon = True t.start() + self.addCleanup(t.join) # wait for them to all sleep for i in range(6): @@ -1110,6 +1167,7 @@ def test_event(self): p.daemon = True p.start() self.assertEqual(wait(), True) + p.join() # # Tests for Barrier - adapted from tests in test/lock_tests.py @@ -1166,10 +1224,19 @@ def __init__(self, namespace, f, args, n, wait_before_exit=False): self._can_exit = namespace.Event() if not wait_before_exit: self._can_exit.set() + + threads = [] for i in range(n): p = namespace.Process(target=self.task) p.daemon = True p.start() + threads.append(p) + + def finalize(threads): + for p in threads: + p.join() + + self._finalizer = weakref.finalize(self, finalize, threads) def task(self): pid = os.getpid() @@ -1192,6 +1259,9 @@ def wait_for_finished(self): def do_finish(self): self._can_exit.set() + def close(self): + self._finalizer() + class AppendTrue(object): def __init__(self, obj): @@ -1224,8 +1294,11 @@ def DummyList(self): def run_threads(self, f, args): b = Bunch(self, f, args, self.N-1) - f(*args) - b.wait_for_finished() + try: + f(*args) + b.wait_for_finished() + finally: + b.close() @classmethod def multipass(cls, barrier, results, n): @@ -1270,6 +1343,7 @@ def test_wait_return(self): self.run_threads(self._test_wait_return_f, (self.barrier, queue)) results = [queue.get() for i in range(self.N)] self.assertEqual(results.count(0), 1) + close_queue(queue) @classmethod def _test_action_f(cls, barrier, results): @@ -1440,6 +1514,7 @@ def test_thousand(self): p = self.Process(target=self._test_thousand_f, args=(self.barrier, passes, child_conn, lock)) p.start() + self.addCleanup(p.join) for i in range(passes): for j in range(self.N): @@ -1738,14 +1813,30 @@ def raise_large_valuerror(wait): time.sleep(wait) raise ValueError("x" * 1024**2) +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): + if when == -1: + raise SayWhenError("Somebody said when") for i in range(total): if i == when: raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -1818,6 +1909,32 @@ def test_map_chunksize(self): except multiprocessing.TimeoutError: self.fail("pool.map_async with chunksize stalled on null list") + def test_map_handle_iterable_exception(self): + if self.TYPE == 'manager': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # SayWhenError seen at the very first of the iterable + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + # again, make sure it's reentrant + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(1, -1), 1) + + with self.assertRaises(SayWhenError): + self.pool.map(sqr, exception_throwing_generator(10, 3), 1) + + class SpecialIterable: + def __iter__(self): + return self + def __next__(self): + raise SayWhenError + def __len__(self): + return 1 + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + with self.assertRaises(SayWhenError): + self.pool.map(sqr, SpecialIterable(), 1) + def test_async(self): res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) get = TimingWrapper(res.get) @@ -1848,6 +1965,13 @@ def test_imap_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1) for i in range(3): self.assertEqual(next(it), i*i) @@ -1874,6 +1998,17 @@ def test_imap_unordered_handle_iterable_exception(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) + # SayWhenError seen at the very first of the iterable + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + # again, make sure it's reentrant + it = self.pool.imap_unordered(sqr, + exception_throwing_generator(1, -1), + 1) + self.assertRaises(SayWhenError, it.__next__) + it = self.pool.imap_unordered(sqr, exception_throwing_generator(10, 3), 1) @@ -1955,7 +2090,7 @@ def test_traceback(self): except Exception as e: exc = e else: - raise AssertionError('expected RuntimeError') + self.fail('expected RuntimeError') self.assertIs(type(exc), RuntimeError) self.assertEqual(exc.args, (123,)) cause = exc.__cause__ @@ -1969,6 +2104,17 @@ def test_traceback(self): sys.excepthook(*sys.exc_info()) self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + # _helper_reraises_exception should not make the error + # a remote exception + with self.Pool(1) as p: + try: + p.map(sqr, exception_throwing_generator(1, -1), 1) + except Exception as e: + exc = e + else: + self.fail('expected SayWhenError') + self.assertIs(type(exc), SayWhenError) + self.assertIs(exc.__cause__, None) @classmethod def _test_wrapped_exception(cls): @@ -2000,6 +2146,20 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.time() - t_start, 0.9) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + time.sleep(DELTA) # let threaded cleanup code run + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") @@ -2265,12 +2425,13 @@ def test_rapid_restart(self): manager.start() p = self.Process(target=self._putter, args=(manager.address, authkey)) - p.daemon = True p.start() + p.join() queue = manager.get_queue() self.assertEqual(queue.get(), 'hello world') del queue manager.shutdown() + manager = QueueManager( address=addr, authkey=authkey, serializer=SERIALIZER) try: @@ -2838,6 +2999,8 @@ def test_access(self): w.close() self.assertEqual(conn.recv(), 'foobar'*2) + p.join() + # # # @@ -2975,6 +3138,14 @@ class _TestFinalize(BaseTestCase): ALLOWED_TYPES = ('processes',) + def setUp(self): + self.registry_backup = util._finalizer_registry.copy() + util._finalizer_registry.clear() + + def tearDown(self): + self.assertFalse(util._finalizer_registry) + util._finalizer_registry.update(self.registry_backup) + @classmethod def _test_finalize(cls, conn): class Foo(object): @@ -3024,6 +3195,61 @@ def test_finalize(self): result = [obj for obj in iter(conn.recv, 'STOP')] self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) + def test_thread_safety(self): + # bpo-24484: _run_finalizers() should be thread-safe + def cb(): + pass + + class Foo(object): + def __init__(self): + self.ref = self # create reference cycle + # insert finalizer at random key + util.Finalize(self, cb, exitpriority=random.randint(1, 100)) + + finish = False + exc = None + + def run_finalizers(): + nonlocal exc + while not finish: + time.sleep(random.random() * 1e-1) + try: + # A GC run will eventually happen during this, + # collecting stale Foo's and mutating the registry + util._run_finalizers() + except Exception as e: + exc = e + + def make_finalizers(): + nonlocal exc + d = {} + while not finish: + try: + # Old Foo's get gradually replaced and later + # collected by the GC (because of the cyclic ref) + d[random.getrandbits(5)] = {Foo() for i in range(10)} + except Exception as e: + exc = e + d.clear() + + old_interval = sys.getswitchinterval() + old_threshold = gc.get_threshold() + try: + sys.setswitchinterval(1e-6) + gc.set_threshold(5, 5, 5) + threads = [threading.Thread(target=run_finalizers), + threading.Thread(target=make_finalizers)] + with test.support.start_threads(threads): + time.sleep(4.0) # Wait a bit to trigger race condition + finish = True + if exc is not None: + raise exc + finally: + sys.setswitchinterval(old_interval) + gc.set_threshold(*old_threshold) + gc.collect() # Collect remaining Foo's + + # # Test that from ... import * works for each module # @@ -3100,16 +3326,16 @@ def test_level(self): logger.setLevel(LEVEL1) p = self.Process(target=self._test_level, args=(writer,)) - p.daemon = True p.start() self.assertEqual(LEVEL1, reader.recv()) + p.join() logger.setLevel(logging.NOTSET) root_logger.setLevel(LEVEL2) p = self.Process(target=self._test_level, args=(writer,)) - p.daemon = True p.start() self.assertEqual(LEVEL2, reader.recv()) + p.join() root_logger.setLevel(root_level) logger.setLevel(level=LOG_LEVEL) @@ -3263,7 +3489,7 @@ def _this_sub_process(q): except pyqueue.Empty: pass -def _test_process(q): +def _test_process(): queue = multiprocessing.Queue() subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,)) subProc.daemon = True @@ -3303,8 +3529,7 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self): - queue = multiprocessing.Queue() - proc = multiprocessing.Process(target=_test_process, args=(queue,)) + proc = multiprocessing.Process(target=_test_process) proc.start() proc.join() @@ -3872,11 +4097,72 @@ def test_semaphore_tracker(self): self.assertRegex(err, expected) self.assertRegex(err, r'semaphore_tracker: %r: \[Errno' % name1) +class TestSimpleQueue(unittest.TestCase): + + @classmethod + def _test_empty(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + # issue 30301, could fail under spawn and forkserver + try: + queue.put(queue.empty()) + queue.put(queue.empty()) + finally: + parent_can_continue.set() + + def test_empty(self): + queue = multiprocessing.SimpleQueue() + child_can_start = multiprocessing.Event() + parent_can_continue = multiprocessing.Event() + + proc = multiprocessing.Process( + target=self._test_empty, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertTrue(queue.empty()) + + child_can_start.set() + parent_can_continue.wait() + + self.assertFalse(queue.empty()) + self.assertEqual(queue.get(), True) + self.assertEqual(queue.get(), False) + self.assertTrue(queue.empty()) + + proc.join() + # # Mixins # -class ProcessesMixin(object): +class BaseMixin(object): + @classmethod + def setUpClass(cls): + cls.dangling = (multiprocessing.process._dangling.copy(), + threading._dangling.copy()) + + @classmethod + def tearDownClass(cls): + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + + processes = set(multiprocessing.process._dangling) - set(cls.dangling[0]) + if processes: + print('Warning -- Dangling processes: %s' % processes, + file=sys.stderr) + processes = None + + threads = set(threading._dangling) - set(cls.dangling[1]) + if threads: + print('Warning -- Dangling threads: %s' % threads, + file=sys.stderr) + threads = None + + +class ProcessesMixin(BaseMixin): TYPE = 'processes' Process = multiprocessing.Process connection = multiprocessing.connection @@ -3899,7 +4185,7 @@ class ProcessesMixin(object): RawArray = staticmethod(multiprocessing.RawArray) -class ManagerMixin(object): +class ManagerMixin(BaseMixin): TYPE = 'manager' Process = multiprocessing.Process Queue = property(operator.attrgetter('manager.Queue')) @@ -3923,6 +4209,7 @@ def Pool(cls, *args, **kwds): @classmethod def setUpClass(cls): + super().setUpClass() cls.manager = multiprocessing.Manager() @classmethod @@ -3930,23 +4217,35 @@ def tearDownClass(cls): # only the manager process should be returned by active_children() # but this can take a bit on slow machines, so wait a few seconds # if there are other children too (see #17395) + start_time = time.monotonic() t = 0.01 - while len(multiprocessing.active_children()) > 1 and t < 5: + while len(multiprocessing.active_children()) > 1: time.sleep(t) t *= 2 + dt = time.monotonic() - start_time + if dt >= 5.0: + print("Warning -- multiprocessing.Manager still has %s active " + "children after %s seconds" + % (multiprocessing.active_children(), dt), + file=sys.stderr) + break + gc.collect() # do garbage collection if cls.manager._number_of_objects() != 0: # This is not really an error since some tests do not # ensure that all processes which hold a reference to a # managed object have been joined. - print('Shared objects which still exist at manager shutdown:') + print('Warning -- Shared objects which still exist at manager ' + 'shutdown:') print(cls.manager._debug_info()) cls.manager.shutdown() cls.manager.join() cls.manager = None + super().tearDownClass() + -class ThreadsMixin(object): +class ThreadsMixin(BaseMixin): TYPE = 'threads' Process = multiprocessing.dummy.Process connection = multiprocessing.dummy.connection @@ -4023,18 +4322,33 @@ def setUpModule(): multiprocessing.get_logger().setLevel(LOG_LEVEL) def tearDownModule(): + need_sleep = False + + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + multiprocessing.set_start_method(old_start_method[0], force=True) # pause a bit so we don't get warning about dangling threads/processes - time.sleep(0.5) + processes = set(multiprocessing.process._dangling) - set(dangling[0]) + if processes: + need_sleep = True + print('Warning -- Dangling processes: %s' % processes, + file=sys.stderr) + processes = None + + threads = set(threading._dangling) - set(dangling[1]) + if threads: + need_sleep = True + print('Warning -- Dangling threads: %s' % threads, + file=sys.stderr) + threads = None + + # Sleep 500 ms to give time to child processes to complete. + if need_sleep: + time.sleep(0.5) multiprocessing.process._cleanup() - gc.collect() - tmp = set(multiprocessing.process._dangling) - set(dangling[0]) - if tmp: - print('Dangling processes:', tmp, file=sys.stderr) - del tmp - tmp = set(threading._dangling) - set(dangling[1]) - if tmp: - print('Dangling threads:', tmp, file=sys.stderr) + test.support.gc_collect() remote_globs['setUpModule'] = setUpModule remote_globs['tearDownModule'] = tearDownModule diff --git a/Lib/test/bisect.py b/Lib/test/bisect.py new file mode 100755 index 00000000000000..0fd577d4e5619f --- /dev/null +++ b/Lib/test/bisect.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Command line tool to bisect failing CPython tests. + +Find the test_os test method which alters the environment: + + ./python -m test.bisect --fail-env-changed test_os + +Find a reference leak in "test_os", write the list of failing tests into the +"bisect" file: + + ./python -m test.bisect -o bisect -R 3:3 test_os + +Load an existing list of tests from a file using -i option: + + ./python -m test --list-cases -m FileTests test_os > tests + ./python -m test.bisect -i tests test_os +""" + +import argparse +import datetime +import os.path +import math +import random +import subprocess +import sys +import tempfile +import time + + +def write_tests(filename, tests): + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + fp.flush() + + +def write_output(filename, tests): + if not filename: + return + print("Write %s tests into %s" % (len(tests), filename)) + write_tests(filename, tests) + return filename + + +def format_shell_args(args): + return ' '.join(args) + + +def list_cases(args): + cmd = [sys.executable, '-m', 'test', '--list-cases'] + cmd.extend(args.test_args) + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + universal_newlines=True) + exitcode = proc.returncode + if exitcode: + cmd = format_shell_args(cmd) + print("Failed to list tests: %s failed with exit code %s" + % (cmd, exitcode)) + sys.exit(exitcode) + tests = proc.stdout.splitlines() + return tests + + +def run_tests(args, tests, huntrleaks=None): + tmp = tempfile.mktemp() + try: + write_tests(tmp, tests) + + cmd = [sys.executable, '-m', 'test', '--matchfile', tmp] + cmd.extend(args.test_args) + print("+ %s" % format_shell_args(cmd)) + proc = subprocess.run(cmd) + return proc.returncode + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input', + help='Test names produced by --list-tests written ' + 'into a file. If not set, run --list-tests') + parser.add_argument('-o', '--output', + help='Result of the bisection') + parser.add_argument('-n', '--max-tests', type=int, default=1, + help='Maximum number of tests to stop the bisection ' + '(default: 1)') + parser.add_argument('-N', '--max-iter', type=int, default=100, + help='Maximum number of bisection iterations ' + '(default: 100)') + # FIXME: document that following arguments are test arguments + + args, test_args = parser.parse_known_args() + args.test_args = test_args + return args + + +def main(): + args = parse_args() + + if args.input: + with open(args.input) as fp: + tests = [line.strip() for line in fp] + else: + tests = list_cases(args) + + print("Start bisection with %s tests" % len(tests)) + print("Test arguments: %s" % format_shell_args(args.test_args)) + print("Bisection will stop when getting %s or less tests " + "(-n/--max-tests option), or after %s iterations " + "(-N/--max-iter option)" + % (args.max_tests, args.max_iter)) + output = write_output(args.output, tests) + print() + + start_time = time.monotonic() + iteration = 1 + try: + while len(tests) > args.max_tests and iteration <= args.max_iter: + ntest = len(tests) + ntest = max(ntest // 2, 1) + subtests = random.sample(tests, ntest) + + print("[+] Iteration %s: run %s tests/%s" + % (iteration, len(subtests), len(tests))) + print() + + exitcode = run_tests(args, subtests) + + print("ran %s tests/%s" % (ntest, len(tests))) + print("exit", exitcode) + if exitcode: + print("Tests failed: use this new subtest") + tests = subtests + output = write_output(args.output, tests) + else: + print("Tests succeeded: skip this subtest, try a new subbset") + print() + iteration += 1 + except KeyboardInterrupt: + print() + print("Bisection interrupted!") + print() + + print("Tests (%s):" % len(tests)) + for test in tests: + print("* %s" % test) + print() + + if output: + print("Output written into %s" % output) + + dt = math.ceil(time.monotonic() - start_time) + if len(tests) <= args.max_tests: + print("Bisection completed in %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) + else: + print("Bisection failed after %s iterations and %s" + % (iteration, datetime.timedelta(seconds=dt))) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/cfgparser.1 b/Lib/test/cfgparser.1 index 3387f52eca6f94..0c0e0d35273a39 100644 --- a/Lib/test/cfgparser.1 +++ b/Lib/test/cfgparser.1 @@ -1,2 +1,3 @@ +# Also used by idlelib.test_idle.test_config. [Foo Bar] foo=newbar diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2350125f6d6dbe..77df1b3744d159 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -61,8 +61,9 @@ def test_constants(self): self.assertEqual(datetime.MAXYEAR, 9999) def test_name_cleanup(self): - if '_Fast' not in str(self): - return + if '_Pure' in self.__class__.__name__: + self.skipTest('Only run for Fast C implementation') + datetime = datetime_module names = set(name for name in dir(datetime) if not name.startswith('__') and not name.endswith('__')) @@ -72,8 +73,9 @@ def test_name_cleanup(self): self.assertEqual(names - allowed, set([])) def test_divide_and_round(self): - if '_Fast' in str(self): - return + if '_Fast' in self.__class__.__name__: + self.skipTest('Only run for Pure Python implementation') + dar = datetime_module._divide_and_round self.assertEqual(dar(-10, -3), 3) @@ -844,6 +846,26 @@ def test_divmod(self): self.assertRaises(TypeError, divmod, t, 10) + def test_issue31293(self): + # The interpreter shouldn't crash in case a timedelta is divided or + # multiplied by a float with a bad as_integer_ratio() method. + def get_bad_float(bad_ratio): + class BadFloat(float): + def as_integer_ratio(self): + return bad_ratio + return BadFloat() + + with self.assertRaises(TypeError): + timedelta() / get_bad_float(1 << 1000) + with self.assertRaises(TypeError): + timedelta() * get_bad_float(1 << 1000) + + for bad_ratio in [(), (42, ), (1, 2, 3)]: + with self.assertRaises(ValueError): + timedelta() / get_bad_float(bad_ratio) + with self.assertRaises(ValueError): + timedelta() * get_bad_float(bad_ratio) + ############################################################################# # date tests @@ -2851,7 +2873,7 @@ def tzname(self, dt): return self.tz self.assertRaises(TypeError, t.strftime, "%Z") # Issue #6697: - if '_Fast' in str(self): + if '_Fast' in self.__class__.__name__: Badtzname.tz = '\ud800' self.assertRaises(ValueError, t.strftime, "%Z") @@ -4313,6 +4335,11 @@ def test_replace(self): dt = dt.replace(fold=1, tzinfo=Eastern) self.assertEqual(t.replace(tzinfo=None).fold, 1) self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) # Check that fold is a keyword-only argument with self.assertRaises(TypeError): t.replace(1, 1, 1, None, 1) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index d194e775eaeb8f..1dbe88efe70ffd 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -371,59 +371,55 @@ def test_sleep(self): @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +# bpo-30320: Need pthread_sigmask() to block the signal, otherwise the test +# is vulnerable to a race condition between the child and the parent processes. +@unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') class SignalEINTRTest(EINTRBaseTest): """ EINTR tests for the signal module. """ - @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), - 'need signal.sigtimedwait()') - def test_sigtimedwait(self): - t0 = time.monotonic() - signal.sigtimedwait([signal.SIGUSR1], self.sleep_time) - dt = time.monotonic() - t0 - self.assertGreaterEqual(dt, self.sleep_time) - - @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), - 'need signal.sigwaitinfo()') - def test_sigwaitinfo(self): - # Issue #25277, #25868: give a few milliseconds to the parent process - # between os.write() and signal.sigwaitinfo() to works around a race - # condition - self.sleep_time = 0.100 - + def check_sigwait(self, wait_func): signum = signal.SIGUSR1 pid = os.getpid() old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) - rpipe, wpipe = os.pipe() - code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, - 'rpipe = %r' % rpipe, - 'os.read(rpipe, 1)', - 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) + old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) + t0 = time.monotonic() - proc = self.subprocess(code, pass_fds=(rpipe,)) - os.close(rpipe) + proc = self.subprocess(code) with kill_on_error(proc): - # sync child-parent - os.write(wpipe, b'x') - os.close(wpipe) + wait_func(signum) + dt = time.monotonic() - t0 + + self.assertEqual(proc.wait(), 0) - # parent + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + def wait_func(signum): signal.sigwaitinfo([signum]) - dt = time.monotonic() - t0 - self.assertEqual(proc.wait(), 0) - self.assertGreaterEqual(dt, self.sleep_time) + self.check_sigwait(wait_func) + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigwaitinfo()') + def test_sigtimedwait(self): + def wait_func(signum): + signal.sigtimedwait([signum], 120.0) + + self.check_sigwait(wait_func) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -437,6 +433,8 @@ def test_select(self): self.stop_alarm() self.assertGreaterEqual(dt, self.sleep_time) + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') def test_poll(self): poller = select.poll() diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 713039dd826872..6af79ad5d4d62b 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -11,7 +11,8 @@ import os, sys, time, unittest import test.support as support -_thread = support.import_module('_thread') + +threading = support.import_module('threading') LONGSLEEP = 2 SHORTSLEEP = 0.5 @@ -20,8 +21,19 @@ class ForkWait(unittest.TestCase): def setUp(self): + self._threading_key = support.threading_setup() self.alive = {} self.stop = 0 + self.threads = [] + + def tearDown(self): + # Stop threads + self.stop = 1 + for thread in self.threads: + thread.join() + thread = None + self.threads.clear() + support.threading_cleanup(*self._threading_key) def f(self, id): while not self.stop: @@ -43,10 +55,11 @@ def wait_impl(self, cpid): self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) - @support.reap_threads def test_wait(self): for i in range(NUM_THREADS): - _thread.start_new(self.f, (i,)) + thread = threading.Thread(target=self.f, args=(i,)) + thread.start() + self.threads.append(thread) # busy-loop to wait for threads deadline = time.monotonic() + 10.0 @@ -75,8 +88,4 @@ def test_wait(self): os._exit(n) else: # Parent - try: - self.wait_impl(cpid) - finally: - # Tell threads to die - self.stop = 1 + self.wait_impl(cpid) diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 7ba0e6e5356166..3427b51b60af86 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,5 +1,5 @@ # We import importlib *ASAP* in order to test #15386 import importlib -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d621f5f9f3ee1f..4999fa734114a1 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -117,11 +117,27 @@ To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. + +--matchfile filters tests using a text file, one pattern per line. +Pattern examples: + +- test method: test_stat_attributes +- test class: FileTests +- test identifier: test_os.FileTests.test_stat_attributes """ -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'tzdata') +ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +# Other resources excluded from --use=all: +# +# - extralagefile (ex: test_zipfile64): really too slow to be enabled +# "by default" +# - tzdata: while needed to validate fully test_datetime, it makes +# test_datetime too slow (15-20 min on some buildbots) and so is disabled by +# default (see bpo-30822). +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') class _ArgParser(argparse.ArgumentParser): @@ -189,8 +205,12 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', + dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('--matchfile', metavar='FILENAME', + dest='match_filename', + help='similar to --match but get patterns from a ' + 'text file, one pattern per line') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -239,8 +259,14 @@ def _create_parser(): group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") + group.add_argument('--list-cases', action='store_true', + help='only write the name of test cases that will be run' + ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization training') + group.add_argument('--fail-env-changed', action='store_true', + help='if a test file alters the environment, mark ' + 'the test as failed') return parser @@ -327,7 +353,7 @@ def _parse_args(args, **kwargs): for a in ns.use: for r in a: if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES + ns.use_resources[:] = ALL_RESOURCES continue if r == 'none': del ns.use_resources[:] @@ -343,5 +369,19 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.verbose: + ns.header = True + if ns.huntrleaks and ns.verbose3: + ns.verbose3 = False + print("WARNING: Disable --verbose3 because it's incompatible with " + "--huntrleaks: see http://bugs.python.org/issue27103", + file=sys.stderr) + if ns.match_filename: + if ns.match_tests is None: + ns.match_tests = [] + filename = os.path.join(support.SAVEDCWD, ns.match_filename) + with open(filename) as fp: + for line in fp: + ns.match_tests.append(line.strip()) return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f0effc973c1b3a..d44f69dda2f70e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,9 +10,10 @@ import tempfile import textwrap import time +import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest, get_abs_module, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, format_test_result) @@ -28,7 +29,13 @@ # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') + TEMPDIR = sysconfig.get_config_var('abs_builddir') + if TEMPDIR is None: + # bpo-30284: On Windows, only srcdir is available. Using abs_builddir + # mostly matters on UNIX when building Python out of the source tree, + # especially when the source tree is read only. + TEMPDIR = sysconfig.get_config_var('srcdir') + TEMPDIR = os.path.join(TEMPDIR, 'build') else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) @@ -107,7 +114,7 @@ def accumulate_result(self, test, result): self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) - elif ok == FAILED: + elif ok in (FAILED, CHILD_ERROR): self.bad.append(test) elif ok == ENV_CHANGED: self.environment_changed.append(test) @@ -116,22 +123,28 @@ def accumulate_result(self, test, result): elif ok == RESOURCE_DENIED: self.skipped.append(test) self.resource_denieds.append(test) + elif ok != INTERRUPTED: + raise ValueError("invalid test result: %r" % ok) def display_progress(self, test_index, test): if self.ns.quiet: return + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count}" if self.bad and not self.ns.pgo: - fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" - else: - fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + line = f"{line}/{len(self.bad)}" + line = f"[{line}] {test}" + + # add the system load prefix: "load avg: 1.80 " + if hasattr(os, 'getloadavg'): + load_avg_1min = os.getloadavg()[0] + line = f"load avg: {load_avg_1min:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " test_time = time.monotonic() - self.start_time test_time = datetime.timedelta(seconds=int(test_time)) - line = fmt.format(count_width=self.test_count_width, - test_index=test_index, - test_count=self.test_count, - nbad=len(self.bad), - test_name=test, - time=test_time) + line = f"{test_time} {line}" print(line, flush=True) def parse_args(self, kwargs): @@ -179,19 +192,14 @@ def find_tests(self, tests): self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' - regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' - r'(?:\[[0-9/ ]+\] *)?' - r'(test_[a-zA-Z0-9_]+)') - regex = re.compile(regex) + regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: for line in fp: + line = line.split('#', 1)[0] line = line.strip() - if line.startswith('#'): - continue - match = regex.match(line) - if match is None: - continue - self.tests.append(match.group(1)) + match = regex.search(line) + if match is not None: + self.tests.append(match.group()) removepy(self.tests) @@ -241,6 +249,33 @@ def list_tests(self): for name in self.selected: print(name) + def _list_cases(self, suite): + for test in suite: + if isinstance(test, unittest.loader._FailedTest): + continue + if isinstance(test, unittest.TestSuite): + self._list_cases(test) + elif isinstance(test, unittest.TestCase): + if support._match_test(test): + print(test.id()) + + def list_cases(self): + support.verbose = False + support.match_tests = self.ns.match_tests + + for test in self.selected: + abstest = get_abs_module(self.ns, test) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(abstest) + self._list_cases(suite) + except unittest.SkipTest: + self.skipped.append(test) + + if self.skipped: + print(file=sys.stderr) + print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) + printlist(self.skipped, file=sys.stderr) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -380,24 +415,28 @@ def _test_forever(self, tests): yield test if self.bad: return + if self.ns.fail_env_changed and self.environment_changed: + return + + def display_header(self): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== cwd:", os.getcwd()) + cpu_count = os.cpu_count() + if cpu_count: + print("== CPU count:", cpu_count) + print("== encodings: locale=%s, FS=%s" + % (locale.getpreferredencoding(False), + sys.getfilesystemencoding())) def run_tests(self): # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== cwd:", os.getcwd()) - print("== encodings: locale=%s, FS=%s" - % (locale.getpreferredencoding(False), - sys.getfilesystemencoding())) - print("Testing with flags:", sys.flags) + if (self.ns.header + or not(self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + self.display_header() if self.ns.randomize: print("Using random seed", self.ns.random_seed) @@ -438,6 +477,8 @@ def finalize(self): result = "FAILURE" elif self.interrupted: result = "INTERRUPTED" + elif self.ns.fail_env_changed and self.environment_changed: + result = "ENV CHANGED" else: result = "SUCCESS" print("Tests result: %s" % result) @@ -487,6 +528,10 @@ def _main(self, tests, kwargs): self.list_tests() sys.exit(0) + if self.ns.list_cases: + self.list_cases() + sys.exit(0) + self.run_tests() self.display_result() @@ -494,7 +539,13 @@ def _main(self, tests, kwargs): self.rerun_failed_tests() self.finalize() - sys.exit(len(self.bad) > 0 or self.interrupted) + if self.bad: + sys.exit(2) + if self.interrupted: + sys.exit(130) + if self.ns.fail_env_changed and self.environment_changed: + sys.exit(3) + sys.exit(0) def removepy(names): @@ -513,7 +564,7 @@ def count(n, word): return "%d %ss" % (n, word) -def printlist(x, width=70, indent=4): +def printlist(x, width=70, indent=4, file=None): """Print the elements of iterable x to stdout. Optional arg width (default 70) is the maximum line length. @@ -524,7 +575,8 @@ def printlist(x, width=70, indent=4): blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + initial_indent=blanks, subsequent_indent=blanks), + file=file) def main(tests=None, **kwargs): diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d2458b1f2e0..0bd8288e276522 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -68,6 +68,14 @@ def dash_R(the_module, test, indirect_test, huntrleaks): for obj in abc.__subclasses__() + [abc]: abcs[obj] = obj._abc_registry.copy() + # bpo-31217: Integer pool to get a single integer object for the same + # value. The pool is used to prevent false alarm when checking for memory + # block leaks. Fill the pool with values in -1000..1000 which are the most + # common (reference, memory block, file descriptor) differences. + int_pool = {value: value for value in range(-1000, 1000)} + def get_pooled_int(value): + return int_pool.setdefault(value, value) + nwarmup, ntracked, fname = huntrleaks fname = os.path.join(support.SAVEDCWD, fname) repcount = nwarmup + ntracked @@ -84,34 +92,46 @@ def dash_R(the_module, test, indirect_test, huntrleaks): indirect_test() alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - print('.', end='', flush=True) + print('.', end='', file=sys.stderr, flush=True) if i >= nwarmup: - rc_deltas[i] = rc_after - rc_before - alloc_deltas[i] = alloc_after - alloc_before - fd_deltas[i] = fd_after - fd_before + rc_deltas[i] = get_pooled_int(rc_after - rc_before) + alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) + fd_deltas[i] = get_pooled_int(fd_after - fd_before) alloc_before = alloc_after rc_before = rc_after fd_before = fd_after print(file=sys.stderr) + # These checkers return False on success, True on failure def check_rc_deltas(deltas): + # Checker for reference counters and memomry blocks. + # + # bpo-30776: Try to ignore false positives: + # + # [3, 0, 0] + # [0, 1, 0] + # [8, -8, 1] + # + # Expected leaks: + # + # [5, 5, 6] + # [10, 1, 1] + return all(delta >= 1 for delta in deltas) + + def check_fd_deltas(deltas): return any(deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False + failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas), - (fd_deltas, 'file descriptors', check_rc_deltas)]: + (alloc_deltas, 'memory blocks', check_rc_deltas), + (fd_deltas, 'file descriptors', check_fd_deltas) + ]: + # ignore warmup runs + deltas = deltas[nwarmup:] if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) + test, deltas, item_name, sum(deltas)) print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) @@ -143,9 +163,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ba0df0a317a500..d716b83e235472 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -71,6 +71,14 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def get_abs_module(ns, test): + if test.startswith('test.') or ns.testdir: + return test + else: + # Always import it from the test package + return 'test.' + test + + def runtest(ns, test): """Run a single test. @@ -135,17 +143,17 @@ def runtest(ns, test): runtest.stringio = None +def post_test_cleanup(): + support.reap_children() + + def runtest_inner(ns, test, display_failure=True): support.unload(test) test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.') or ns.testdir: - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test + abstest = get_abs_module(ns, test) clear_caches() with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: start_time = time.time() @@ -166,6 +174,7 @@ def test_runner(): if ns.huntrleaks: refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) test_time = time.time() - start_time + post_test_cleanup() except support.ResourceDenied as msg: if not ns.quiet and not ns.pgo: print(test, "skipped --", msg, flush=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 9604c166009fe9..779ff01a649c6c 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -41,7 +41,7 @@ def run_test_in_subprocess(testname, ns): slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), - '-X', 'faulthandler', + '-u', # Unbuffered stdout and stderr '-m', 'test.regrtest', '--slaveargs', slaveargs] if ns.pgo: @@ -124,13 +124,13 @@ def _runtest(self): finally: self.current_test = None - stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return True + return False + stdout, _, result = stdout.strip().rpartition("\n") if not result: self.output.put((None, None, None, None)) return True @@ -203,6 +203,8 @@ def get_running(workers): and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): text += ' (%.0f sec)' % test_time + elif ok == CHILD_ERROR: + text = '%s (%s)' % (text, test_time) running = get_running(workers) if running and not regrtest.ns.pgo: text += ' -- running: %s' % ', '.join(running) @@ -216,9 +218,6 @@ def get_running(workers): if result[0] == INTERRUPTED: raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - msg = "Child error on {}: {}".format(test, result[1]) - raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 96ad3af8df4cf7..8309f266bfbe9b 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -279,7 +279,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not self.quiet and not self.pgo: print(f"Warning -- {name} was modified by {self.testname}", file=sys.stderr, flush=True) - if self.verbose > 1: - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print(f" Before: {original}\n After: {current} ", + file=sys.stderr, flush=True) return False diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index a6cb3b169b6d73..4fa154f734d070 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -461,21 +461,28 @@ def _check_notify(self, cond): # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. N = 5 + ready = [] results1 = [] results2 = [] phase_num = 0 def f(): cond.acquire() + ready.append(phase_num) result = cond.wait() cond.release() results1.append((result, phase_num)) cond.acquire() + ready.append(phase_num) result = cond.wait() cond.release() results2.append((result, phase_num)) b = Bunch(f, N) b.wait_for_started() - _wait() + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issues #8799 and #30727. + while len(ready) < 5: + _wait() + ready.clear() self.assertEqual(results1, []) # Notify 3 threads at first cond.acquire() @@ -487,9 +494,9 @@ def f(): _wait() self.assertEqual(results1, [(True, 1)] * 3) self.assertEqual(results2, []) - # first wait, to ensure all workers settle into cond.wait() before - # we continue. See issue #8799 - _wait() + # make sure all awaken workers settle into cond.wait() + while len(ready) < 3: + _wait() # Notify 5 threads: they might be in their first or second wait cond.acquire() cond.notify(5) @@ -500,7 +507,9 @@ def f(): _wait() self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) self.assertEqual(results2, [(True, 2)] * 3) - _wait() # make sure all workers settle into cond.wait() + # make sure all workers settle into cond.wait() + while len(ready) < 5: + _wait() # Notify all threads: they are all in their second wait cond.acquire() cond.notify_all() diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py new file mode 100644 index 00000000000000..6d35c58396d424 --- /dev/null +++ b/Lib/test/mod_generics_cache.py @@ -0,0 +1,53 @@ +"""Module for testing the behavior of generics across different modules.""" + +import sys +from textwrap import dedent +from typing import TypeVar, Generic, Optional + + +if sys.version_info[:2] >= (3, 6): + exec(dedent(""" + default_a: Optional['A'] = None + default_b: Optional['B'] = None + + T = TypeVar('T') + + + class A(Generic[T]): + some_b: 'B' + + + class B(Generic[T]): + class A(Generic[T]): + pass + + my_inner_a1: 'B.A' + my_inner_a2: A + my_outer_a: 'A' # unless somebody calls get_type_hints with localns=B.__dict__ + """)) +else: # This should stay in sync with the syntax above. + __annotations__ = dict( + default_a=Optional['A'], + default_b=Optional['B'], + ) + default_a = None + default_b = None + + T = TypeVar('T') + + + class A(Generic[T]): + __annotations__ = dict( + some_b='B' + ) + + + class B(Generic[T]): + class A(Generic[T]): + pass + + __annotations__ = dict( + my_inner_a1='B.A', + my_inner_a2=A, + my_outer_a='A' # unless somebody calls get_type_hints with localns=B.__dict__ + ) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py new file mode 100644 index 00000000000000..5447ab8274b4e7 --- /dev/null +++ b/Lib/test/pythoninfo.py @@ -0,0 +1,478 @@ +""" +Collect various informations about Python to help debugging test failures. +""" +from __future__ import print_function +import errno +import re +import sys +import traceback + + +def normalize_text(text): + if text is None: + return None + text = str(text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +class PythonInfo: + def __init__(self): + self.info = {} + + def add(self, key, value): + if key in self.info: + raise ValueError("duplicate key: %r" % key) + + if value is None: + return + + if not isinstance(value, int): + if not isinstance(value, str): + # convert other objects like sys.flags to string + value = str(value) + + value = value.strip() + if not value: + return + + self.info[key] = value + + def get_infos(self): + """ + Get informations as a key:value dictionary where values are strings. + """ + return {key: str(value) for key, value in self.info.items()} + + +def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None): + for attr in attributes: + value = getattr(obj, attr, None) + if value is None: + continue + name = name_fmt % attr + if formatter is not None: + value = formatter(attr, value) + info_add(name, value) + + +def call_func(info_add, name, mod, func_name, *, formatter=None): + try: + func = getattr(mod, func_name) + except AttributeError: + return + value = func() + if formatter is not None: + value = formatter(value) + info_add(name, value) + + +def collect_sys(info_add): + attributes = ( + '_framework', + 'abiflags', + 'api_version', + 'builtin_module_names', + 'byteorder', + 'dont_write_bytecode', + 'executable', + 'flags', + 'float_info', + 'float_repr_style', + 'hash_info', + 'hexversion', + 'implementation', + 'int_info', + 'maxsize', + 'maxunicode', + 'path', + 'platform', + 'prefix', + 'thread_info', + 'version', + 'version_info', + 'winver', + ) + copy_attributes(info_add, sys, 'sys.%s', attributes) + + call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') + call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') + + encoding = sys.getfilesystemencoding() + if hasattr(sys, 'getfilesystemencodeerrors'): + encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors()) + info_add('sys.filesystem_encoding', encoding) + + for name in ('stdin', 'stdout', 'stderr'): + stream = getattr(sys, name) + if stream is None: + continue + encoding = getattr(stream, 'encoding', None) + if not encoding: + continue + errors = getattr(stream, 'errors', None) + if errors: + encoding = '%s/%s' % (encoding, errors) + info_add('sys.%s.encoding' % name, encoding) + + +def collect_platform(info_add): + import platform + + arch = platform.architecture() + arch = ' '.join(filter(bool, arch)) + info_add('platform.architecture', arch) + + info_add('platform.python_implementation', + platform.python_implementation()) + info_add('platform.platform', + platform.platform(aliased=True)) + + +def collect_locale(info_add): + import locale + + info_add('locale.encoding', locale.getpreferredencoding(False)) + + +def collect_os(info_add): + import os + + def format_attr(attr, value): + if attr in ('supports_follow_symlinks', 'supports_fd', + 'supports_effective_ids'): + return str(sorted(func.__name__ for func in value)) + else: + return value + + attributes = ( + 'name', + 'supports_bytes_environ', + 'supports_effective_ids', + 'supports_fd', + 'supports_follow_symlinks', + ) + copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) + + info_add("os.cwd", os.getcwd()) + + call_func(info_add, 'os.uid', os, 'getuid') + call_func(info_add, 'os.gid', os, 'getgid') + call_func(info_add, 'os.uname', os, 'uname') + + if hasattr(os, 'getgroups'): + groups = os.getgroups() + groups = map(str, groups) + groups = ', '.join(groups) + info_add("os.groups", groups) + + if hasattr(os, 'getlogin'): + try: + login = os.getlogin() + except OSError: + # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl + # for device" on Travis CI + pass + else: + info_add("os.login", login) + + if hasattr(os, 'cpu_count'): + cpu_count = os.cpu_count() + if cpu_count: + info_add('os.cpu_count', cpu_count) + + call_func(info_add, 'os.loadavg', os, 'getloadavg') + + # Get environment variables: filter to list + # to not leak sensitive information + ENV_VARS = ( + "CC", + "COMSPEC", + "DISPLAY", + "DISTUTILS_USE_SDK", + "DYLD_LIBRARY_PATH", + "HOME", + "HOMEDRIVE", + "HOMEPATH", + "LANG", + "LD_LIBRARY_PATH", + "MACOSX_DEPLOYMENT_TARGET", + "MAKEFLAGS", + "MSSDK", + "PATH", + "SDK_TOOLS_BIN", + "SHELL", + "TEMP", + "TERM", + "TMP", + "TMPDIR", + "USERPROFILE", + "WAYLAND_DISPLAY", + ) + for name, value in os.environ.items(): + uname = name.upper() + if (uname in ENV_VARS or uname.startswith(("PYTHON", "LC_")) + # Visual Studio: VS140COMNTOOLS + or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): + info_add('os.environ[%s]' % name, value) + + if hasattr(os, 'umask'): + mask = os.umask(0) + os.umask(mask) + info_add("os.umask", '%03o' % mask) + + if hasattr(os, 'getrandom'): + # PEP 524: Check if system urandom is initialized + try: + try: + os.getrandom(1, os.GRND_NONBLOCK) + state = 'ready (initialized)' + except BlockingIOError as exc: + state = 'not seeded yet (%s)' % exc + info_add('os.getrandom', state) + except OSError as exc: + # Python was compiled on a more recent Linux version + # than the current Linux kernel: ignore OSError(ENOSYS) + if exc.errno != errno.ENOSYS: + raise + + +def collect_readline(info_add): + try: + import readline + except ImportError: + return + + def format_attr(attr, value): + if isinstance(value, int): + return "%#x" % value + else: + return value + + attributes = ( + "_READLINE_VERSION", + "_READLINE_RUNTIME_VERSION", + "_READLINE_LIBRARY_VERSION", + ) + copy_attributes(info_add, readline, 'readline.%s', attributes, + formatter=format_attr) + + +def collect_gdb(info_add): + import subprocess + + try: + proc = subprocess.Popen(["gdb", "-nx", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + version = proc.communicate()[0] + except OSError: + return + + # Only keep the first line + version = version.splitlines()[0] + info_add('gdb_version', version) + + +def collect_tkinter(info_add): + try: + import _tkinter + except ImportError: + pass + else: + attributes = ('TK_VERSION', 'TCL_VERSION') + copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes) + + try: + import tkinter + except ImportError: + pass + else: + tcl = tkinter.Tcl() + patchlevel = tcl.call('info', 'patchlevel') + info_add('tkinter.info_patchlevel', patchlevel) + + +def collect_time(info_add): + import time + + attributes = ( + 'altzone', + 'daylight', + 'timezone', + 'tzname', + ) + copy_attributes(info_add, time, 'time.%s', attributes) + + if not hasattr(time, 'get_clock_info'): + return + + for clock in ('time', 'perf_counter'): + tinfo = time.get_clock_info(clock) + info_add('time.%s' % clock, tinfo) + + +def collect_sysconfig(info_add): + import sysconfig + + for name in ( + 'ABIFLAGS', + 'ANDROID_API_LEVEL', + 'CC', + 'CCSHARED', + 'CFLAGS', + 'CFLAGSFORSHARED', + 'PY_LDFLAGS', + 'CONFIG_ARGS', + 'HOST_GNU_TYPE', + 'MACHDEP', + 'MULTIARCH', + 'OPT', + 'PY_CFLAGS', + 'PY_CFLAGS_NODIST', + 'Py_DEBUG', + 'Py_ENABLE_SHARED', + 'SHELL', + 'SOABI', + 'prefix', + ): + value = sysconfig.get_config_var(name) + if name == 'ANDROID_API_LEVEL' and not value: + # skip ANDROID_API_LEVEL=0 + continue + value = normalize_text(value) + info_add('sysconfig[%s]' % name, value) + + +def collect_ssl(info_add): + try: + import ssl + except ImportError: + return + + def format_attr(attr, value): + if attr.startswith('OP_'): + return '%#8x' % value + else: + return value + + attributes = ( + 'OPENSSL_VERSION', + 'OPENSSL_VERSION_INFO', + 'HAS_SNI', + 'OP_ALL', + 'OP_NO_TLSv1_1', + ) + copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr) + + +def collect_socket(info_add): + import socket + + hostname = socket.gethostname() + info_add('socket.hostname', hostname) + + +def collect_sqlite(info_add): + try: + import sqlite3 + except ImportError: + return + + attributes = ('version', 'sqlite_version') + copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes) + + +def collect_zlib(info_add): + try: + import zlib + except ImportError: + return + + attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') + copy_attributes(info_add, zlib, 'zlib.%s', attributes) + + +def collect_expat(info_add): + try: + from xml.parsers import expat + except ImportError: + return + + attributes = ('EXPAT_VERSION',) + copy_attributes(info_add, expat, 'expat.%s', attributes) + + +def collect_decimal(info_add): + try: + import _decimal + except ImportError: + return + + attributes = ('__libmpdec_version__',) + copy_attributes(info_add, _decimal, '_decimal.%s', attributes) + + +def collect_info(info): + error = False + info_add = info.add + + for collect_func in ( + # collect_os() should be the first, to check the getrandom() status + collect_os, + + collect_gdb, + collect_locale, + collect_platform, + collect_readline, + collect_socket, + collect_sqlite, + collect_ssl, + collect_sys, + collect_sysconfig, + collect_time, + collect_tkinter, + collect_zlib, + collect_expat, + collect_decimal, + ): + try: + collect_func(info_add) + except Exception as exc: + error = True + print("ERROR: %s() failed" % (collect_func.__name__), + file=sys.stderr) + traceback.print_exc(file=sys.stderr) + print(file=sys.stderr) + sys.stderr.flush() + + return error + + +def dump_info(info, file=None): + title = "Python debug information" + print(title) + print("=" * len(title)) + print() + + infos = info.get_infos() + infos = sorted(infos.items()) + for key, value in infos: + value = value.replace("\n", " ") + print("%s: %s" % (key, value)) + print() + + +def main(): + info = PythonInfo() + error = collect_info(info) + dump_info(info) + + if error: + print("Collection failed: exit with error", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/sha256.pem b/Lib/test/sha256.pem deleted file mode 100644 index d3db4b85c0a882..00000000000000 --- a/Lib/test/sha256.pem +++ /dev/null @@ -1,128 +0,0 @@ -# Certificate chain for https://sha256.tbs-internet.com - 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com - i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC ------BEGIN CERTIFICATE----- -MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl -bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u -ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV -BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV -BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM -VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS -c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 -LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m -PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf -PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E -LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo -qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 -IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU -xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC -BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG -CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw -MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D -UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv -bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv -bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw -AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw -NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH -Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV -HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt -aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX -ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 -UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN -21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun -aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT -XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q ------END CERTIFICATE----- - 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC - i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root ------BEGIN CERTIFICATE----- -MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv -MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk -ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF -eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow -gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl -bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u -ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv -cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg -Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 -rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 -9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ -ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk -owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G -Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk -9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf -2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ -MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 -AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk -ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k -by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw -cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV -VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B -ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN -AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 -euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY -1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 -RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz -8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV -v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= ------END CERTIFICATE----- - 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC ------BEGIN CERTIFICATE----- -MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug -Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT -AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 -ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB -IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 -4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 -2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh -alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv -u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW -xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p -XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd -tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX -BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov -L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN -AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO -rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd -FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM -+bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI -3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb -+M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= ------END CERTIFICATE----- - 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC - i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC ------BEGIN CERTIFICATE----- -MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB -kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug -Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho -dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw -IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG -EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD -VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu -dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 -E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ -D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK -4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq -lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW -bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB -o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT -MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js -LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr -BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB -AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft -Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj -j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH -KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv -2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 -mfnGV/TJVTl4uix5yaaIK/QI ------END CERTIFICATE----- diff --git a/Lib/test/sndhdrdata/README b/Lib/test/sndhdrdata/README index 8a17c0041a463f..b2cb664615b9c7 100644 --- a/Lib/test/sndhdrdata/README +++ b/Lib/test/sndhdrdata/README @@ -3,10 +3,3 @@ following commands: dd if=/dev/zero of=sndhdr.raw bs=20 count=1 sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15d8fc849b9a0a..a5b8c46f9495b0 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1898,6 +1898,23 @@ def _run_suite(suite): raise TestFailed(err) +def _match_test(test): + global match_tests + + if match_tests is None: + return True + test_id = test.id() + + for match_test in match_tests: + if fnmatch.fnmatchcase(test_id, match_test): + return True + + for name in test_id.split("."): + if fnmatch.fnmatchcase(name, match_test): + return True + return False + + def run_unittest(*classes): """Run tests from unittest.TestCase-derived classes.""" valid_types = (unittest.TestSuite, unittest.TestCase) @@ -1912,14 +1929,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - def case_pred(test): - if match_tests is None: - return True - for name in test.id().split("."): - if fnmatch.fnmatchcase(name, match_tests): - return True - return False - _filter_suite(suite, case_pred) + _filter_suite(suite, _match_test) _run_suite(suite) #======================================================================= @@ -2043,7 +2053,6 @@ def reap_children(): stick around to hog resources and create problems when looking for refleaks. """ - # Reap all our dead child processes so we don't leave zombies around. # These hog resources and might be causing some of the buildbots to die. if hasattr(os, 'waitpid'): @@ -2054,6 +2063,8 @@ def reap_children(): pid, status = os.waitpid(any_process, os.WNOHANG) if pid == 0: break + print("Warning -- reap_children() reaped child process %s" + % pid, file=sys.stderr) except: break @@ -2105,12 +2116,15 @@ def swap_attr(obj, attr, new_val): restoring the old value at the end of the block. If `attr` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if hasattr(obj, attr): real_val = getattr(obj, attr) setattr(obj, attr, new_val) try: - yield + yield real_val finally: setattr(obj, attr, real_val) else: @@ -2118,7 +2132,8 @@ def swap_attr(obj, attr, new_val): try: yield finally: - delattr(obj, attr) + if hasattr(obj, attr): + delattr(obj, attr) @contextlib.contextmanager def swap_item(obj, item, new_val): @@ -2132,12 +2147,15 @@ def swap_item(obj, item, new_val): restoring the old value at the end of the block. If `item` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if item in obj: real_val = obj[item] obj[item] = new_val try: - yield + yield real_val finally: obj[item] = real_val else: @@ -2145,7 +2163,8 @@ def swap_item(obj, item, new_val): try: yield finally: - del obj[item] + if item in obj: + del obj[item] def strip_python_stderr(stderr): """Strip the stderr of a Python process from potential debug output @@ -2433,6 +2452,7 @@ def __enter__(self): (0, self.old_value[1])) except (ValueError, OSError): pass + if sys.platform == 'darwin': # Check if the 'Crash Reporter' on OSX was configured # in 'Developer' mode and warn that it will get triggered @@ -2440,10 +2460,14 @@ def __enter__(self): # # This assumes that this context manager is used in tests # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': + cmd = ['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with proc: + stdout = proc.communicate()[0] + if stdout.strip() == b'developer': print("this test triggers the Crash Reporter, " "that is intentional", end='', flush=True) @@ -2581,3 +2605,19 @@ def setswitchinterval(interval): if _is_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) + + +@contextlib.contextmanager +def disable_faulthandler(): + # use sys.__stderr__ instead of sys.stderr, since regrtest replaces + # sys.stderr with a StringIO which has no file descriptor when a test + # is run with -W/--verbose3. + fd = sys.__stderr__.fileno() + + is_enabled = faulthandler.is_enabled() + try: + faulthandler.disable() + yield + finally: + if is_enabled: + faulthandler.enable(file=fd, all_threads=True) diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index e1765f0d5a54cb..4bc838200413af 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -404,5 +404,17 @@ class C(A, B): self.assertEqual(B.counter, 1) +class TestABCWithInitSubclass(unittest.TestCase): + def test_works_with_init_subclass(self): + saved_kwargs = {} + class ReceivesClassKwargs: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__() + saved_kwargs.update(kwargs) + class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): + pass + self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py index 1bd1f89c8aa609..a731a5136ba5fd 100644 --- a/Lib/test/test_aifc.py +++ b/Lib/test/test_aifc.py @@ -1,5 +1,6 @@ -from test.support import findfile, TESTFN, unlink +from test.support import check_no_resource_warning, findfile, TESTFN, unlink import unittest +from unittest import mock from test import audiotests from audioop import byteswap import io @@ -149,6 +150,21 @@ def test_skipunknown(self): #This file contains chunk types aifc doesn't recognize. self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif')) + def test_close_opened_files_on_error(self): + non_aifc_file = findfile('pluck-pcm8.wav', subdir='audiodata') + with check_no_resource_warning(self): + with self.assertRaises(aifc.Error): + # Try opening a non-AIFC file, with the expectation that + # `aifc.open` will fail (without raising a ResourceWarning) + self.f = aifc.open(non_aifc_file, 'rb') + + # Aifc_write.initfp() won't raise in normal case. But some errors + # (e.g. MemoryError, KeyboardInterrupt, etc..) can happen. + with mock.patch.object(aifc.Aifc_write, 'initfp', + side_effect=RuntimeError): + with self.assertRaises(RuntimeError): + self.fout = aifc.open(TESTFN, 'wb') + def test_params_added(self): f = self.f = aifc.open(TESTFN, 'wb') f.aiff() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967ccff3e7c..d67f9195eba04c 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ def test_frombytearray(self): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ def test_overflow(self): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ def test_overflow(self): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8c62408bd8a392..e68d0de2f8be1b 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -421,6 +421,16 @@ def test_empty_yield_from(self): compile(empty_yield_from, "", "exec") self.assertIn("field value is required", str(cm.exception)) + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + class ASTHelpers_Test(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 28d92a9f4e3eac..27781a2d91b336 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1,6 +1,7 @@ """Tests for events.py.""" import collections.abc +import concurrent.futures import functools import gc import io @@ -57,6 +58,15 @@ def osx_tiger(): return version < (10, 5) +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + ONLYCERT = data_file('ssl_cert.pem') ONLYKEY = data_file('ssl_key.pem') SIGNED_CERTFILE = data_file('keycert3.pem') @@ -248,8 +258,8 @@ def tearDown(self): if not self.loop.is_closed(): test_utils.run_briefly(self.loop) - self.loop.close() - gc.collect() + self.doCleanups() + support.gc_collect() super().tearDown() def test_run_until_complete_nesting(self): @@ -1970,19 +1980,26 @@ def test_subprocess_terminate(self): @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") def test_subprocess_send_signal(self): - prog = os.path.join(os.path.dirname(__file__), 'echo.py') - - connect = self.loop.subprocess_exec( - functools.partial(MySubprocessProtocol, self.loop), - sys.executable, prog) - transp, proto = self.loop.run_until_complete(connect) - self.assertIsInstance(proto, MySubprocessProtocol) - self.loop.run_until_complete(proto.connected) - - transp.send_signal(signal.SIGHUP) - self.loop.run_until_complete(proto.completed) - self.assertEqual(-signal.SIGHUP, proto.returncode) - transp.close() + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.send_signal(signal.SIGHUP) + self.loop.run_until_complete(proto.completed) + self.assertEqual(-signal.SIGHUP, proto.returncode) + transp.close() + finally: + signal.signal(signal.SIGHUP, old_handler) def test_subprocess_stderr(self): prog = os.path.join(os.path.dirname(__file__), 'echo2.py') @@ -2181,6 +2198,20 @@ def tearDown(self): asyncio.set_child_watcher(None) super().tearDown() + def test_get_event_loop_new_process(self): + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + result = await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result + + self.unpatch_get_running_loop() + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, SubprocessTestsMixin, diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 89afdcaff28665..4320a901f49cf6 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -1,6 +1,7 @@ """Tests for futures.py.""" import concurrent.futures +import gc import re import sys import threading @@ -19,9 +20,11 @@ def _fakefunc(f): return f + def first_cb(): pass + def last_cb(): pass @@ -97,8 +100,8 @@ def test_ensure_future(self): class BaseFutureTests: - def _new_future(self, loop=None): - raise NotImplementedError + def _new_future(self, *args, **kwargs): + return self.cls(*args, **kwargs) def setUp(self): super().setUp() @@ -144,6 +147,39 @@ def test_constructor_positional(self): # Make sure Future doesn't accept a positional argument self.assertRaises(TypeError, self._new_future, 42) + def test_uninitialized(self): + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.result) + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.exception) + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_result(None) + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_exception(Exception) + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.cancel() + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.add_done_callback(lambda f: None) + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.remove_done_callback(lambda f: None) + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut._schedule_callbacks() + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + repr(fut) + except AttributeError: + pass + fut = self.cls.__new__(self.cls, loop=self.loop) + fut.cancelled() + fut.done() + iter(fut) + def test_cancel(self): f = self._new_future(loop=self.loop) self.assertTrue(f.cancel()) @@ -318,6 +354,14 @@ def test_tb_logger_abandoned(self, m_log): del fut self.assertFalse(m_log.error.called) + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(Exception()) + fut.cancel() + del fut + self.assertFalse(m_log.error.called) + @mock.patch('asyncio.base_events.logger') def test_tb_logger_result_unretrieved(self, m_log): fut = self._new_future(loop=self.loop) @@ -369,6 +413,7 @@ def run(arg): self.assertTrue(asyncio.isfuture(f2)) self.assertEqual(res, 'oi') self.assertNotEqual(ident, threading.get_ident()) + ex.shutdown(wait=True) def test_wrap_future_future(self): f1 = self._new_future(loop=self.loop) @@ -384,6 +429,7 @@ def run(arg): f1 = ex.submit(run, 'oi') f2 = asyncio.wrap_future(f1) self.assertIs(self.loop, f2._loop) + ex.shutdown(wait=True) def test_wrap_future_cancel(self): f1 = concurrent.futures.Future() @@ -475,19 +521,24 @@ def test_future_iter_throw(self): Exception("elephant"), Exception("elephant")) self.assertRaises(TypeError, fi.throw, list) + def test_future_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + for i in range(100): + fut = self._new_future(loop=self.loop) + fut.set_result(Evil()) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') class CFutureTests(BaseFutureTests, test_utils.TestCase): - - def _new_future(self, *args, **kwargs): - return futures._CFuture(*args, **kwargs) + cls = getattr(futures, '_CFuture') class PyFutureTests(BaseFutureTests, test_utils.TestCase): - - def _new_future(self, *args, **kwargs): - return futures._PyFuture(*args, **kwargs) + cls = futures._PyFuture class BaseFutureDoneCallbackTests(): @@ -569,6 +620,57 @@ def test_remove_done_callback(self): self.assertEqual(bag, [2]) self.assertEqual(f.result(), 'foo') + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation_1(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + + def test_schedule_callbacks_list_mutation_2(self): + # see http://bugs.python.org/issue30828 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + max_extra_cbs = 100 + extra_cbs = 0 + + class evil: + def __eq__(self, other): + nonlocal extra_cbs + extra_cbs += 1 + if extra_cbs < max_extra_cbs: + fut.add_done_callback(id) + return False + + fut.remove_done_callback(evil()) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 152948c8138975..c85e8b1a32f73a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -176,6 +176,28 @@ def lockit(name, blocker): self.assertTrue(tb.cancelled()) self.assertTrue(tc.done()) + def test_finished_waiter_cancelled(self): + lock = asyncio.Lock(loop=self.loop) + + ta = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertTrue(lock.locked()) + + tb = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual(len(lock._waiters), 1) + + # Create a second waiter, wake up the first, and cancel it. + # Without the fix, the second was not woken up. + tc = asyncio.Task(lock.acquire(), loop=self.loop) + lock.release() + tb.cancel() + test_utils.run_briefly(self.loop) + + self.assertTrue(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + def test_release_not_acquired(self): lock = asyncio.Lock(loop=self.loop) diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 4dfc61259f8731..d76da661ab92da 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -529,7 +529,6 @@ def test_loop_self_reading_fut(self): self.loop._loop_self_reading) def test_loop_self_reading_exception(self): - self.loop.close = mock.Mock() self.loop.call_exception_handler = mock.Mock() self.proactor.recv.side_effect = OSError() self.loop._loop_self_reading() diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 6bf7862ecf9c0a..c50b3e49565c92 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,16 +24,14 @@ class TestBaseSelectorEventLoop(BaseSelectorEventLoop): - def close(self): - # Don't call the close() method of the parent class, because the - # selector is mocked - self._closed = True - def _make_self_pipe(self): self._ssock = mock.Mock() self._csock = mock.Mock() self._internal_fds += 1 + def _close_self_pipe(self): + pass + def list_to_buffer(l=()): return bytearray().join(l) diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6967e5b5..bcd236ea2632ed 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -42,6 +42,7 @@ def mock_handshake(callback): sslpipe.do_handshake.side_effect = mock_handshake with mock.patch('asyncio.sslproto._SSLPipe', return_value=sslpipe): ssl_proto.connection_made(transport) + return transport def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an @@ -95,5 +96,31 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_close_during_handshake(self): + # bpo-29743 Closing transport during handshake process leaks socket + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + + def do_handshake(callback): + return [] + + transport = self.connection_made(ssl_proto) + test_utils.run_briefly(self.loop) + + ssl_proto._app_transport.close() + self.assertTrue(transport.abort.called) + + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7..e8822c36698200 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -166,25 +166,32 @@ def test_terminate(self): @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") def test_send_signal(self): - code = 'import time; print("sleeping", flush=True); time.sleep(3600)' - args = [sys.executable, '-c', code] - create = asyncio.create_subprocess_exec(*args, - stdout=subprocess.PIPE, - loop=self.loop) - proc = self.loop.run_until_complete(create) + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + code = 'import time; print("sleeping", flush=True); time.sleep(3600)' + args = [sys.executable, '-c', code] + create = asyncio.create_subprocess_exec(*args, + stdout=subprocess.PIPE, + loop=self.loop) + proc = self.loop.run_until_complete(create) - @asyncio.coroutine - def send_signal(proc): - # basic synchronization to wait until the program is sleeping - line = yield from proc.stdout.readline() - self.assertEqual(line, b'sleeping\n') + @asyncio.coroutine + def send_signal(proc): + # basic synchronization to wait until the program is sleeping + line = yield from proc.stdout.readline() + self.assertEqual(line, b'sleeping\n') - proc.send_signal(signal.SIGHUP) - returncode = (yield from proc.wait()) - return returncode + proc.send_signal(signal.SIGHUP) + returncode = (yield from proc.wait()) + return returncode - returncode = self.loop.run_until_complete(send_signal(proc)) - self.assertEqual(-signal.SIGHUP, returncode) + returncode = self.loop.run_until_complete(send_signal(proc)) + self.assertEqual(-signal.SIGHUP, returncode) + finally: + signal.signal(signal.SIGHUP, old_handler) def prepare_broken_pipe_test(self): # buffer large enough to feed the whole pipe buffer @@ -459,6 +466,30 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a18d49ae374112..243faf6b049958 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3,6 +3,7 @@ import collections import contextlib import functools +import gc import io import os import re @@ -92,6 +93,20 @@ def setUp(self): self.loop.set_task_factory(self.new_task) self.loop.create_future = lambda: self.new_future(self.loop) + def test_task_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + @asyncio.coroutine + def run(): + return Evil() + + self.loop.run_until_complete( + asyncio.gather(*[ + self.new_task(self.loop, run()) for _ in range(100) + ], loop=self.loop)) + def test_other_loop_future(self): other_loop = asyncio.new_event_loop() fut = self.new_future(other_loop) @@ -588,6 +603,24 @@ def task(): self.assertFalse(t._must_cancel) # White-box test. self.assertFalse(t.cancel()) + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + def test_stop_while_run_in_complete(self): def gen(): @@ -1462,6 +1495,14 @@ def test_current_task(self): def coro(loop): self.assertTrue(Task.current_task(loop=loop) is task) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(loop) + try: + self.assertIs(Task.current_task(None), task) + self.assertIs(Task.current_task(), task) + finally: + asyncio.set_event_loop(None) + task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) self.assertIsNone(Task.current_task(loop=self.loop)) @@ -1806,8 +1847,17 @@ def kill_me(loop): # schedule the task coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) + self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + # See http://bugs.python.org/issue29271 for details: + asyncio.set_event_loop(self.loop) + try: + self.assertEqual(Task.all_tasks(), {task}) + self.assertEqual(Task.all_tasks(None), {task}) + finally: + asyncio.set_event_loop(None) + # execute the task so it waits for future self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) @@ -1830,6 +1880,25 @@ def kill_me(loop): }) mock_handler.reset_mock() + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def coro(): + raise TypeError + + @asyncio.coroutine + def runner(): + task = self.new_task(loop, coro()) + yield from asyncio.sleep(0.05, loop=loop) + task.cancel() + task = None + + loop.run_until_complete(runner()) + self.assertFalse(m_log.error.called) + @mock.patch('asyncio.coroutines.logger') def test_coroutine_never_yielded(self, m_log): with set_coroutine_debug(True): diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 1afcae107b0dbe..c72eef1afdb454 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -118,7 +118,9 @@ def test_wait_for_handle(self): self.assertEqual(done, False) self.assertFalse(fut.result()) - self.assertTrue(0.48 < elapsed < 0.9, elapsed) + # bpo-31008: Tolerate only 450 ms (at least 500 ms expected), + # because of bad clock resolution on Windows + self.assertTrue(0.45 <= elapsed <= 0.9, elapsed) _overlapped.SetEvent(event) diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 51c65737a441de..07edf2275bea0e 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -433,7 +433,10 @@ def test_close_twice(self): f = asyncore.file_wrapper(fd) os.close(fd) - f.close() + os.close(f.fd) # file_wrapper dupped fd + with self.assertRaises(OSError): + f.close() + self.assertEqual(f.fd, -1) # calling close twice should not fail f.close() @@ -502,7 +505,7 @@ def handle_connect(self): class BaseTestAPI: def tearDown(self): - asyncore.close_all() + asyncore.close_all(ignore_all=True) def loop_waiting_for_flag(self, instance, timeout=5): timeout = float(timeout) / 100 @@ -661,6 +664,9 @@ def test_handle_expt(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + class TestClient(BaseClient): def handle_expt(self): self.socket.recv(1024, socket.MSG_OOB) @@ -752,50 +758,50 @@ def test_bind(self): def test_set_reuse_addr(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") - sock = socket.socket(self.family) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - finally: - sock.close() + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads def test_quick_connect(self): # see: http://bugs.python.org/issue10340 - if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - server = BaseServer(self.family, self.addr) - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=500)) - t.start() - def cleanup(): - t.join(timeout=TIMEOUT) - if t.is_alive(): - self.fail("join() timed out") - self.addCleanup(cleanup) - - s = socket.socket(self.family, socket.SOCK_STREAM) - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - try: - s.connect(server.address) - except OSError: - pass - finally: - s.close() + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") class TestAPI_UseIPv4Sockets(BaseTestAPI): family = socket.AF_INET diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25419cea7..1d0b018aafaf93 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -23,6 +23,9 @@ def raise1(): def raise2(): raise SystemError +def exit(): + raise SystemExit + class GeneralTest(unittest.TestCase): @@ -76,6 +79,13 @@ def test_raise_unnormalized(self): self.assertRaises(ZeroDivisionError, atexit._run_exitfuncs) self.assertIn("ZeroDivisionError", self.stream.getvalue()) + def test_exit(self): + # be sure a SystemExit is handled properly + atexit.register(exit) + + self.assertRaises(SystemExit, atexit._run_exitfuncs) + self.assertEqual(self.stream.getvalue(), '') + def test_print_tracebacks(self): # Issue #18776: the tracebacks should be printed when errors occur. def f(): @@ -143,6 +153,7 @@ def test_bound_methods(self): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_audioop.py b/Lib/test/test_audioop.py index 8f34d72427e183..9baa62ad45c045 100644 --- a/Lib/test/test_audioop.py +++ b/Lib/test/test_audioop.py @@ -405,6 +405,10 @@ def test_ratecv(self): self.assertEqual(audioop.ratecv(datas[w], w, 1, 8000, 8000, None, 30, 10)[0], expected[w]) + self.assertRaises(TypeError, audioop.ratecv, b'', 1, 1, 8000, 8000, 42) + self.assertRaises(TypeError, audioop.ratecv, + b'', 1, 1, 8000, 8000, (1, (42,))) + def test_reverse(self): for w in 1, 2, 3, 4: self.assertEqual(audioop.reverse(b'', w), b'') diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 4f86aaa0c0eb49..47547396b8cb54 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,14 @@ def check_type_errors(self, f): int_data = memoryview(b"1234").cast('I') self.assertRaises(TypeError, f, int_data) + def test_encodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.encodestring(b"www.python.org") + + def test_decodestring_warns(self): + with self.assertWarns(DeprecationWarning): + base64.decodestring(b"d3d3LnB5dGhvbi5vcmc=\n") + def test_encodebytes(self): eq = self.assertEqual eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n") diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 416316c02852c8..49c4a53c052dc2 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -151,6 +151,8 @@ def test_import(self): self.assertRaises(TypeError, __import__, 1, 2, 3, 4) self.assertRaises(ValueError, __import__, '') self.assertRaises(TypeError, __import__, 'sys', name='sys') + # embedded null character + self.assertRaises(ModuleNotFoundError, __import__, 'string\x00') def test_abs(self): # int @@ -1002,6 +1004,10 @@ def test_open(self): self.assertEqual(fp.read(300), 'XXX'*100) self.assertEqual(fp.read(1000), 'YYY'*100) + # embedded null bytes and characters + self.assertRaises(ValueError, open, 'a\x00b') + self.assertRaises(ValueError, open, b'a\x00b') + def test_open_default_encoding(self): old_environ = dict(os.environ) try: @@ -1554,6 +1560,10 @@ def run_child(self, child, terminal_input): self.fail("got %d lines in pipe but expected 2, child output was:\n%s" % (len(lines), child_output)) os.close(fd) + + # Wait until the child process completes + os.waitpid(pid, 0) + return lines def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d39cae24..cd82fa64570cb5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ def test_mod(self): a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ def test_imod(self): b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index e2b8e0fd1233d9..2e8819be5dac68 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,4 +1,10 @@ +import datetime import unittest +from test.support import cpython_only +try: + import _testcapi +except ImportError: + _testcapi = None # The test cases here cover several paths through the function calling # code. They depend on the METH_XXX flag that is used to define a C @@ -122,5 +128,175 @@ def test_oldargs1_2_kw(self): self.assertRaises(TypeError, [].count, x=2, y=2) +def pyfunc(arg1, arg2): + return [arg1, arg2] + + +def pyfunc_noarg(): + return "noarg" + + +class PythonClass: + def method(self, arg1, arg2): + return [arg1, arg2] + + def method_noarg(self): + return "noarg" + + @classmethod + def class_method(cls): + return "classmethod" + + @staticmethod + def static_method(): + return "staticmethod" + + +PYTHON_INSTANCE = PythonClass() + + +IGNORE_RESULT = object() + + +@cpython_only +class FastCallTests(unittest.TestCase): + # Test calls with positional arguments + CALLS_POSARGS = ( + # (func, args: tuple, result) + + # Python function with 2 arguments + (pyfunc, (1, 2), [1, 2]), + + # Python function without argument + (pyfunc_noarg, (), "noarg"), + + # Python class methods + (PythonClass.class_method, (), "classmethod"), + (PythonClass.static_method, (), "staticmethod"), + + # Python instance methods + (PYTHON_INSTANCE.method, (1, 2), [1, 2]), + (PYTHON_INSTANCE.method_noarg, (), "noarg"), + (PYTHON_INSTANCE.class_method, (), "classmethod"), + (PYTHON_INSTANCE.static_method, (), "staticmethod"), + + # C function: METH_NOARGS + (globals, (), IGNORE_RESULT), + + # C function: METH_O + (id, ("hello",), IGNORE_RESULT), + + # C function: METH_VARARGS + (dir, (1,), IGNORE_RESULT), + + # C function: METH_VARARGS | METH_KEYWORDS + (min, (5, 9), 5), + + # C function: METH_FASTCALL + (divmod, (1000, 33), (30, 10)), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00', 'little'), 1), + + # bpo-30524: Test that calling a C type static method with no argument + # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS + (datetime.datetime.now, (), IGNORE_RESULT), + ) + + # Test calls with positional and keyword arguments + CALLS_KWARGS = ( + # (func, args: tuple, kwargs: dict, result) + + # Python function with 2 arguments + (pyfunc, (1,), {'arg2': 2}, [1, 2]), + (pyfunc, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # Python instance methods + (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), + (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # C function: METH_VARARGS | METH_KEYWORDS + (max, ([],), {'default': 9}, 9), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), + (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), + ) + + def check_result(self, result, expected): + if expected is IGNORE_RESULT: + return + self.assertEqual(result, expected) + + def test_fastcall(self): + # Test _PyObject_FastCall() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + result = _testcapi.pyobject_fastcall(func, args) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0 + result = _testcapi.pyobject_fastcall(func, None) + self.check_result(result, expected) + + def test_fastcall_dict(self): + # Test _PyObject_FastCallDict() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, args, None) + self.check_result(result, expected) + + # kwargs={} + result = _testcapi.pyobject_fastcalldict(func, args, {}) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0, kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, None, None) + self.check_result(result, expected) + + # args=NULL, nargs=0, kwargs={} + result = _testcapi.pyobject_fastcalldict(func, None, {}) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + result = _testcapi.pyobject_fastcalldict(func, args, kwargs) + self.check_result(result, expected) + + def test_fastcall_keywords(self): + # Test _PyObject_FastCallKeywords() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, args, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, args, ()) + self.check_result(result, expected) + + if not args: + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, None, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, None, ()) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + kwnames = tuple(kwargs.keys()) + args = args + tuple(kwargs.values()) + result = _testcapi.pyobject_fastcallkeywords(func, args, kwnames) + self.check_result(result, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c3625d4879138..216851c2d36c8b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -98,7 +98,7 @@ class Z(object): def __len__(self): return 1 self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -106,7 +106,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -116,7 +116,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -490,9 +490,8 @@ def test_skipitem(self): # test the format unit when not skipped format = c + "i" try: - # (note: the format string must be bytes!) _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, - format.encode("ascii"), keywords) + format, keywords) when_not_skipped = False except SystemError as e: s = "argument 1 (impossible)" @@ -504,7 +503,7 @@ def test_skipitem(self): optional_format = "|" + format try: _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, - optional_format.encode("ascii"), keywords) + optional_format, keywords) when_skipped = False except SystemError as e: s = "impossible: '{}'".format(format) @@ -517,40 +516,64 @@ def test_skipitem(self): self.assertIs(when_skipped, when_not_skipped, message) def test_parse_tuple_and_keywords(self): - # parse_tuple_and_keywords error handling tests + # Test handling errors in the parse_tuple_and_keywords helper itself self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, (), {}, 42, []) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', 42) + (), {}, '', 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [''] * 42) + (), {}, '', [''] * 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [42]) + (), {}, '', [42]) + + def test_bad_use(self): + # Test handling invalid format and keywords in + # PyArg_ParseTupleAndKeywords() + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '||O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1, 2), {}, '|O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$$O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$|O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|OO', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|$O', ['']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|OO', ['a', '']) def test_positional_only(self): parse = _testcapi.parse_tuple_and_keywords - parse((1, 2, 3), {}, b'OOO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OOO', ['', '', 'a']) + parse((1, 2, 3), {}, 'OOO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OOO', ['', '', 'a']) - parse((1,), {}, b'O|OO', ['', '', 'a']) + parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) + parse((1,), {}, 'O|OO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|OO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OO$O', ['', '', 'a']) + parse((), {}, 'O|OO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes exactly 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OO$O', ['', '', 'a']) - parse((1,), {}, b'O|O$O', ['', '', 'a']) + parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) + parse((1,), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|O$O', ['', '', 'a']) + parse((), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): - parse((1,), {}, b'O|$OO', ['', '', 'a']) + parse((1,), {}, 'O|$OO', ['', '', 'a']) with self.assertRaisesRegex(SystemError, 'Empty keyword'): - parse((1,), {}, b'O|OO', ['', 'a', '']) + parse((1,), {}, 'O|OO', ['', 'a', '']) @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4d554a397b4a51..ecc01f277954d5 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -568,5 +568,32 @@ class B(A): a = A(hash(A.f)^(-1)) hash(a.f) + def testSetattrWrapperNameIntern(self): + # Issue #25794: __setattr__ should intern the attribute name + class A: + pass + + def add(self, other): + return 'summa' + + name = str(b'__add__', 'ascii') # shouldn't be optimized + self.assertIsNot(name, '__add__') # not interned + type.__setattr__(A, name, add) + self.assertEqual(A() + 1, 'summa') + + name2 = str(b'__add__', 'ascii') + self.assertIsNot(name2, '__add__') + self.assertIsNot(name2, name) + type.__delattr__(A, name2) + with self.assertRaises(TypeError): + A() + 1 + + def testSetattrNonStringName(self): + class A: + pass + + with self.assertRaises(TypeError): + type.__setattr__(A, b'x', None) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e058ecd086df3c..1587daf8f582dd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -572,6 +572,73 @@ def test_syntaxerror_indented_caret_position(self): self.assertNotIn("\f", text) self.assertIn("\n 1 + 1 = 2\n ^", text) + def test_consistent_sys_path_for_direct_execution(self): + # This test case ensures that the following all give the same + # sys.path configuration: + # + # ./python -s script_dir/__main__.py + # ./python -s script_dir + # ./python -I script_dir + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir, support.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from directly executing __main__.py + # We omit PYTHONPATH and user site to align with isolated mode + p = spawn_python("-Es", script_name, cwd=work_dir) + out_by_name = kill_python(p).decode().splitlines() + self.assertEqual(out_by_name[0], script_dir) + self.assertNotIn(work_dir, out_by_name) + # Directory execution should give the same output + p = spawn_python("-Es", script_dir, cwd=work_dir) + out_by_dir = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir, out_by_name) + # As should directory execution in isolated mode + p = spawn_python("-I", script_dir, cwd=work_dir) + out_by_dir_isolated = kill_python(p).decode().splitlines() + self.assertEqual(out_by_dir_isolated, out_by_dir, out_by_name) + + def test_consistent_sys_path_for_module_execution(self): + # This test case ensures that the following both give the same + # sys.path configuration: + # ./python -sm script_pkg.__main__ + # ./python -sm script_pkg + # + # And that this fails as unable to find the package: + # ./python -Im script_pkg + script = textwrap.dedent("""\ + import sys + for entry in sys.path: + print(entry) + """) + # Always show full path diffs on errors + self.maxDiff = None + with support.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + script_name = _make_test_script(script_dir, '__main__', script) + # Reference output comes from `-m script_pkg.__main__` + # We omit PYTHONPATH and user site to better align with the + # direct execution test cases + p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir) + out_by_module = kill_python(p).decode().splitlines() + self.assertEqual(out_by_module[0], '') + self.assertNotIn(script_dir, out_by_module) + # Package execution should give the same output + p = spawn_python("-sm", "script_pkg", cwd=work_dir) + out_by_package = kill_python(p).decode().splitlines() + self.assertEqual(out_by_package, out_by_module) + # Isolated mode should fail with an import error + exitcode, stdout, stderr = assert_python_failure( + "-Im", "script_pkg", cwd=work_dir + ) + traceback_lines = stderr.decode().splitlines() + self.assertIn("No module named script_pkg", traceback_lines[-1]) def test_main(): support.run_unittest(CmdLineTest) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7975ea0ef5e17d..68537482ac91f3 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -103,9 +103,15 @@ """ import sys +import threading import unittest import weakref -from test.support import run_doctest, run_unittest, cpython_only +try: + import ctypes +except ImportError: + ctypes = None +from test.support import (run_doctest, run_unittest, cpython_only, + check_impl_detail) def consts(t): @@ -212,11 +218,105 @@ def callback(code): self.assertTrue(self.called) +if check_impl_detail(cpython=True) and ctypes is not None: + py = ctypes.pythonapi + freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) + + RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = py._PyCode_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = py._PyCode_GetExtra + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp)) + GetExtra.restype = ctypes.c_int + + LAST_FREED = None + def myfree(ptr): + global LAST_FREED + LAST_FREED = ptr + + FREE_FUNC = freefunc(myfree) + FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + + class CoExtra(unittest.TestCase): + def get_func(self): + # Defining a function causes the containing function to have a + # reference to the code object. We need the code objects to go + # away, so we eval a lambda. + return eval('lambda:42') + + def test_get_non_code(self): + f = self.get_func() + + self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + + def test_bad_index(self): + f = self.get_func() + self.assertRaises(SystemError, SetExtra, f.__code__, + FREE_INDEX+100, ctypes.c_voidp(100)) + self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, + ctypes.c_voidp(100)), 0) + + def test_free_called(self): + # Verify that the provided free function gets invoked + # when the code object is cleaned up. + f = self.get_func() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) + del f + self.assertEqual(LAST_FREED, 100) + + def test_get_set(self): + # Test basic get/set round tripping. + f = self.get_func() + + extra = ctypes.c_voidp() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) + # reset should free... + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) + self.assertEqual(LAST_FREED, 200) + + extra = ctypes.c_voidp() + GetExtra(f.__code__, FREE_INDEX, extra) + self.assertEqual(extra.value, 300) + del f + + def test_free_different_thread(self): + # Freeing a code object on a different thread then + # where the co_extra was set should be safe. + f = self.get_func() + class ThreadTest(threading.Thread): + def __init__(self, f, test): + super().__init__() + self.f = f + self.test = test + def run(self): + del self.f + self.test.assertEqual(LAST_FREED, 500) + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) + tt = ThreadTest(f, self) + del f + tt.start() + tt.join() + self.assertEqual(LAST_FREED, 500) + def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) - run_unittest(CodeTest, CodeConstsTest, CodeWeakRefTest) - + tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] + if check_impl_detail(cpython=True) and ctypes is not None: + tests.append(CoExtra) + run_unittest(*tests) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 3bdf7d0e14b3b7..2a450718d5aae5 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -48,6 +48,12 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), ) has_iso10646 = True @@ -80,6 +86,10 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode ) if __name__ == "__main__": diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc6704cb0..47f756213ddf00 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ def test_issue26915(self): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index c249ca724bf0ce..cee49343e268a2 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -387,6 +387,29 @@ def __complex__(self): self.assertAlmostEqual(complex(complex1(1j)), 2j) self.assertRaises(TypeError, complex, complex2(1j)) + @support.requires_IEEE_754 + def test_constructor_special_numbers(self): + class complex2(complex): + pass + for x in 0.0, -0.0, INF, -INF, NAN: + for y in 0.0, -0.0, INF, -INF, NAN: + with self.subTest(x=x, y=y): + z = complex(x, y) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(x, y) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex(complex2(x, y)) + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + z = complex2(complex(x, y)) + self.assertIs(type(z), complex2) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + def test_underscores(self): # check underscores for lit in VALID_UNDERSCORE_LITERALS: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 23e95b212447c8..5ddce09a08bc47 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -11,6 +11,7 @@ from test.support.script_helper import assert_python_ok +import itertools import os import sys import threading @@ -59,10 +60,25 @@ def my_method(self): pass +def make_dummy_object(_): + return MyObject() + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._thread_key = test.support.threading_setup() + + def tearDown(self): + test.support.reap_children() + test.support.threading_cleanup(*self._thread_key) + + class ExecutorMixin: worker_count = 5 def setUp(self): + super().setUp() + self.t1 = time.time() try: self.executor = self.executor_type(max_workers=self.worker_count) @@ -72,11 +88,15 @@ def setUp(self): def tearDown(self): self.executor.shutdown(wait=True) + self.executor = None + dt = time.time() - self.t1 if test.support.verbose: print("%.2fs" % dt, end=' ') self.assertLess(dt, 60, "synchronization issue: test lasted too long") + super().tearDown() + def _prime_executor(self): # Make sure that the executor is ready to do work before running the # tests. This should reduce the probability of timeouts in the tests. @@ -123,7 +143,7 @@ def test_hang_issue12364(self): f.result() -class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, unittest.TestCase): +class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase): def _prime_executor(self): pass @@ -172,14 +192,13 @@ def test_thread_names_default(self): del executor for t in threads: - # We don't particularly care what the default name is, just that - # it has a default name implying that it is a ThreadPoolExecutor - # followed by what looks like a thread number. - self.assertRegex(t.name, r'^.*ThreadPoolExecutor.*_[0-4]$') + # Ensure that our default name is reasonably sane and unique when + # no thread_name_prefix was supplied. + self.assertRegex(t.name, r'ThreadPoolExecutor-\d+_[0-4]$') t.join() -class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest, unittest.TestCase): +class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest, BaseTestCase): def _prime_executor(self): pass @@ -208,11 +227,14 @@ def test_del_shutdown(self): list(executor.map(abs, range(-5, 5))) queue_management_thread = executor._queue_management_thread processes = executor._processes + call_queue = executor._call_queue del executor queue_management_thread.join() for p in processes.values(): p.join() + call_queue.close() + call_queue.join_thread() class WaitTests: @@ -316,7 +338,7 @@ def test_timeout(self): self.assertEqual(set([future2]), pending) -class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, unittest.TestCase): +class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): def test_pending_calls_race(self): # Issue #14406: multi-threaded race condition when waiting on all @@ -334,7 +356,7 @@ def future_func(): sys.setswitchinterval(oldswitchinterval) -class ProcessPoolWaitTests(ProcessPoolMixin, WaitTests, unittest.TestCase): +class ProcessPoolWaitTests(ProcessPoolMixin, WaitTests, BaseTestCase): pass @@ -378,16 +400,51 @@ def test_zero_timeout(self): def test_duplicate_futures(self): # Issue 20367. Duplicate futures should not raise exceptions or give # duplicate responses. + # Issue #31641: accept arbitrary iterables. future1 = self.executor.submit(time.sleep, 2) - completed = [f for f in futures.as_completed([future1,future1])] + completed = [ + f for f in futures.as_completed(itertools.repeat(future1, 3)) + ] self.assertEqual(len(completed), 1) + def test_free_reference_yielded_future(self): + # Issue #14406: Generator should not keep references + # to finished futures. + futures_list = [Future() for _ in range(8)] + futures_list.append(create_future(state=CANCELLED_AND_NOTIFIED)) + futures_list.append(create_future(state=FINISHED, result=42)) -class ThreadPoolAsCompletedTests(ThreadPoolMixin, AsCompletedTests, unittest.TestCase): + with self.assertRaises(futures.TimeoutError): + for future in futures.as_completed(futures_list, timeout=0): + futures_list.remove(future) + wr = weakref.ref(future) + del future + self.assertIsNone(wr()) + + futures_list[0].set_result("test") + for future in futures.as_completed(futures_list): + futures_list.remove(future) + wr = weakref.ref(future) + del future + self.assertIsNone(wr()) + if futures_list: + futures_list[0].set_result("test") + + def test_correct_timeout_exception_msg(self): + futures_list = [CANCELLED_AND_NOTIFIED_FUTURE, PENDING_FUTURE, + RUNNING_FUTURE, SUCCESSFUL_FUTURE] + + with self.assertRaises(futures.TimeoutError) as cm: + list(futures.as_completed(futures_list, timeout=0)) + + self.assertEqual(str(cm.exception), '2 (of 4) futures unfinished') + + +class ThreadPoolAsCompletedTests(ThreadPoolMixin, AsCompletedTests, BaseTestCase): pass -class ProcessPoolAsCompletedTests(ProcessPoolMixin, AsCompletedTests, unittest.TestCase): +class ProcessPoolAsCompletedTests(ProcessPoolMixin, AsCompletedTests, BaseTestCase): pass @@ -407,6 +464,10 @@ def test_map(self): list(self.executor.map(pow, range(10), range(10))), list(map(pow, range(10), range(10)))) + self.assertEqual( + list(self.executor.map(pow, range(10), range(10), chunksize=3)), + list(map(pow, range(10), range(10)))) + def test_map_exception(self): i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) self.assertEqual(i.__next__(), (0, 1)) @@ -457,8 +518,16 @@ def test_max_workers_negative(self): "than 0"): self.executor_type(max_workers=number) + def test_free_reference(self): + # Issue #14406: Result iterator should not keep an internal + # reference to result objects. + for obj in self.executor.map(make_dummy_object, range(10)): + wr = weakref.ref(obj) + del obj + self.assertIsNone(wr()) + -class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, unittest.TestCase): +class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase): def test_map_submits_without_iteration(self): """Tests verifying issue 11777.""" finished = [] @@ -475,7 +544,7 @@ def test_default_workers(self): (os.cpu_count() or 1) * 5) -class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest, unittest.TestCase): +class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest, BaseTestCase): def test_killed_child(self): # When a child process is abruptly terminated, the whole pool gets # "broken". @@ -531,7 +600,7 @@ def test_traceback(self): f1.getvalue()) -class FutureTests(unittest.TestCase): +class FutureTests(BaseTestCase): def test_done_callback_with_result(self): callback_result = None def fn(callback_future): @@ -711,6 +780,7 @@ def notification(): t.start() self.assertEqual(f1.result(timeout=5), 42) + t.join() def test_result_with_cancel(self): # TODO(brian@sweetapp.com): This test is timing dependent. @@ -724,6 +794,7 @@ def notification(): t.start() self.assertRaises(futures.CancelledError, f1.result, timeout=5) + t.join() def test_exception_with_timeout(self): self.assertRaises(futures.TimeoutError, @@ -752,6 +823,7 @@ def notification(): t.start() self.assertTrue(isinstance(f1.exception(timeout=5), OSError)) + t.join() @test.support.reap_threads def test_main(): diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 0d06080da33c52..72c3f19fb41f56 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,7 @@ import configparser import io import os -import sys +import pathlib import textwrap import unittest import warnings @@ -721,6 +721,16 @@ def test_read_returns_file_list(self): parsed_files = cf.read(file1) self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we pass only a Path object: + cf = self.newconfig() + parsed_files = cf.read(pathlib.Path(file1)) + self.assertEqual(parsed_files, [file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") + # check when we passed both a filename and a Path object: + cf = self.newconfig() + parsed_files = cf.read([pathlib.Path(file1), file1]) + self.assertEqual(parsed_files, [file1, file1]) + self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() parsed_files = cf.read(["nonexistent-file"]) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804af57047..2301f759d80858 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -44,6 +44,16 @@ def __exit__(self, *args): self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + class NoEnter(ManagerFromScratch): + __enter__ = None + + self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + + class NoExit(ManagerFromScratch): + __exit__ = None + + self.assertFalse(issubclass(NoExit, AbstractContextManager)) + class ContextManagerTestCase(unittest.TestCase): @@ -152,6 +162,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Lib/test/test_copyreg.py b/Lib/test/test_copyreg.py index 52e887cb36c41e..e3f1cd81aab205 100644 --- a/Lib/test/test_copyreg.py +++ b/Lib/test/test_copyreg.py @@ -16,6 +16,12 @@ class WithWeakref(object): class WithPrivate(object): __slots__ = ('__spam',) +class _WithLeadingUnderscoreAndPrivate(object): + __slots__ = ('__spam',) + +class ___(object): + __slots__ = ('__spam',) + class WithSingleString(object): __slots__ = 'spam' @@ -104,6 +110,10 @@ def test_slotnames(self): self.assertEqual(copyreg._slotnames(WithWeakref), []) expected = ['_WithPrivate__spam'] self.assertEqual(copyreg._slotnames(WithPrivate), expected) + expected = ['_WithLeadingUnderscoreAndPrivate__spam'] + self.assertEqual(copyreg._slotnames(_WithLeadingUnderscoreAndPrivate), + expected) + self.assertEqual(copyreg._slotnames(___), ['__spam']) self.assertEqual(copyreg._slotnames(WithSingleString), ['spam']) expected = ['eggs', 'spam'] expected.sort() diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 78439a2acae2d8..2b79a17ea703f5 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ async def waiter(coro): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): @@ -1680,6 +1695,44 @@ async def foo(): warnings.simplefilter("error") run_async(foo()) + def test_for_11(self): + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __anext__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + + def test_for_12(self): + class F: + def __aiter__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + with self.assertRaisesRegex(TypeError, + 'an invalid object from __aiter__') as c: + main().send(None) + + err = c.exception + self.assertIsInstance(err.__cause__, ZeroDivisionError) + def test_for_tuple(self): class Done(Exception): pass @@ -2064,6 +2117,7 @@ def wrap(gen): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 3d8c50bcb6c6ae..0d0b160d7645a5 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -81,7 +81,7 @@ def test_window_funcs(self): win2 = curses.newwin(15,15, 5,5) for meth in [stdscr.addch, stdscr.addstr]: - for args in [('a'), ('a', curses.A_BOLD), + for args in [('a',), ('a', curses.A_BOLD), (4,4, 'a'), (5,5, 'a', curses.A_BOLD)]: with self.subTest(meth=meth.__qualname__, args=args): meth(*args) @@ -194,6 +194,15 @@ def test_window_funcs(self): self.assertRaises(ValueError, stdscr.instr, -2) self.assertRaises(ValueError, stdscr.instr, 2, 3, -2) + def test_embedded_null_chars(self): + # reject embedded null bytes and characters + stdscr = self.stdscr + for arg in ['a', b'a']: + with self.subTest(arg=arg): + self.assertRaises(ValueError, stdscr.addstr, 'a\0') + self.assertRaises(ValueError, stdscr.addnstr, 'a\0', 1) + self.assertRaises(ValueError, stdscr.insstr, 'a\0') + self.assertRaises(ValueError, stdscr.insnstr, 'a\0', 1) def test_module_funcs(self): "Test module-level functions" diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 242e1bba035bf9..d659f369d54e46 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -20,7 +20,7 @@ # XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might # not believe this, but in spite of all the sys.modules trickery running a _Pure # test last will leave a mix of pure and native datetime stuff lying around. -test_classes = [] +all_test_classes = [] for module, suffix in zip(test_modules, test_suffixes): test_classes = [] @@ -32,8 +32,10 @@ elif issubclass(cls, unittest.TestSuite): suit = cls() test_classes.extend(type(test) for test in suit) + test_classes = sorted(set(test_classes), key=lambda cls: cls.__qualname__) for cls in test_classes: - cls.__name__ = name + suffix + cls.__name__ += suffix + cls.__qualname__ += suffix @classmethod def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() @@ -46,9 +48,10 @@ def tearDownClass(cls_): sys.modules.update(cls_._save_sys_modules) cls.setUpClass = setUpClass cls.tearDownClass = tearDownClass + all_test_classes.extend(test_classes) def test_main(): - run_unittest(*test_classes) + run_unittest(*all_test_classes) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 617a37eec8223a..57094ad6695262 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -1614,6 +1614,9 @@ def test_threading(self): for sig in Signals[self.decimal]: self.assertFalse(DefaultContext.flags[sig]) + th1.join() + th2.join() + DefaultContext.prec = save_prec DefaultContext.Emax = save_emax DefaultContext.Emin = save_emin diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 832bb9c8e2dbcc..8013f37c88da0a 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1085,6 +1085,91 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in issue 27945 part 3. + class X(): + def __del__(self): + dict_b.clear() + + def __eq__(self, other): + dict_a.clear() + return True + + def __hash__(self): + return 13 + + dict_a = {X(): 0} + dict_b = {X(): X()} + self.assertTrue(dict_a == dict_b) + + def test_fromkeys_operator_modifying_dict_operand(self): + # test fix for seg fault reported in issue 27945 part 4a. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1): 1, X(2): 2} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_fromkeys_operator_modifying_set_operand(self): + # test fix for seg fault reported in issue 27945 part 4b. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1), X(2)} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_dictitems_contains_use_after_free(self): + class X: + def __eq__(self, other): + d.clear() + return NotImplemented + + d = {0: set()} + (0, X()) in d.items() + + def test_init_use_after_free(self): + class X: + def __hash__(self): + pair[:] = [] + return 13 + + pair = [X(), 123] + dict([pair]) + + def test_oob_indexing_dictiter_iternextitem(self): + class X(int): + def __del__(self): + d.clear() + + d = {i: X(i) for i in range(8)} + + def iter_and_mutate(): + for result in d.items(): + if result[0] == 2: + d[2] = None # free d[2] --> X(2).__del__ was called + + self.assertRaises(RuntimeError, iter_and_mutate) + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 26ece6911f2377..e0ec87d2080185 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2711,5 +2711,17 @@ def test_whitespace_splitting(self): self._test(parser.get_unstructured('xxx ' + 'y'*77), 'xxx \n ' + 'y'*77 + '\n') + def test_long_filename_attachment(self): + folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"') + self.assertEqual( + 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"\n', + folded + ) + folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"') + self.assertEqual( + 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"\n', + folded + ) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 7c8877fdcb090e..c4f182903afefe 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -162,6 +162,13 @@ def test_set_mangle_from_via_policy(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_compat32_max_line_length_does_not_fold_when_none(self): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=policy.compat32.clone(max_line_length=None)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 9b04c18b0e2968..89966893092851 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -20,6 +20,7 @@ class EnsurepipMixin: def setUp(self): run_pip_patch = unittest.mock.patch("ensurepip._run_pip") self.run_pip = run_pip_patch.start() + self.run_pip.return_value = 0 self.addCleanup(run_pip_patch.stop) # Avoid side effects on the actual os module @@ -255,7 +256,7 @@ def test_bootstrap_version(self): self.assertFalse(self.run_pip.called) def test_basic_bootstrapping(self): - ensurepip._main([]) + exit_code = ensurepip._main([]) self.run_pip.assert_called_once_with( [ @@ -267,6 +268,13 @@ def test_basic_bootstrapping(self): additional_paths = self.run_pip.call_args[0][1] self.assertEqual(len(additional_paths), 2) + self.assertEqual(exit_code, 0) + + def test_bootstrapping_error_code(self): + self.run_pip.return_value = 2 + exit_code = ensurepip._main([]) + self.assertEqual(exit_code, 2) + class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase): @@ -280,7 +288,7 @@ def test_uninstall_version(self): def test_basic_uninstall(self): with fake_pip(): - ensurepip._uninstall._main([]) + exit_code = ensurepip._uninstall._main([]) self.run_pip.assert_called_once_with( [ @@ -289,6 +297,13 @@ def test_basic_uninstall(self): ] ) + self.assertEqual(exit_code, 0) + + def test_uninstall_error_code(self): + with fake_pip(): + self.run_pip.return_value = 2 + exit_code = ensurepip._uninstall._main([]) + self.assertEqual(exit_code, 2) if __name__ == "__main__": diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 13a89fc2db6390..eaea8ee3abdfc9 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2291,6 +2291,26 @@ def test_programatic_function_from_dict(self): self.assertIs(type(e), Perm) + def test_programatic_function_from_empty_list(self): + Perm = enum.IntFlag('Perm', []) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 0, Perm) + Thing = enum.Enum('Thing', []) + lst = list(Thing) + self.assertEqual(len(lst), len(Thing)) + self.assertEqual(len(Thing), 0, Thing) + + + def test_programatic_function_from_empty_tuple(self): + Perm = enum.IntFlag('Perm', ()) + lst = list(Perm) + self.assertEqual(len(lst), len(Perm)) + self.assertEqual(len(Perm), 0, Perm) + Thing = enum.Enum('Thing', ()) + self.assertEqual(len(lst), len(Thing)) + self.assertEqual(len(Thing), 0, Thing) + def test_containment(self): Perm = self.Perm R, W, X = Perm diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 48379222c37659..53851cb6cfda66 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -155,6 +156,34 @@ def ckmsg(src, msg): ckmsg(s, "'continue' not properly in loop") ckmsg("continue\n", "'continue' not properly in loop") + def testSyntaxErrorMissingParens(self): + def ckmsg(src, msg, exception=SyntaxError): + try: + compile(src, '', 'exec') + except exception as e: + if e.msg != msg: + self.fail("expected %s, got %s" % (msg, e.msg)) + else: + self.fail("failed to get expected SyntaxError") + + s = '''print "old style"''' + ckmsg(s, "Missing parentheses in call to 'print'. " + "Did you mean print(\"old style\")?") + + s = '''print "old style",''' + ckmsg(s, "Missing parentheses in call to 'print'. " + "Did you mean print(\"old style\", end=\" \")?") + + s = '''exec "old style"''' + ckmsg(s, "Missing parentheses in call to 'exec'") + + # should not apply to subclasses, see issue #31161 + s = '''if True:\nprint "No indent"''' + ckmsg(s, "expected an indented block", IndentationError) + + s = '''if True:\n print()\n\texec "mixed tabs and spaces"''' + ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) + def testSyntaxErrorOffset(self): def check(src, lineno, offset): with self.assertRaises(SyntaxError) as cm: @@ -1119,6 +1148,25 @@ def test_non_str_argument(self): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 043df013112517..2c1848337b2d7f 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -163,6 +163,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *(broken() for i in range(1))) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenIterable1: ... def __iter__(self): @@ -172,6 +176,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenIterable1()) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenIterable2: ... def __iter__(self): @@ -182,6 +190,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenIterable2()) + Traceback (most recent call last): + ... + TypeError: myerror >>> class BrokenSequence: ... def __getitem__(self, idx): @@ -191,6 +203,10 @@ Traceback (most recent call last): ... TypeError: myerror + >>> g(*range(1), *BrokenSequence()) + Traceback (most recent call last): + ... + TypeError: myerror Make sure that the function doesn't stomp the dictionary diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index bdd8d1a2a6163f..20fe1e2655946f 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,46 @@ def test_raise_exception(self): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_nonfatal_exception(self): + # These exceptions are not strictly errors. Letting + # faulthandler display the traceback when they are + # raised is likely to result in noise. However, they + # may still terminate the process if there is no + # handler installed for them (which there typically + # is, e.g. for debug messages). + for exc in ( + 0x00000000, + 0x34567890, + 0x40000000, + 0x40001000, + 0x70000000, + 0x7FFFFFFF, + ): + output, exitcode = self.get_output(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(0x{exc:x}) + """ + ) + self.assertEqual(output, []) + # On Windows older than 7 SP1, the actual exception code has + # bit 29 cleared. + self.assertIn(exitcode, + (exc, exc & ~0x10000000)) + + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_disable_windows_exc_handler(self): + code = dedent(""" + import faulthandler + faulthandler.enable() + faulthandler.disable() + code = faulthandler._EXCEPTION_ACCESS_VIOLATION + faulthandler._raise_exception(code) + """) + output, exitcode = self.get_output(code) + self.assertEqual(output, []) + self.assertEqual(exitcode, 0xC0000005) if __name__ == "__main__": diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210ae1997ad..57a02656206ff4 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ def testReprNoCloseFD(self): finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473db503b76..c5ca50c8f71206 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ def test_float_memoryview(self): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): @@ -601,6 +613,12 @@ def test_float_specials_do_unpack(self): (', 2000. @@ -604,7 +718,7 @@ def test_main(): # Here's the second example po file example, used to generate the UMO_DATA # containing utf-8 encoded Unicode strings -''' +b''' # Dummy translation for the Python test_gettext.py module. # Copyright (C) 2001 Python Software Foundation # Barry Warsaw , 2000. @@ -627,7 +741,7 @@ def test_main(): # Here's the third example po file, used to generate MMO_DATA -''' +b''' msgid "" msgstr "" "Project-Id-Version: No Project 0.0\n" @@ -646,7 +760,7 @@ def test_main(): # messages.po, used for bug 17898 # -''' +b''' # test file for http://bugs.python.org/issue17898 msgid "" msgstr "" diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 69095a3fb98d74..e511947858c0a3 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -50,6 +50,8 @@ def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrall, 42) + # embedded null character + self.assertRaises(ValueError, grp.getgrnam, 'a\x00b') # try to get some errors bynames = {} diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index f748b461907f6f..647719ea62ce1a 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -750,28 +750,28 @@ def test_threaded_hashing(self): hasher = hashlib.sha1() num_threads = 5 smallest_data = b'swineflu' - data = smallest_data*200000 + data = smallest_data * 200000 expected_hash = hashlib.sha1(data*num_threads).hexdigest() - def hash_in_chunks(chunk_size, event): + def hash_in_chunks(chunk_size): index = 0 while index < len(data): - hasher.update(data[index:index+chunk_size]) + hasher.update(data[index:index + chunk_size]) index += chunk_size - event.set() - events = [] + threads = [] for threadnum in range(num_threads): - chunk_size = len(data) // (10**threadnum) + chunk_size = len(data) // (10 ** threadnum) self.assertGreater(chunk_size, 0) self.assertEqual(chunk_size % len(smallest_data), 0) - event = threading.Event() - events.append(event) - threading.Thread(target=hash_in_chunks, - args=(chunk_size, event)).start() - - for event in events: - event.wait() + thread = threading.Thread(target=hash_in_chunks, + args=(chunk_size,)) + threads.append(thread) + + for thread in threads: + thread.start() + for thread in threads: + thread.join() self.assertEqual(expected_hash, hasher.hexdigest()) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 4e931446b9ef84..22b3bf5235c417 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -52,6 +52,7 @@ def run(self): def stop(self): self.server.shutdown() + self.join() class BaseTestCase(unittest.TestCase): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index fac6b57081283b..93ba61a1a99e5d 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -13,7 +13,7 @@ import inspect from test.support import (reap_threads, verbose, transient_internet, - run_with_tz, run_with_locale) + run_with_tz, run_with_locale, cpython_only) import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -504,6 +504,15 @@ def test_ssl_verified(self): ssl_context=ssl_context) client.shutdown() + # Mock the private method _connect(), so mark the test as specific + # to CPython stdlib + @cpython_only + def test_certfile_arg_warn(self): + with support.check_warnings(('', DeprecationWarning)): + with mock.patch.object(self.imap_class, 'open'): + with mock.patch.object(self.imap_class, '_connect'): + self.imap_class('localhost', 143, certfile=CERTFILE) + class ThreadedNetworkedTests(unittest.TestCase): server_class = socketserver.TCPServer imap_class = imaplib.IMAP4 @@ -966,19 +975,6 @@ def test_logincapa(self): _server = self.imap_class(self.host, self.port) self.check_logincapa(_server) - def test_logincapa_with_client_certfile(self): - with transient_internet(self.host): - with support.check_warnings(('', DeprecationWarning)): - _server = self.imap_class(self.host, self.port, - certfile=CERTFILE) - self.check_logincapa(_server) - - def test_logincapa_with_client_ssl_context(self): - with transient_internet(self.host): - _server = self.imap_class( - self.host, self.port, ssl_context=self.create_ssl_context()) - self.check_logincapa(_server) - def test_logout(self): with transient_internet(self.host): _server = self.imap_class(self.host, self.port) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 4ece3654108e49..d513eedc7bb4aa 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -314,6 +314,21 @@ def test_multiple_calls_to_get_data(self): loader.get_data(imp.__file__) # File should be closed loader.get_data(imp.__file__) # Will need to create a newly opened file + def test_load_source(self): + with self.assertRaisesRegex(ValueError, 'embedded null'): + imp.load_source(__name__, __file__ + "\0") + + @support.cpython_only + def test_issue31315(self): + # There shouldn't be an assertion failure in imp.create_dynamic(), + # when spec.name is not a string. + create_dynamic = support.get_attribute(imp, 'create_dynamic') + class BadSpec: + name = None + origin = 'foo' + with self.assertRaises(TypeError): + create_dynamic(BadSpec()) + class ReloadTests(unittest.TestCase): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 760908efe671fe..b73a96f757005e 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -10,6 +10,8 @@ import random import stat import sys +import threading +import time import unittest import unittest.mock as mock import textwrap @@ -22,8 +24,9 @@ EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, unlink, unload, create_empty_file, cpython_only, TESTFN_UNENCODABLE, - temp_dir) + temp_dir, DirsOnSysPath) from test.support import script_helper +from test.test_importlib.util import uncache skip_if_dont_write_bytecode = unittest.skipIf( @@ -350,6 +353,44 @@ def __getattr__(self, _): with self.assertRaises(ImportError): from test_from_import_AttributeError import does_not_exist + @cpython_only + def test_issue31492(self): + # There shouldn't be an assertion failure in case of failing to import + # from a module with a bad __name__ attribute, or in case of failing + # to access an attribute of such a module. + with swap_attr(os, '__name__', None): + with self.assertRaises(ImportError): + from os import does_not_exist + + with self.assertRaises(AttributeError): + os.does_not_exist + + def test_concurrency(self): + sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'data')) + try: + exc = None + def run(): + event.wait() + try: + import package + except BaseException as e: + nonlocal exc + exc = e + + for i in range(10): + event = threading.Event() + threads = [threading.Thread(target=run) for x in range(2)] + try: + with test.support.start_threads(threads, event.set): + time.sleep(0) + finally: + sys.modules.pop('package', None) + sys.modules.pop('package.submodule', None) + if exc is not None: + raise exc + finally: + del sys.path[0] + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -567,7 +608,7 @@ def test_UNC_path(self): try: os.listdir(unc) except OSError as e: - if e.errno in (errno.EPERM, errno.EACCES): + if e.errno in (errno.EPERM, errno.EACCES, errno.ENOENT): # See issue #15338 self.skipTest("cannot access administrative share %r" % (unc,)) raise @@ -612,11 +653,11 @@ def check_relative(): # Check relative import fails with only __package__ wrong ns = dict(__package__='foo', __name__='test.notarealmodule') - self.assertRaises(SystemError, check_relative) + self.assertRaises(ModuleNotFoundError, check_relative) # Check relative import fails with __package__ and __name__ wrong ns = dict(__package__='foo', __name__='notarealpkg.notarealmodule') - self.assertRaises(SystemError, check_relative) + self.assertRaises(ModuleNotFoundError, check_relative) # Check relative import fails with package set to a non-string ns = dict(__package__=object()) @@ -631,6 +672,20 @@ def test_absolute_import_without_future(self): self.fail("explicit relative import triggered an " "implicit absolute import") + def test_import_from_non_package(self): + path = os.path.join(os.path.dirname(__file__), 'data', 'package2') + with uncache('submodule1', 'submodule2'), DirsOnSysPath(path): + with self.assertRaises(ImportError): + import submodule1 + self.assertNotIn('submodule1', sys.modules) + self.assertNotIn('submodule2', sys.modules) + + def test_import_from_unloaded_package(self): + with uncache('package2', 'package2.submodule1', 'package2.submodule2'), \ + DirsOnSysPath(os.path.join(os.path.dirname(__file__), 'data')): + import package2.submodule1 + package2.submodule1.submodule2 + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): diff --git a/Lib/test/test_import/data/package/__init__.py b/Lib/test/test_import/data/package/__init__.py new file mode 100644 index 00000000000000..a4f2bc3400ce5e --- /dev/null +++ b/Lib/test/test_import/data/package/__init__.py @@ -0,0 +1,2 @@ +import package.submodule +package.submodule diff --git a/Lib/test/test_import/data/package/submodule.py b/Lib/test/test_import/data/package/submodule.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_import/data/package2/submodule1.py b/Lib/test/test_import/data/package2/submodule1.py new file mode 100644 index 00000000000000..0698ed6de2a1db --- /dev/null +++ b/Lib/test/test_import/data/package2/submodule1.py @@ -0,0 +1,3 @@ +import sys +sys.modules.pop(__package__, None) +from . import submodule2 diff --git a/Lib/test/test_import/data/package2/submodule2.py b/Lib/test/test_import/data/package2/submodule2.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 7f64548748eec3..b26634ea4537a1 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -81,7 +81,7 @@ def test_warn_when_package_and_spec_disagree(self): def test_bad__package__(self): globals = {'__package__': ''} - with self.assertRaises(SystemError): + with self.assertRaises(ModuleNotFoundError): self.__import__('', globals, {}, ['relimport'], 1) def test_bunk__package__(self): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index d615375b342091..ac18e5c34b87fa 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -4,6 +4,7 @@ machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') +import importlib.util import os import pathlib import string @@ -757,5 +758,48 @@ def test_source_from_cache_path_like_arg(self): ) = util.test_both(PEP3147Tests, util=importlib_util) +class MagicNumberTests(unittest.TestCase): + """ + Test release compatibility issues relating to importlib + """ + @unittest.skipUnless( + sys.version_info.releaselevel in ('final', 'release'), + 'only applies to candidate or final python release levels' + ) + def test_magic_number(self): + """ + Each python minor release should generally have a MAGIC_NUMBER + that does not change once the release reaches candidate status. + + Once a release reaches candidate status, the value of the constant + EXPECTED_MAGIC_NUMBER in this test should be changed. + This test will then check that the actual MAGIC_NUMBER matches + the expected value for the release. + + In exceptional cases, it may be required to change the MAGIC_NUMBER + for a maintenance release. In this case the change should be + discussed in python-dev. If a change is required, community + stakeholders such as OS package maintainers must be notified + in advance. Such exceptional releases will then require an + adjustment to this test case. + """ + EXPECTED_MAGIC_NUMBER = 3379 + actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') + + msg = ( + "To avoid breaking backwards compatibility with cached bytecode " + "files that can't be automatically regenerated by the current " + "user, candidate and final releases require the current " + "importlib.util.MAGIC_NUMBER to match the expected " + "magic number in this test. Set the expected " + "magic number in this test to the current MAGIC_NUMBER to " + "continue with the release.\n\n" + "Changing the MAGIC_NUMBER for a maintenance release " + "requires discussion in python-dev and notification of " + "community stakeholders." + ) + self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 88eaabe6765ab2..b194b27aba3bc0 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -232,6 +232,30 @@ def foo(self): self.assertFalse(inspect.isabstract(int)) self.assertFalse(inspect.isabstract(5)) + def test_isabstract_during_init_subclass(self): + from abc import ABCMeta, abstractmethod + isabstract_checks = [] + class AbstractChecker(metaclass=ABCMeta): + def __init_subclass__(cls): + isabstract_checks.append(inspect.isabstract(cls)) + class AbstractClassExample(AbstractChecker): + @abstractmethod + def foo(self): + pass + class ClassExample(AbstractClassExample): + def foo(self): + pass + self.assertEqual(isabstract_checks, [True, False]) + + isabstract_checks.clear() + class AbstractChild(AbstractClassExample): + pass + class AbstractGrandchild(AbstractChild): + pass + class ConcreteGrandchild(ClassExample): + pass + self.assertEqual(isabstract_checks, [True, True, False]) + class TestInterpreterStack(IsTestBase): def __init__(self, *args, **kwargs): @@ -387,6 +411,11 @@ def test_cleandoc(self): def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) def test_getmodule(self): # Check actual module @@ -1960,6 +1989,41 @@ def test(a, b:'foo'=10, *args:'bar', spam:'baz', ham=123, **kwargs:int): ('kwargs', ..., int, "var_keyword")), ...)) + def test_signature_without_self(self): + def test_args_only(*args): # NOQA + pass + + def test_args_kwargs_only(*args, **kwargs): # NOQA + pass + + class A: + @classmethod + def test_classmethod(*args): # NOQA + pass + + @staticmethod + def test_staticmethod(*args): # NOQA + pass + + f1 = functools.partialmethod((test_classmethod), 1) + f2 = functools.partialmethod((test_args_only), 1) + f3 = functools.partialmethod((test_staticmethod), 1) + f4 = functools.partialmethod((test_args_kwargs_only),1) + + self.assertEqual(self.signature(test_args_only), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(test_args_kwargs_only), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + self.assertEqual(self.signature(A.f1), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f2), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f3), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f4), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -3493,6 +3557,19 @@ def test_builtins_have_signatures(self): self.assertIsNone(obj.__text_signature__) +class NTimesUnwrappable: + def __init__(self, n): + self.n = n + self._next = None + + @property + def __wrapped__(self): + if self.n <= 0: + raise Exception("Unwrapped too many times") + if self._next is None: + self._next = NTimesUnwrappable(self.n - 1) + return self._next + class TestUnwrap(unittest.TestCase): def test_unwrap_one(self): @@ -3548,6 +3625,11 @@ class C: __wrapped__ = func self.assertIsNone(inspect.unwrap(C())) + def test_recursion_limit(self): + obj = NTimesUnwrappable(sys.getrecursionlimit() + 1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(obj) + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 14bbd6192a0572..a36076e01047d6 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -506,5 +506,13 @@ def check(s, base=None): check('123\ud800') check('123\ud800', 10) + def test_issue31619(self): + self.assertEqual(int('1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1', 2), + 0b1010101010101010101010101010101) + self.assertEqual(int('1_2_3_4_5_6_7_0_1_2_3', 8), 0o12345670123) + self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) + self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index aaa64eadffe1ea..08c041ed937410 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -543,6 +543,22 @@ def test_readline(self): with self.open(support.TESTFN, "r") as f: self.assertRaises(TypeError, f.readline, 5.3) + def test_readline_nonsizeable(self): + # Issue #30061 + # Crash when readline() returns an object without __len__ + class R(self.IOBase): + def readline(self): + return None + self.assertRaises((TypeError, StopIteration), next, R()) + + def test_next_nonsizeable(self): + # Issue #30061 + # Crash when __next__() returns an object without __len__ + class R(self.IOBase): + def __next__(self): + return None + self.assertRaises(TypeError, R().readlines, 1) + def test_raw_bytes_io(self): f = self.BytesIO() self.write_ops(f) @@ -1014,6 +1030,16 @@ def test_repr(self): raw.name = b"dummy" self.assertEqual(repr(b), "<%s name=b'dummy'>" % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2450,16 @@ def test_repr(self): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) @@ -3127,6 +3163,14 @@ def test_read_nonbytes(self): t = self.TextIOWrapper(self.StringIO('a')) self.assertRaises(TypeError, t.read) + def test_illegal_encoder(self): + # Issue 31271: Calling write() while the return value of encoder's + # encode() is invalid shouldn't cause an assertion failure. + rot13 = codecs.lookup("rot13") + with support.swap_attr(rot13, '_is_text_encoding', True): + t = io.TextIOWrapper(io.BytesIO(b'foo'), encoding="rot13") + self.assertRaises(TypeError, t.write, 'bar') + def test_illegal_decoder(self): # Issue #17106 # Bypass the early encoding check added in issue 20404 @@ -3147,6 +3191,26 @@ def _make_illegal_wrapper(): t = _make_illegal_wrapper() self.assertRaises(TypeError, t.read) + # Issue 31243: calling read() while the return value of decoder's + # getstate() is invalid should neither crash the interpreter nor + # raise a SystemError. + def _make_very_illegal_wrapper(getstate_ret_val): + class BadDecoder: + def getstate(self): + return getstate_ret_val + def _get_bad_decoder(dummy): + return BadDecoder() + quopri = codecs.lookup("quopri") + with support.swap_attr(quopri, 'incrementaldecoder', + _get_bad_decoder): + return _make_illegal_wrapper() + t = _make_very_illegal_wrapper(42) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper(()) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper((1, 2)) + self.assertRaises(TypeError, t.read, 42) + def _check_create_at_shutdown(self, **kwargs): # Issue #20037: creating a TextIOWrapper at shutdown # shouldn't crash the interpreter. @@ -3374,6 +3438,7 @@ def test_newline_decoder(self): decoder = codecs.getincrementaldecoder("utf-8")() decoder = self.IncrementalNewlineDecoder(decoder, translate=True) self.check_newline_decoding_utf8(decoder) + self.assertRaises(TypeError, decoder.setstate, 42) def test_newline_bytes(self): # Issue 5433: Excessive optimization in IncrementalNewlineDecoder @@ -3478,6 +3543,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.readinto1, bytearray(1024)) self.assertRaises(ValueError, f.readline) self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) self.assertRaises(ValueError, f.seek, 0) self.assertRaises(ValueError, f.tell) self.assertRaises(ValueError, f.truncate) @@ -3683,6 +3749,7 @@ def check_daemon_threads_shutdown_deadlock(self, stream_name): import sys import time import threading + from test.support import SuppressCrashReport file = sys.{stream_name} @@ -3691,6 +3758,10 @@ def run(): file.write('.') file.flush() + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + thread = threading.Thread(target=run) thread.daemon = True thread.start() diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 0e39516dc033e3..5d9633024f7064 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1405,14 +1405,35 @@ def testAddressComparison(self): ipaddress.ip_address('::2')) def testInterfaceComparison(self): - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.1')) - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.2')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::1')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::2')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') == + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') < + ipaddress.ip_interface('1.1.1.2/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.1/16')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/24') > + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.2/16')) + + self.assertTrue(ipaddress.ip_interface('::1/64') == + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::1/80')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::2/64')) + self.assertTrue(ipaddress.ip_interface('::2/48') < + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/80') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::2/64') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') > + ipaddress.ip_interface('::2/48')) def testNetworkComparison(self): # ip1 and ip2 have the same network address diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57caade12b..a978134e88e754 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,38 @@ def gen2(x): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + + def test_issue30347_1(self): + def f(n): + if n == 5: + list(b) + return n != 6 + for (k, b) in groupby(range(10), f): + list(b) # shouldn't crash + + def test_issue30347_2(self): + class K: + def __init__(self, v): + pass + def __eq__(self, other): + nonlocal i + i += 1 + if i == 1: + next(g, None) + return True + i = 0 + g = next(groupby(range(10), K))[1] + for j in range(2): + next(g, None) # shouldn't crash + + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 109a2466c21bdb..5dad6920870421 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,6 +1,11 @@ from test.test_json import CTest +class BadBool: + def __bool__(self): + 1/0 + + class TestSpeedups(CTest): def test_scanstring(self): self.assertEqual(self.json.decoder.scanstring.__module__, "_json") @@ -17,8 +22,50 @@ class TestDecode(CTest): def test_make_scanner(self): self.assertRaises(AttributeError, self.json.scanner.c_make_scanner, 1) + def test_bad_bool_args(self): + def test(value): + self.json.decoder.JSONDecoder(strict=BadBool()).decode(value) + self.assertRaises(ZeroDivisionError, test, '""') + self.assertRaises(ZeroDivisionError, test, '{}') + + +class TestEncode(CTest): def test_make_encoder(self): self.assertRaises(TypeError, self.json.encoder.c_make_encoder, (True, False), b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) + + def test_bad_str_encoder(self): + # Issue #31505: There shouldn't be an assertion failure in case + # c_make_encoder() receives a bad encoder() argument. + def bad_encoder1(*args): + return None + enc = self.json.encoder.c_make_encoder(None, lambda obj: str(obj), + bad_encoder1, None, ': ', ', ', + False, False, False) + with self.assertRaises(TypeError): + enc('spam', 4) + with self.assertRaises(TypeError): + enc({'spam': 42}, 4) + + def bad_encoder2(*args): + 1/0 + enc = self.json.encoder.c_make_encoder(None, lambda obj: str(obj), + bad_encoder2, None, ': ', ', ', + False, False, False) + with self.assertRaises(ZeroDivisionError): + enc('spam', 4) + + def test_bad_bool_args(self): + def test(name): + self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) + self.assertRaises(ZeroDivisionError, test, 'skipkeys') + self.assertRaises(ZeroDivisionError, test, 'ensure_ascii') + self.assertRaises(ZeroDivisionError, test, 'check_circular') + self.assertRaises(ZeroDivisionError, test, 'allow_nan') + self.assertRaises(ZeroDivisionError, test, 'sort_keys') + + def test_unsortable_keys(self): + with self.assertRaises(TypeError): + self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f373664e1278..9d93f931ca3958 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1f8265f077e662..ddb169fe58957c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -139,7 +139,7 @@ def test_main(verbose=None): import gc counts = [None] * 5 for i in range(len(counts)): - support.run_doctest(test_genexps, verbose) + support.run_doctest(test_listcomps, verbose) gc.collect() counts[i] = sys.gettotalrefcount() print(counts) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 99fab58cb9326d..650d7373d8e1df 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -339,9 +339,14 @@ def test_strcoll(self): self.assertLess(locale.strcoll('a', 'b'), 0) self.assertEqual(locale.strcoll('a', 'a'), 0) self.assertGreater(locale.strcoll('b', 'a'), 0) + # embedded null character + self.assertRaises(ValueError, locale.strcoll, 'a\0', 'a') + self.assertRaises(ValueError, locale.strcoll, 'a', 'a\0') def test_strxfrm(self): self.assertLess(locale.strxfrm('a'), locale.strxfrm('b')) + # embedded null character + self.assertRaises(ValueError, locale.strxfrm, 'a\0') class TestEnUSCollation(BaseLocalizedTest, TestCollation): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9dedc098ba5aca..25046c3c38ab3e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -770,6 +770,7 @@ def stop(self, timeout=None): """ self.close() self._thread.join(timeout) + asyncore.close_all(map=self._map, ignore_all=True) self._thread = None class ControlMixin(object): @@ -1663,7 +1664,7 @@ def setUp(self): server.ready.wait() hcls = logging.handlers.SysLogHandler if isinstance(server.server_address, tuple): - self.sl_hdlr = hcls(('localhost', server.port)) + self.sl_hdlr = hcls((server.server_address[0], server.port)) else: self.sl_hdlr = hcls(server.server_address) self.log_output = '' @@ -1723,6 +1724,24 @@ def tearDown(self): SysLogHandlerTest.tearDown(self) support.unlink(self.address) +@unittest.skipUnless(support.IPV6_ENABLED, + 'IPv6 support required for this test.') +@unittest.skipUnless(threading, 'Threading required for this test.') +class IPv6SysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with IPv6 host.""" + + server_class = TestUDPServer + address = ('::1', 0) + + def setUp(self): + self.server_class.address_family = socket.AF_INET6 + super(IPv6SysLogHandlerTest, self).setUp() + + def tearDown(self): + self.server_class.address_family = socket.AF_INET + super(IPv6SysLogHandlerTest, self).tearDown() + @unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -3163,6 +3182,7 @@ def setup_and_log(log_queue, ident): handler.close() @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_queue_queue(self, mock_handle): for i in range(self.repeat): log_queue = queue.Queue() @@ -3172,10 +3192,13 @@ def test_handle_called_with_queue_queue(self, mock_handle): @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_mp_queue(self, mock_handle): for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + log_queue.close() + log_queue.join_thread() self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') @@ -3188,6 +3211,7 @@ def get_all_from_queue(log_queue): return [] @support.requires_multiprocessing_queue + @support.reap_threads def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3200,6 +3224,9 @@ def test_no_messages_in_queue_after_stop(self): self.setup_and_log(queue, '%s_%s' %(self.id(), i)) # time.sleep(1) items = list(self.get_all_from_queue(queue)) + queue.close() + queue.join_thread() + expected = [[], [logging.handlers.QueueListener._sentinel]] self.assertIn(items, expected, 'Found unexpected messages in queue: %s' % ( @@ -3928,6 +3955,17 @@ def test_has_handlers(self): self.assertFalse(self.logger.hasHandlers()) self.assertFalse(self.adapter.hasHandlers()) + def test_nested(self): + msg = 'Adapters can be nested, yo.' + adapter_adapter = logging.LoggerAdapter(logger=self.adapter, extra=None) + adapter_adapter.log(logging.CRITICAL, msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.CRITICAL) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + class LoggerTest(BaseTest): @@ -4370,7 +4408,7 @@ def test_main(): QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, BasicConfigTest, LoggerAdapterTest, LoggerTest, SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, LastResortTest, LogRecordTest, - ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, + ExceptionTest, SysLogHandlerTest, IPv6SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, MiscTestCase diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 2ba944335ae87d..3807b95b158263 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -746,7 +746,7 @@ def test_create_tmp(self, repetitions=10): hostname = hostname.replace(':', r'\072') pid = os.getpid() pattern = re.compile(r"(?P

    \d+)" - r"Q(?P\d+)\.(?P[^:/]+)") + r"Q(?P\d+)\.(?P[^:/]*)") previous_groups = None for x in range(repetitions): tmp_file = self._box._create_tmp() diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index c46ded11866589..3e84f3429ee854 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -286,7 +286,12 @@ class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase): def setUpClass(cls): support.requires("network") with support.transient_internet(cls.NNTP_HOST): - cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) + try: + cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) + except EOFError: + raise unittest.SkipTest(f"{cls} got EOF error on connecting " + f"to {cls.NNTP_HOST!r}") @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 5b590e1af34495..30424564450227 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -36,15 +36,23 @@ def unistr(data): class NormalizationTest(unittest.TestCase): def test_main(self): - part = None - part1_data = {} # Hit the exception early try: testdata = open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version) + except PermissionError: + self.skipTest(f"Permission error when downloading {TESTDATAURL} " + f"into the test data directory") except (OSError, HTTPException): - self.skipTest("Could not retrieve " + TESTDATAURL) - self.addCleanup(testdata.close) + self.fail(f"Could not retrieve {TESTDATAURL}") + + with testdata: + self.run_normalization_tests(testdata) + + def run_normalization_tests(self, testdata): + part = None + part1_data = {} + for line in testdata: if '#' in line: line = line.split('#')[0] diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e9fdb0719f4bf4..b65ccb71591da5 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -473,7 +473,9 @@ def test_access_denied(self): # force CreateFile to fail with ERROR_ACCESS_DENIED. DETACHED_PROCESS = 8 subprocess.check_call( - ['icacls.exe', fname, '/deny', 'Users:(S)'], + # bpo-30584: Use security identifier *S-1-5-32-545 instead + # of localized "Users" to not depend on the locale. + ['icacls.exe', fname, '/deny', '*S-1-5-32-545:(S)'], creationflags=DETACHED_PROCESS ) result = os.stat(fname) @@ -620,8 +622,13 @@ def _test_utime_current(self, set_time): if not self.support_subsecond(self.fname): delta = 1.0 + elif os.name == 'nt': + # On Windows, the usual resolution of time.time() is 15.6 ms. + # bpo-30649: Tolerate 50 ms for slow Windows buildbots. + delta = 0.050 else: - # On Windows, the usual resolution of time.time() is 15.6 ms + # bpo-30649: PPC64 Fedora 3.x buildbot requires + # at least a delta of 14 ms delta = 0.020 st = os.stat(self.fname) msg = ("st_time=%r, current=%r, dt=%r" @@ -829,6 +836,30 @@ def test_key_type(self): self.assertIs(cm.exception.args[0], missing) self.assertTrue(cm.exception.__suppress_context__) + def _test_environ_iteration(self, collection): + iterator = iter(collection) + new_key = "__new_key__" + + next(iterator) # start iteration over os.environ.items + + # add a new key in os.environ mapping + os.environ[new_key] = "test_environ_iteration" + + try: + next(iterator) # force iteration over modified mapping + self.assertEqual(os.environ[new_key], "test_environ_iteration") + finally: + del os.environ[new_key] + + def test_iter_error_when_changing_os_environ(self): + self._test_environ_iteration(os.environ) + + def test_iter_error_when_changing_os_environ_items(self): + self._test_environ_iteration(os.environ.items()) + + def test_iter_error_when_changing_os_environ_values(self): + self._test_environ_iteration(os.environ.values()) + class WalkTests(unittest.TestCase): """Tests for os.walk().""" @@ -1546,6 +1577,27 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) + def test_execve_invalid_env(self): + args = [sys.executable, '-c', 'pass'] + + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): @@ -2357,6 +2409,61 @@ def test_spawnve_noargs(self): self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + def _test_invalid_env(self, spawn): + args = [sys.executable, '-c', 'pass'] + + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the enviroment variable value + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + fp.write('import sys, os\n' + 'if os.getenv("FRUIT") != "orange=lemon":\n' + ' raise AssertionError') + args = [sys.executable, filename] + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + self.assertEqual(exitcode, 0) + + @requires_os_func('spawnve') + def test_spawnve_invalid_env(self): + self._test_invalid_env(os.spawnve) + + @requires_os_func('spawnvpe') + def test_spawnvpe_invalid_env(self): + self._test_invalid_env(os.spawnvpe) + + # The introduction of this TestCase caused at least two different errors on # *nix buildbots. Temporarily skip this to let the buildbots move along. @unittest.skip("Skip due to platform/environment differences on *NIX buildbots") @@ -2514,6 +2621,7 @@ def tearDown(self): self.client.close() if self.server.running: self.server.stop() + self.server = None def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): """A higher level wrapper representing how an application is @@ -3295,6 +3403,22 @@ def test_bytes(self): self.assertEqual(entry.path, os.fsencode(os.path.join(self.path, 'file.txt'))) + def test_bytes_like(self): + self.create_file("file.txt") + + for cls in bytearray, memoryview: + path_bytes = cls(os.fsencode(self.path)) + with self.assertWarns(DeprecationWarning): + entries = list(os.scandir(path_bytes)) + self.assertEqual(len(entries), 1, entries) + entry = entries[0] + + self.assertEqual(entry.name, b'file.txt') + self.assertEqual(entry.path, + os.fsencode(os.path.join(self.path, 'file.txt'))) + self.assertIs(type(entry.name), bytes) + self.assertIs(type(entry.path), bytes) + def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index d6e6f7157782ec..70cabb28598218 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,4 +1,6 @@ +import copy import parser +import pickle import unittest import operator import struct @@ -424,6 +426,52 @@ def test_junk(self): # not even remotely valid: self.check_bad_tree((1, 2, 3), "") + def test_illegal_terminal(self): + tree = \ + (257, + (269, + (270, + (271, + (277, + (1,))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too small items in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, b'pass'))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-string second item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', '0', 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-integer third item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', 0, 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too many items in terminal node") + def test_illegal_yield_1(self): # Illegal yield statement: def f(): return 1; yield 1 tree = \ @@ -628,6 +676,24 @@ def test_missing_import_source(self): (4, ''), (0, '')) self.check_bad_tree(tree, "from import fred") + def test_illegal_encoding(self): + # Illegal encoding declaration + tree = \ + (339, + (257, (0, ''))) + self.check_bad_tree(tree, "missed encoding") + tree = \ + (339, + (257, (0, '')), + b'iso-8859-1') + self.check_bad_tree(tree, "non-string encoding") + tree = \ + (339, + (257, (0, '')), + '\udcff') + with self.assertRaises(UnicodeEncodeError): + parser.sequence2st(tree) + class CompileTestCase(unittest.TestCase): @@ -772,6 +838,21 @@ def test_comparisons(self): self.assertRaises(TypeError, operator.lt, st1, 1815) self.assertRaises(TypeError, operator.gt, b'waterloo', st2) + def test_copy_pickle(self): + sts = [ + parser.expr('2 + 3'), + parser.suite('x = 2; y = x + 3'), + parser.expr('list(x**3 for x in range(20))') + ] + for st in sts: + st_copy = copy.copy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + st_copy = copy.deepcopy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + st_copy = pickle.loads(pickle.dumps(st, proto)) + self.assertEqual(st_copy.totuple(), st.totuple()) + check_sizeof = support.check_sizeof @support.cpython_only diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5802c1cb..6502413a73d061 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1506,10 +1507,10 @@ def test_resolve_common(self): os.path.join(BASE, 'foo')) p = P(BASE, 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) + os.path.join(BASE, 'foo', 'in', 'spam')) p = P(BASE, '..', 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo'))) + os.path.abspath(os.path.join('foo', 'in', 'spam'))) # These are all relative symlinks p = P(BASE, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -1521,16 +1522,18 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) # Now create absolute symlinks d = tempfile.mkdtemp(suffix='-dirD') self.addCleanup(support.rmtree, d) @@ -1540,16 +1543,17 @@ def test_resolve_common(self): self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + False) p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo'), False) + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) @with_symlinks def test_resolve_dot(self): @@ -1563,7 +1567,7 @@ def test_resolve_dot(self): r = q / '3' / '4' self.assertRaises(FileNotFoundError, r.resolve, strict=True) # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3') + self.assertEqual(r.resolve(strict=False), p / '3' / '4') def test_with(self): p = self.cls(BASE) @@ -1776,6 +1780,11 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': @@ -1811,6 +1820,35 @@ def test_mkdir_no_parents_file(self): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5f875eff6c7ede..ea6752b61d8f77 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -67,12 +67,12 @@ def test_processor(self): def setUp(self): self.save_version = sys.version - self.save_mercurial = sys._mercurial + self.save_git = sys._git self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._mercurial = self.save_mercurial + sys._git = self.save_git sys.platform = self.save_platform def test_sys_version(self): @@ -102,7 +102,7 @@ def test_sys_version(self): ('CPython', '2.4.3', '', '', 'truncation', '', 'GCC')), ): # branch and revision are not "parsed", but fetched - # from sys._mercurial. Ignore them + # from sys._git. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -149,10 +149,10 @@ def test_sys_version(self): sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_mercurial"): - del sys._mercurial + if hasattr(sys, "_git"): + del sys._git else: - sys._mercurial = subversion + sys._git = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index e5b16dc98a4111..1269199423c0ce 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -254,6 +254,8 @@ def setUp(self): def tearDown(self): self.client.close() self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None def test_getwelcome(self): self.assertEqual(self.client.getwelcome(), @@ -436,6 +438,8 @@ def tearDown(self): # this exception self.client.close() self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None def test_stls(self): self.assertRaises(poplib.error_proto, self.client.stls) @@ -461,7 +465,8 @@ def setUp(self): def tearDown(self): self.thread.join() - del self.thread # Clear out any dangling Thread objects. + # Explicitly clear the attribute to prevent dangling thread + self.thread = None def server(self, evt, serv): serv.listen() @@ -483,7 +488,7 @@ def testTimeoutDefault(self): finally: socket.setdefaulttimeout(None) self.assertEqual(pop.sock.gettimeout(), 30) - pop.sock.close() + pop.close() def testTimeoutNone(self): self.assertIsNone(socket.getdefaulttimeout()) @@ -493,12 +498,12 @@ def testTimeoutNone(self): finally: socket.setdefaulttimeout(None) self.assertIsNone(pop.sock.gettimeout()) - pop.sock.close() + pop.close() def testTimeoutValue(self): pop = poplib.POP3(HOST, self.port, timeout=30) self.assertEqual(pop.sock.gettimeout(), 30) - pop.sock.close() + pop.close() def test_main(): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 029d0815e94589..9df3206324fc67 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -478,6 +478,10 @@ def test_makedev(self): self.assertRaises(TypeError, posix.minor) self.assertRaises((ValueError, OverflowError), posix.minor, -1) + if sys.platform.startswith('freebsd') and dev >= 0x1_0000_0000: + self.skipTest("bpo-31044: on FreeBSD CURRENT, minor() truncates " + "64-bit dev to 32-bit") + self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) @@ -581,17 +585,25 @@ def test_chdir(self): self.assertRaises(OSError, posix.chdir, support.TESTFN) def test_listdir(self): - self.assertTrue(support.TESTFN in posix.listdir(os.curdir)) + self.assertIn(support.TESTFN, posix.listdir(os.curdir)) def test_listdir_default(self): # When listdir is called without argument, # it's the same as listdir(os.curdir). - self.assertTrue(support.TESTFN in posix.listdir()) + self.assertIn(support.TESTFN, posix.listdir()) def test_listdir_bytes(self): # When listdir is called with a bytes object, # the returned strings are of type bytes. - self.assertTrue(os.fsencode(support.TESTFN) in posix.listdir(b'.')) + self.assertIn(os.fsencode(support.TESTFN), posix.listdir(b'.')) + + def test_listdir_bytes_like(self): + for cls in bytearray, memoryview: + with self.assertWarns(DeprecationWarning): + names = posix.listdir(cls(b'.')) + self.assertIn(os.fsencode(support.TESTFN), names) + for name in names: + self.assertIs(type(name), bytes) @unittest.skipUnless(posix.listdir in os.supports_fd, "test needs fd support for posix.listdir()") @@ -751,6 +763,21 @@ def test_environ(self): self.assertEqual(type(k), item_type) self.assertEqual(type(v), item_type) + @unittest.skipUnless(hasattr(os, "putenv"), "requires os.putenv()") + def test_putenv(self): + with self.assertRaises(ValueError): + os.putenv('FRUIT\0VEGETABLE', 'cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') + with self.assertRaises(ValueError): + os.putenv('FRUIT', 'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv('FRUIT=ORANGE', 'lemon') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT=ORANGE', b'lemon') + @unittest.skipUnless(hasattr(posix, 'getcwd'), 'test needs posix.getcwd()') def test_getcwd_long_pathnames(self): dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef' diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 7eea349115a947..e6434feaf5eec6 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -1,4 +1,5 @@ import unittest +import sys from io import StringIO from test import support @@ -128,5 +129,65 @@ def flush(self): raise RuntimeError self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True) + +class TestPy2MigrationHint(unittest.TestCase): + """Test that correct hint is produced analogous to Python3 syntax, + if print statement is executed as in Python 2. + """ + + def test_normal_string(self): + python2_print_str = 'print "Hello World"' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World")', str(context.exception)) + + def test_string_with_soft_space(self): + python2_print_str = 'print "Hello World",' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World", end=" ")', str(context.exception)) + + def test_string_with_excessive_whitespace(self): + python2_print_str = 'print "Hello World", ' + with self.assertRaises(SyntaxError) as context: + exec(python2_print_str) + + self.assertIn('print("Hello World", end=" ")', str(context.exception)) + + def test_stream_redirection_hint_for_py2_migration(self): + # Test correct hint produced for Py2 redirection syntax + with self.assertRaises(TypeError) as context: + print >> sys.stderr, "message" + self.assertIn('Did you mean "print(, ' + 'file=)"?', str(context.exception)) + + # Test correct hint is produced in the case where RHS implements + # __rrshift__ but returns NotImplemented + with self.assertRaises(TypeError) as context: + print >> 42 + self.assertIn('Did you mean "print(, ' + 'file=)"?', str(context.exception)) + + # Test stream redirection hint is specific to print + with self.assertRaises(TypeError) as context: + max >> sys.stderr + self.assertNotIn('Did you mean ', str(context.exception)) + + # Test stream redirection hint is specific to rshift + with self.assertRaises(TypeError) as context: + print << sys.stderr + self.assertNotIn('Did you mean', str(context.exception)) + + # Ensure right operand implementing rrshift still works + class OverrideRRShift: + def __rrshift__(self, lhs): + return 42 # Force result independent of LHS + + self.assertEqual(print >> OverrideRRShift(), 42) + + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 15f88e4fcd7984..45ebea51a050bb 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -10,6 +10,7 @@ import select import signal import socket +import io # readline import unittest TEST_STRING_1 = b"I wish to buy a fish license.\n" @@ -23,6 +24,16 @@ def debug(msg): pass +# Note that os.read() is nondeterministic so we need to be very careful +# to make the test suite deterministic. A normal call to os.read() may +# give us less than expected. +# +# Beware, on my Linux system, if I put 'foo\n' into a terminal fd, I get +# back 'foo\r\n' at the other end. The behavior depends on the termios +# setting. The newline translation may be OS-specific. To make the +# test suite deterministic and OS-independent, the functions _readline +# and normalize_output can be used. + def normalize_output(data): # Some operating systems do conversions on newline. We could possibly # fix that by doing the appropriate termios.tcsetattr()s. I couldn't @@ -44,6 +55,12 @@ def normalize_output(data): return data +def _readline(fd): + """Read one line. May block forever if no newline is read.""" + reader = io.FileIO(fd, mode='rb', closefd=False) + return reader.readline() + + # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. @@ -98,14 +115,14 @@ def test_basic(self): debug("Writing to slave_fd") os.write(slave_fd, TEST_STRING_1) - s1 = os.read(master_fd, 1024) + s1 = _readline(master_fd) self.assertEqual(b'I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") os.write(slave_fd, TEST_STRING_2[:5]) os.write(slave_fd, TEST_STRING_2[5:]) - s2 = os.read(master_fd, 1024) + s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) os.close(slave_fd) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ed438d5664c7c0..0383a6732401be 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -361,7 +361,7 @@ def get_pydoc_html(module): def get_pydoc_link(module): "Returns a documentation web link of a module" dirname = os.path.dirname - basedir = dirname(dirname(__file__)) + basedir = dirname(dirname(os.path.realpath(__file__))) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 4ccaa39adff69f..c6860958dda6c3 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -46,28 +46,27 @@ def run(self): class BlockingTestMixin: - def tearDown(self): - self.t = None - def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): - self.t = _TriggerThread(trigger_func, trigger_args) - self.t.start() - self.result = block_func(*block_args) - # If block_func returned before our thread made the call, we failed! - if not self.t.startedEvent.is_set(): - self.fail("blocking function '%r' appeared not to block" % - block_func) - self.t.join(10) # make sure the thread terminates - if self.t.is_alive(): - self.fail("trigger function '%r' appeared to not return" % - trigger_func) - return self.result + thread = _TriggerThread(trigger_func, trigger_args) + thread.start() + try: + self.result = block_func(*block_args) + # If block_func returned before our thread made the call, we failed! + if not thread.startedEvent.is_set(): + self.fail("blocking function '%r' appeared not to block" % + block_func) + return self.result + finally: + thread.join(10) # make sure the thread terminates + if thread.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) # Call this instead if block_func is supposed to raise an exception. def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, trigger_args, expected_exception_class): - self.t = _TriggerThread(trigger_func, trigger_args) - self.t.start() + thread = _TriggerThread(trigger_func, trigger_args) + thread.start() try: try: block_func(*block_args) @@ -77,11 +76,11 @@ def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, self.fail("expected exception of kind %r" % expected_exception_class) finally: - self.t.join(10) # make sure the thread terminates - if self.t.is_alive(): + thread.join(10) # make sure the thread terminates + if thread.is_alive(): self.fail("trigger function '%r' appeared to not return" % trigger_func) - if not self.t.startedEvent.is_set(): + if not thread.startedEvent.is_set(): self.fail("trigger thread ended but event never set") @@ -159,8 +158,11 @@ def worker(self, q): def queue_join_test(self, q): self.cum = 0 + threads = [] for i in (0,1): - threading.Thread(target=self.worker, args=(q,)).start() + thread = threading.Thread(target=self.worker, args=(q,)) + thread.start() + threads.append(thread) for i in range(100): q.put(i) q.join() @@ -169,6 +171,8 @@ def queue_join_test(self, q): for i in (0,1): q.put(-1) # instruct the threads to close q.join() # verify that you can join twice + for thread in threads: + thread.join() def test_queue_task_done(self): # Test to make sure a queue task completed successfully. diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5b6a4f06baaf41..b42bf2a7124cf1 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -423,10 +423,49 @@ def test_bug_27706(self): ['0x1.b0580f98a7dbep-1', '0x1.84129978f9c1ap-1', '0x1.aeaa51052e978p-2', '0x1.092178fb945a6p-2']) + def test_bug_31478(self): + # There shouldn't be an assertion failure in _random.Random.seed() in + # case the argument has a bad __abs__() method. + class BadInt(int): + def __abs__(self): + return None + try: + self.gen.seed(BadInt()) + except TypeError: + pass + + def test_bug_31482(self): + # Verify that version 1 seeds are unaffected by hash randomization + # when the seeds are expressed as bytes rather than strings. + # The hash(b) values listed are the Python2.7 hash() values + # which were used for seeding. + + self.gen.seed(b'nofar', version=1) # hash('nofar') == 5990528763808513177 + self.assertEqual([self.gen.random().hex() for i in range(4)], + ['0x1.8645314505ad7p-1', '0x1.afb1f82e40a40p-5', + '0x1.2a59d2285e971p-1', '0x1.56977142a7880p-6']) + + self.gen.seed(b'rachel', version=1) # hash('rachel') == -9091735575445484789 + self.assertEqual([self.gen.random().hex() for i in range(4)], + ['0x1.0b294cc856fcdp-1', '0x1.2ad22d79e77b8p-3', + '0x1.3052b9c072678p-2', '0x1.578f332106574p-3']) + + self.gen.seed(b'', version=1) # hash('') == 0 + self.assertEqual([self.gen.random().hex() for i in range(4)], + ['0x1.b0580f98a7dbep-1', '0x1.84129978f9c1ap-1', + '0x1.aeaa51052e978p-2', '0x1.092178fb945a6p-2']) + + b = b'\x00\x20\x40\x60\x80\xA0\xC0\xE0\xF0' + self.gen.seed(b, version=1) # hash(b) == 5015594239749365497 + self.assertEqual([self.gen.random().hex() for i in range(4)], + ['0x1.52c2fde444d23p-1', '0x1.875174f0daea4p-2', + '0x1.9e9b2c50e5cd2p-1', '0x1.fa57768bd321cp-2']) + def test_setstate_first_arg(self): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) def test_setstate_middle_arg(self): + start_state = self.gen.getstate() # Wrong type, s/b tuple self.assertRaises(TypeError, self.gen.setstate, (2, None, None)) # Wrong length, s/b 625 @@ -440,6 +479,10 @@ def test_setstate_middle_arg(self): self.gen.setstate((2, (1,)*624+(625,), None)) with self.assertRaises((ValueError, OverflowError)): self.gen.setstate((2, (1,)*624+(-1,), None)) + # Failed calls to setstate() should not have changed the state. + bits100 = self.gen.getrandbits(100) + self.gen.setstate(start_state) + self.assertEqual(self.gen.getrandbits(100), bits100) # Little trick to make "tuple(x % (2**32) for x in internalstate)" # raise ValueError. I cannot think of a simple way to achieve this, so diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e518f61e1f..679759ec6d676a 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ def test_large_operands(self): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ def _range_len(x): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ def _range_len(x): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b945cf094e99c1..e23e5a9a6caf11 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -471,7 +471,7 @@ def test_match_getitem(self): m[(0,)] with self.assertRaisesRegex(IndexError, 'no such group'): m[(0, 1)] - with self.assertRaisesRegex(KeyError, 'a2'): + with self.assertRaisesRegex(IndexError, 'no such group'): 'a1={a2}'.format_map(m) m = pat.match('ac') @@ -1303,48 +1303,102 @@ def test_inline_flags(self): upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below lower_char = '\u1ea1' # Latin Small Letter A with Dot Below - p = re.compile(upper_char, re.I | re.U) - q = p.match(lower_char) + p = re.compile('.' + upper_char, re.I | re.S) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('.' + lower_char, re.I | re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile(lower_char, re.I | re.U) - q = p.match(upper_char) + p = re.compile('(?i).' + upper_char, re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?i)' + upper_char, re.U) - q = p.match(lower_char) + p = re.compile('(?i).' + lower_char, re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?i)' + lower_char, re.U) - q = p.match(upper_char) + p = re.compile('(?is).' + upper_char) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?iu)' + upper_char) - q = p.match(lower_char) + p = re.compile('(?is).' + lower_char) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?iu)' + lower_char) - q = p.match(upper_char) + p = re.compile('(?s)(?i).' + upper_char) + q = p.match('\n' + lower_char) self.assertTrue(q) - self.assertTrue(re.match('(?ixu) ' + upper_char, lower_char)) - self.assertTrue(re.match('(?ixu) ' + lower_char, upper_char)) + p = re.compile('(?s)(?i).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + self.assertTrue(re.match('(?ix) ' + upper_char, lower_char)) + self.assertTrue(re.match('(?ix) ' + lower_char, upper_char)) + self.assertTrue(re.match(' (?i) ' + upper_char, lower_char, re.X)) + self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) p = upper_char + '(?i)' with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s' % p + 'Flags not at the start of the expression %r' % p ) + self.assertEqual(warns.warnings[0].filename, __file__) p = upper_char + '(?i)%s' % ('.?' * 100) with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s (truncated)' % p[:20] + 'Flags not at the start of the expression %r (truncated)' % p[:20] ) + self.assertEqual(warns.warnings[0].filename, __file__) + + # bpo-30605: Compiling a bytes instance regex was throwing a BytesWarning + with warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + p = b'A(?i)' + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, b'a')) + self.assertEqual( + str(warns.warnings[0].message), + 'Flags not at the start of the expression %r' % p + ) + self.assertEqual(warns.warnings[0].filename, __file__) + + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + def test_dollar_matches_twice(self): "$ matches the end of string, and just before the terminating \n" diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 06a9149374aeab..9c979652a1d0e9 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -9,7 +9,7 @@ import sys import tempfile import unittest -from test.support import import_module, unlink, TESTFN +from test.support import import_module, unlink, temp_dir, TESTFN from test.support.script_helper import assert_python_ok # Skip tests if there is no readline module @@ -210,13 +210,57 @@ def display(substitution, matches, longest_match_length): self.assertIn(b"result " + expected + b"\r\n", output) self.assertIn(b"history " + expected + b"\r\n", output) + # We have 2 reasons to skip this test: + # - readline: history size was added in 6.0 + # See https://cnswww.cns.cwru.edu/php/chet/readline/CHANGES + # - editline: history size is broken on OS X 10.11.6. + # Newer versions were not tested yet. + @unittest.skipIf(readline._READLINE_VERSION < 0x600, + "this readline version does not support history-size") + @unittest.skipIf(is_editline, + "editline history size configuration is broken") + def test_history_size(self): + history_size = 10 + with temp_dir() as test_dir: + inputrc = os.path.join(test_dir, "inputrc") + with open(inputrc, "wb") as f: + f.write(b"set history-size %d\n" % history_size) + + history_file = os.path.join(test_dir, "history") + with open(history_file, "wb") as f: + # history_size * 2 items crashes readline + data = b"".join(b"item %d\n" % i + for i in range(history_size * 2)) + f.write(data) + + script = """ +import os +import readline + +history_file = os.environ["HISTORY_FILE"] +readline.read_history_file(history_file) +input() +readline.write_history_file(history_file) +""" + + env = dict(os.environ) + env["INPUTRC"] = inputrc + env["HISTORY_FILE"] = history_file + + run_pty(script, input=b"last input\r", env=env) + + with open(history_file, "rb") as f: + lines = f.readlines() + self.assertEqual(len(lines), history_size) + self.assertEqual(lines[-1].strip(), b"last input") + -def run_pty(script, input=b"dummy input\r"): +def run_pty(script, input=b"dummy input\r", env=None): pty = import_module('pty') output = bytearray() [master, slave] = pty.openpty() args = (sys.executable, '-c', script) - proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) os.close(slave) with ExitStack() as cleanup: cleanup.enter_context(proc) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 751df1537d2c9b..f63ed647f80e0b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -118,6 +118,9 @@ def test_header(self): ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) + ns = libregrtest._parse_args(['--verbose']) + self.assertTrue(ns.header) + def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): @@ -156,9 +159,24 @@ def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, 'pattern') + self.assertEqual(ns.match_tests, ['pattern']) self.checkError([opt], 'expected one argument') + ns = libregrtest._parse_args(['-m', 'pattern1', + '-m', 'pattern2']) + self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--matchfile', filename]) + self.assertEqual(ns.match_tests, + ['match', 'matchfile1', 'matchfile2']) + def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): @@ -173,15 +191,26 @@ def test_use(self): with self.subTest(opt=opt): ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.RESOURCE_NAMES) + + expected = list(libregrtest.ALL_RESOURCES) expected.remove('gui') ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') + # all + a resource not part of "all" + ns = libregrtest._parse_args([opt, 'all,tzdata']) + self.assertEqual(ns.use_resources, + list(libregrtest.ALL_RESOURCES) + ['tzdata']) + + # test another resource which is not part of "all" + ns = libregrtest._parse_args([opt, 'extralargefile']) + self.assertEqual(ns.use_resources, ['extralargefile']) + def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): @@ -272,7 +301,6 @@ def test_forever(self): ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) - def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') @@ -354,25 +382,25 @@ def check_line(self, output, regex): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + regex = (r'^[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' % self.TESTNAME_REGEX) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) def check_executed_tests(self, output, tests, skipped=(), failed=(), - omitted=(), randomize=False, interrupted=False): + env_changed=(), omitted=(), + randomize=False, interrupted=False, + fail_env_changed=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] if isinstance(failed, str): failed = [failed] + if isinstance(env_changed, str): + env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - ntest = len(tests) - nskipped = len(skipped) - nfailed = len(failed) - nomitted = len(omitted) executed = self.parse_executed_tests(output) if randomize: @@ -398,11 +426,17 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) + if env_changed: + regex = list_regex('%s test%s altered the execution environment', + env_changed) + self.check_line(output, regex) + if omitted: regex = list_regex('%s test%s omitted', omitted) self.check_line(output, regex) - good = ntest - nskipped - nfailed - nomitted + good = (len(tests) - len(skipped) - len(failed) + - len(omitted) - len(env_changed)) if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -412,10 +446,12 @@ def list_regex(line_format, tests): if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') - if nfailed: + if failed: result = 'FAILURE' elif interrupted: result = 'INTERRUPTED' + elif fail_env_changed and env_changed: + result = 'ENV CHANGED' else: result = 'SUCCESS' self.check_line(output, 'Tests result: %s' % result) @@ -454,7 +490,6 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): args = [sys.executable, '-X', 'faulthandler', '-I', *args] proc = self.run_command(args, **kw) @@ -588,7 +623,7 @@ def test_failing(self): test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] - output = self.run_tests(*tests, exitcode=1) + output = self.run_tests(*tests, exitcode=2) self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): @@ -676,10 +711,18 @@ def test_fromfile(self): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + # test format 'Lib/test/test_opcodes.py' + with open(filename, "w") as fp: + for name in tests: + print('Lib/test/%s.py' % name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) - output = self.run_tests(test, exitcode=1) + output = self.run_tests(test, exitcode=130) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -708,7 +751,7 @@ def test_slow_interrupted(self): args = ("--slowest", "-j2", test) else: args = ("--slowest", test) - output = self.run_tests(*args, exitcode=1) + output = self.run_tests(*args, exitcode=130) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -748,9 +791,43 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) - output = self.run_tests('--forever', test, exitcode=1) + output = self.run_tests('--forever', test, exitcode=2) self.check_executed_tests(output, [test]*3, failed=test) + def check_leak(self, code, what): + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=2, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) + self.assertIn(line2, output) + + with open(filename) as fp: + reflog = fp.read() + self.assertIn(line2, reflog) + + @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks(self): + # test --huntrleaks + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test_leak(self): + GLOBAL_LIST.append(object()) + """) + self.check_leak(code, 'references') + @unittest.skipUnless(Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak @@ -775,24 +852,7 @@ def test_leak(self): fd = os.open(__file__, os.O_RDONLY) # bug: never cloes the file descriptor """) - test = self.create_test('huntrleaks', code=code) - - filename = 'reflog.txt' - self.addCleanup(support.unlink, filename) - output = self.run_tests('--huntrleaks', '3:3:', test, - exitcode=1, - stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test) - - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) - - line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test - self.assertIn(line2, output) - - with open(filename) as fp: - reflog = fp.read() - self.assertIn(line2, reflog) + self.check_leak(code, 'file descriptors') def test_list_tests(self): # test --list-tests @@ -801,6 +861,107 @@ def test_list_tests(self): self.assertEqual(output.rstrip().splitlines(), tests) + def test_list_cases(self): + # test --list-cases + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + """) + testname = self.create_test(code=code) + + # Test --list-cases + all_methods = ['%s.Tests.test_method1' % testname, + '%s.Tests.test_method2' % testname] + output = self.run_tests('--list-cases', testname) + self.assertEqual(output.splitlines(), all_methods) + + # Test --list-cases with --match + all_methods = ['%s.Tests.test_method1' % testname] + output = self.run_tests('--list-cases', + '-m', 'test_method1', + testname) + self.assertEqual(output.splitlines(), all_methods) + + def test_crashed(self): + # Any code which causes a crash + code = 'import faulthandler; faulthandler._sigsegv()' + crash_test = self.create_test(name="crash", code=code) + ok_test = self.create_test(name="ok") + + tests = [crash_test, ok_test] + output = self.run_tests("-j2", *tests, exitcode=2) + self.check_executed_tests(output, tests, failed=crash_test, + randomize=True) + + def parse_methods(self, output): + regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) + return [match.group(1) for match in regex.finditer(output)] + + def test_matchfile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # by default, all methods should be run + output = self.run_tests("-v", testname) + methods = self.parse_methods(output) + self.assertEqual(methods, all_methods) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only match the method name + 'test_method1', + # match the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--matchfile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method1', 'test_method3'] + self.assertEqual(methods, subset) + + def test_env_changed(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_env_changed(self): + open("env_changed", "w").close() + """) + testname = self.create_test(code=code) + + # don't fail by default + output = self.run_tests(testname) + self.check_executed_tests(output, [testname], env_changed=testname) + + # fail with --fail-env-changed + output = self.run_tests("--fail-env-changed", testname, exitcode=3) + self.check_executed_tests(output, [testname], env_changed=testname, + fail_env_changed=True) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 2411895d9d12b1..2eb62905ffa882 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -4,6 +4,7 @@ from xml.sax import make_parser, ContentHandler, \ SAXException, SAXReaderNotAvailable, SAXParseException import unittest +from unittest import mock try: make_parser() except SAXReaderNotAvailable: @@ -175,12 +176,8 @@ def test_parse_bytes(self): with self.assertRaises(SAXException): self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1', None))) make_xml_file(self.data, 'iso-8859-1', None) - with support.check_warnings(('unclosed file', ResourceWarning)): - # XXX Failed parser leaks an opened file. - with self.assertRaises(SAXException): - self.check_parse(TESTFN) - # Collect leaked file. - gc.collect() + with self.assertRaises(SAXException): + self.check_parse(TESTFN) with open(TESTFN, 'rb') as f: with self.assertRaises(SAXException): self.check_parse(f) @@ -194,6 +191,21 @@ def test_parse_InputSource(self): input.setEncoding('iso-8859-1') self.check_parse(input) + def test_parse_close_source(self): + builtin_open = open + fileobj = None + + def mock_open(*args): + nonlocal fileobj + fileobj = builtin_open(*args) + return fileobj + + with mock.patch('xml.sax.saxutils.open', side_effect=mock_open): + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + self.assertTrue(fileobj.closed) + def check_parseString(self, s): from xml.sax import parseString result = StringIO() diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 2ad3a21c86d2c1..b2ab1af296d682 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1294,7 +1294,7 @@ def _boo(filename, extract_dir, extra): @unittest.skipUnless(hasattr(shutil, 'disk_usage'), "disk_usage not available on this platform") def test_disk_usage(self): - usage = shutil.disk_usage(os.getcwd()) + usage = shutil.disk_usage(os.path.dirname(__file__)) self.assertGreater(usage.total, 0) self.assertGreater(usage.used, 0) self.assertGreaterEqual(usage.free, 0) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index ab42ed70cc91ba..8d6386793d4a57 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -3,11 +3,13 @@ from contextlib import closing import enum import gc +import os import pickle +import random import select import signal import socket -import struct +import statistics import subprocess import traceback import sys, os, time, errno @@ -611,6 +613,15 @@ def test_itimer_prof(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) + def test_setitimer_tiny(self): + # bpo-30807: C setitimer() takes a microsecond-resolution interval. + # Check that float -> timeval conversion doesn't round + # the interval down to zero, which would disable the timer. + self.itimer = signal.ITIMER_REAL + signal.setitimer(self.itimer, 1e-6) + time.sleep(1) + self.assertEqual(self.hndl_called, True) + class PendingSignalsTests(unittest.TestCase): """ @@ -946,6 +957,135 @@ def handler(signum, frame): (exitcode, stdout)) +class StressTest(unittest.TestCase): + """ + Stress signal delivery, especially when a signal arrives in + the middle of recomputing the signal state or executing + previously tripped signal handlers. + """ + + def setsig(self, signum, handler): + old_handler = signal.signal(signum, handler) + self.addCleanup(signal.signal, signum, old_handler) + + def measure_itimer_resolution(self): + N = 20 + times = [] + + def handler(signum=None, frame=None): + if len(times) < N: + times.append(time.perf_counter()) + # 1 µs is the smallest possible timer interval, + # we want to measure what the concrete duration + # will be on this platform + signal.setitimer(signal.ITIMER_REAL, 1e-6) + + self.addCleanup(signal.setitimer, signal.ITIMER_REAL, 0) + self.setsig(signal.SIGALRM, handler) + handler() + while len(times) < N: + time.sleep(1e-3) + + durations = [times[i+1] - times[i] for i in range(len(times) - 1)] + med = statistics.median(durations) + if support.verbose: + print("detected median itimer() resolution: %.6f s." % (med,)) + return med + + def decide_itimer_count(self): + # Some systems have poor setitimer() resolution (for example + # measured around 20 ms. on FreeBSD 9), so decide on a reasonable + # number of sequential timers based on that. + reso = self.measure_itimer_resolution() + if reso <= 1e-4: + return 10000 + elif reso <= 1e-2: + return 100 + else: + self.skipTest("detected itimer resolution (%.3f s.) too high " + "(> 10 ms.) on this platform (or system too busy)" + % (reso,)) + + @unittest.skipUnless(hasattr(signal, "setitimer"), + "test needs setitimer()") + def test_stress_delivery_dependent(self): + """ + This test uses dependent signal handlers. + """ + N = self.decide_itimer_count() + sigs = [] + + def first_handler(signum, frame): + # 1e-6 is the minimum non-zero value for `setitimer()`. + # Choose a random delay so as to improve chances of + # triggering a race condition. Ideally the signal is received + # when inside critical signal-handling routines such as + # Py_MakePendingCalls(). + signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) + + def second_handler(signum=None, frame=None): + sigs.append(signum) + + # Here on Linux, SIGPROF > SIGALRM > SIGUSR1. By using both + # ascending and descending sequences (SIGUSR1 then SIGALRM, + # SIGPROF then SIGALRM), we maximize chances of hitting a bug. + self.setsig(signal.SIGPROF, first_handler) + self.setsig(signal.SIGUSR1, first_handler) + self.setsig(signal.SIGALRM, second_handler) # for ITIMER_REAL + + expected_sigs = 0 + deadline = time.time() + 15.0 + + while expected_sigs < N: + os.kill(os.getpid(), signal.SIGPROF) + expected_sigs += 1 + # Wait for handlers to run to avoid signal coalescing + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + os.kill(os.getpid(), signal.SIGUSR1) + expected_sigs += 1 + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + # All ITIMER_REAL signals should have been delivered to the + # Python handler + self.assertEqual(len(sigs), N, "Some signals were lost") + + @unittest.skipUnless(hasattr(signal, "setitimer"), + "test needs setitimer()") + def test_stress_delivery_simultaneous(self): + """ + This test uses simultaneous signal handlers. + """ + N = self.decide_itimer_count() + sigs = [] + + def handler(signum, frame): + sigs.append(signum) + + self.setsig(signal.SIGUSR1, handler) + self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL + + expected_sigs = 0 + deadline = time.time() + 15.0 + + while expected_sigs < N: + # Hopefully the SIGALRM will be received somewhere during + # initial processing of SIGUSR1. + signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) + os.kill(os.getpid(), signal.SIGUSR1) + + expected_sigs += 2 + # Wait for handlers to run to avoid signal coalescing + while len(sigs) < expected_sigs and time.time() < deadline: + time.sleep(1e-5) + + # All ITIMER_REAL signals should have been delivered to the + # Python handler + self.assertEqual(len(sigs), N, "Some signals were lost") + + def tearDownModule(): support.reap_children() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b242766057..4029617aa1d9c1 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -27,14 +27,27 @@ import site -if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): - # need to add user site directory for tests - try: - os.makedirs(site.USER_SITE) - site.addsitedir(site.USER_SITE) - except PermissionError as exc: - raise unittest.SkipTest('unable to create user site directory (%r): %s' - % (site.USER_SITE, exc)) + +OLD_SYS_PATH = None + + +def setUpModule(): + global OLD_SYS_PATH + OLD_SYS_PATH = sys.path[:] + + if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): + # need to add user site directory for tests + try: + os.makedirs(site.USER_SITE) + # modify sys.path: will be restored by tearDownModule() + site.addsitedir(site.USER_SITE) + except PermissionError as exc: + raise unittest.SkipTest('unable to create user site directory (%r): %s' + % (site.USER_SITE, exc)) + + +def tearDownModule(): + sys.path[:] = OLD_SYS_PATH class HelperFunctionsTests(unittest.TestCase): @@ -501,15 +514,15 @@ def _create_underpth_exe(self, lines): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 28539f360f5974..dd3a6eeca69c34 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -604,7 +604,9 @@ def setUp(self): self.sock.settimeout(15) self.port = support.bind_port(self.sock) servargs = (self.evt, self.respdata, self.sock) - threading.Thread(target=server, args=servargs).start() + thread = threading.Thread(target=server, args=servargs) + thread.start() + self.addCleanup(thread.join) self.evt.wait() self.evt.clear() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd76b24a5..80dfc405c71809 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ def testHostnameRes(self): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ def test_host_resolution(self): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): @@ -896,6 +906,7 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] @@ -4649,6 +4660,10 @@ def bind(self, sock, path): else: raise + def testUnbound(self): + # Issue #30205 + self.assertIn(self.sock.getsockname(), ('', None)) + def testStrAddr(self): # Test binding to and retrieving a normal string pathname. path = os.path.abspath(support.TESTFN) @@ -5469,7 +5484,7 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5492,8 +5507,7 @@ def test_aead_aes_gcm(self): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5501,7 +5515,7 @@ def test_aead_aes_gcm(self): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5512,7 +5526,7 @@ def test_aead_aes_gcm(self): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5520,7 +5534,7 @@ def test_aead_aes_gcm(self): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5530,8 +5544,8 @@ def test_aead_aes_gcm(self): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 140a6abf9efb9c..43621337e03dee 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -68,6 +68,20 @@ def simple_subprocess(testcase): testcase.assertEqual(72 << 8, status) +def close_server(server): + server.server_close() + + if hasattr(server, 'active_children'): + # ForkingMixIn: Manually reap all child processes, since server_close() + # calls waitpid() in non-blocking mode using the WNOHANG flag. + for pid in server.active_children.copy(): + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + server.active_children.clear() + + @unittest.skipUnless(threading, 'Threading required for this test.') class SocketServerTest(unittest.TestCase): """Test all socket servers.""" @@ -142,7 +156,7 @@ def run_server(self, svrcls, hdlrbase, testfunc): if verbose: print("waiting for server") server.shutdown() t.join() - server.server_close() + close_server(server) self.assertEqual(-1, server.socket.fileno()) if verbose: print("done") @@ -266,7 +280,7 @@ class MyHandler(socketserver.StreamRequestHandler): s.shutdown() for t, s in threads: t.join() - s.server_close() + close_server(s) def test_tcpserver_bind_leak(self): # Issue #22435: the server socket wouldn't be closed if bind()/listen() @@ -292,6 +306,7 @@ class ErrorHandlerTest(unittest.TestCase): def tearDown(self): test.support.unlink(test.support.TESTFN) + reap_children() def test_sync_handled(self): BaseErrorTestServer(ValueError) @@ -337,7 +352,7 @@ def __init__(self, exception): try: self.handle_request() finally: - self.server_close() + close_server(self) self.wait_done() def handle_error(self, request, client_address): @@ -371,10 +386,7 @@ def wait_done(self): if HAVE_FORKING: class ForkingErrorTestServer(socketserver.ForkingMixIn, BaseErrorTestServer): - def wait_done(self): - [child] = self.active_children - os.waitpid(child, 0) - self.active_children.clear() + pass class SocketWriterTest(unittest.TestCase): @@ -386,7 +398,7 @@ def handle(self): self.server.request_fileno = self.request.fileno() server = socketserver.TCPServer((HOST, 0), Handler) - self.addCleanup(server.server_close) + self.addCleanup(close_server, server) s = socket.socket( server.address_family, socket.SOCK_STREAM, socket.IPPROTO_TCP) with s: @@ -410,7 +422,7 @@ def handle(self): self.server.sent2 = self.wfile.write(big_chunk) server = socketserver.TCPServer((HOST, 0), Handler) - self.addCleanup(server.server_close) + self.addCleanup(close_server, server) interrupted = threading.Event() def signal_handler(signum, frame): @@ -486,7 +498,7 @@ def shutdown_request(self, request): s.close() server.handle_request() self.assertEqual(server.shutdown_called, 1) - server.server_close() + close_server(server) if __name__ == "__main__": diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d203cddbdfb5ee..54644e1596c7e7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -18,6 +18,10 @@ import weakref import platform import functools +try: + import ctypes +except ImportError: + ctypes = None ssl = support.import_module("ssl") @@ -172,6 +176,13 @@ def test_constants(self): ssl.OP_NO_COMPRESSION self.assertIn(ssl.HAS_SNI, {True, False}) self.assertIn(ssl.HAS_ECDH, {True, False}) + ssl.OP_NO_SSLv2 + ssl.OP_NO_SSLv3 + ssl.OP_NO_TLSv1 + ssl.OP_NO_TLSv1_3 + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 1): + ssl.OP_NO_TLSv1_1 + ssl.OP_NO_TLSv1_2 def test_str_for_enums(self): # Make sure that the PROTOCOL_* constants have enum-like string @@ -1736,6 +1747,7 @@ def test_bio_handshake(self): sslobj = ctx.wrap_bio(incoming, outgoing, False, 'localhost') self.assertIs(sslobj._sslobj.owner, sslobj) self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.version()) self.assertIsNotNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: @@ -1743,6 +1755,7 @@ def test_bio_handshake(self): self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) self.assertTrue(sslobj.cipher()) self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) self.assertTrue(sslobj.getpeercert()) if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: self.assertTrue(sslobj.get_channel_binding('tls-unique')) @@ -1793,34 +1806,6 @@ def test_get_server_certificate_ipv6(self): _test_get_server_certificate(self, 'ipv6.google.com', 443) _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) - def test_algorithms(self): - # Issue #8484: all algorithms should be available when verifying a - # certificate. - # SHA256 was added in OpenSSL 0.9.8 - if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): - self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) - # sha256.tbs-internet.com needs SNI to use the correct certificate - if not ssl.HAS_SNI: - self.skipTest("SNI needed for this test") - # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) - remote = ("sha256.tbs-internet.com", 443) - sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") - with support.transient_internet("sha256.tbs-internet.com"): - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(sha256_cert) - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname="sha256.tbs-internet.com") - try: - s.connect(remote) - if support.verbose: - sys.stdout.write("\nCipher with %r is %r\n" % - (remote, s.cipher())) - sys.stdout.write("Certificate is:\n%s\n" % - pprint.pformat(s.getpeercert())) - finally: - s.close() - def _test_get_server_certificate(test, host, port, cert=None): pem = ssl.get_server_certificate((host, port)) @@ -1871,15 +1856,22 @@ def wrap_conn(self): self.sock, server_side=True) self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) - except (ssl.SSLError, ConnectionResetError) as e: + except (ssl.SSLError, ConnectionResetError, OSError) as e: # We treat ConnectionResetError as though it were an # SSLError - OpenSSL on Ubuntu abruptly closes the # connection when asked to use an unsupported protocol. # + # OSError may occur with wrong protocols, e.g. both + # sides use PROTOCOL_TLS_SERVER. + # # XXX Various errors can have happened here, for example # a mismatching protocol version, an invalid certificate, # or a low-level bug. This should be made more discriminating. - self.server.conn_errors.append(e) + # + # bpo-31323: Store the exception as string to prevent + # a reference leak: server -> conn_errors -> exception + # -> traceback -> self (ConnectionHandler) -> server + self.server.conn_errors.append(str(e)) if self.server.chatty: handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") self.running = False @@ -2065,7 +2057,7 @@ class AsyncoreEchoServer(threading.Thread): class EchoServer (asyncore.dispatcher): - class ConnectionHandler (asyncore.dispatcher_with_send): + class ConnectionHandler(asyncore.dispatcher_with_send): def __init__(self, conn, certfile): self.socket = test_wrap_socket(conn, server_side=True, @@ -2156,6 +2148,8 @@ def __exit__(self, *args): self.join() if support.verbose: sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) def start (self, flag=None): self.flag = flag @@ -2903,7 +2897,13 @@ def _recvfrom_into(): s.send(data) buffer = bytearray(len(data)) self.assertEqual(s.read(-1, buffer), len(data)) - self.assertEqual(buffer, data) + self.assertEqual(buffer, data) # sendall accepts bytes-like objects + + if ctypes is not None: + ubyte = ctypes.c_ubyte * len(data) + byteslike = ubyte.from_buffer_copy(data) + s.sendall(byteslike) + self.assertEqual(s.read(), data) # Make sure sendmsg et al are disallowed to avoid # inadvertent disclosure of data and/or corruption @@ -3085,7 +3085,7 @@ def test_default_ciphers(self): with context.wrap_socket(socket.socket()) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", str(server.conn_errors[0])) + self.assertIn("no shared cipher", server.conn_errors[0]) def test_version_basic(self): """ @@ -3102,12 +3102,33 @@ def test_version_basic(self): self.assertEqual(s.version(), 'TLSv1') self.assertIs(s.version(), None) + @unittest.skipUnless(ssl.HAS_TLSv1_3, + "test requires TLSv1.3 enabled OpenSSL") + def test_tls1_3(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.load_cert_chain(CERTFILE) + # disable all but TLS 1.3 + context.options |= ( + ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2 + ) + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn(s.cipher()[0], [ + 'TLS13-AES-256-GCM-SHA384', + 'TLS13-CHACHA20-POLY1305-SHA256', + 'TLS13-AES-128-GCM-SHA256', + ]) + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") def test_default_ecdh_curve(self): # Issue #21015: elliptic curve-based Diffie Hellman key exchange # should be enabled by default on SSL contexts. context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.load_cert_chain(CERTFILE) + # TLSv1.3 defaults to PFS key agreement and no longer has KEA in + # cipher name. + context.options |= ssl.OP_NO_TLSv1_3 # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled # explicitly using the 'ECCdraft' cipher alias. Otherwise, # our default cipher list should prefer ECDH-based ciphers @@ -3256,8 +3277,9 @@ def test_alpn_protocols(self): except ssl.SSLError as e: stats = e - if expected is None and IS_OPENSSL_1_1: - # OpenSSL 1.1.0 raises handshake error + if (expected is None and IS_OPENSSL_1_1 + and ssl.OPENSSL_VERSION_INFO < (1, 1, 0, 6)): + # OpenSSL 1.1.0 to 1.1.0e raises handshake error self.assertIsInstance(stats, ssl.SSLError) else: msg = "failed trying %s (s) and %s (c).\n" \ @@ -3535,6 +3557,10 @@ def test_session_handling(self): context2.load_verify_locations(CERTFILE) context2.load_cert_chain(CERTFILE) + # TODO: session reuse does not work with TLS 1.3 + context.options |= ssl.OP_NO_TLSv1_3 + context2.options |= ssl.OP_NO_TLSv1_3 + server = ThreadedEchoServer(context=context, chatty=False) with server: with context.wrap_socket(socket.socket()) as s: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e63f9f254cd2b2..5016513decd8c7 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -21,12 +21,19 @@ import ctypes except ImportError: ctypes = None +else: + import ctypes.util try: import threading except ImportError: threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") @@ -42,6 +49,8 @@ else: SETBINARY = '' +NONEXISTING_CMD = ('nonexisting_i_hope',) + class BaseTestCase(unittest.TestCase): def setUp(self): @@ -54,6 +63,8 @@ def tearDown(self): inst.wait() subprocess._cleanup() self.assertFalse(subprocess._active, "subprocess._active not empty") + self.doCleanups() + support.reap_children() def assertStderrEqual(self, stderr, expected, msg=None): # In a debug build, stuff like "[6580 refs]" is printed to stderr at @@ -347,6 +358,16 @@ def test_cwd(self): temp_dir = self._normalize_cwd(temp_dir) self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + @unittest.skipIf(mswindows, "pending resolution of issue #15533") def test_cwd_with_relative_arg(self): # Check that Popen looks for args[0] relative to cwd if args[0] @@ -634,6 +655,46 @@ def test_empty_env(self): # environment b"['__CF_USER_TEXT_ENCODING']")) + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' @@ -1061,10 +1122,11 @@ def _test_bufsize_equal_one(self, line, expected, universal_newlines): p.stdin.write(line) # expect that it flushes the line in text mode os.close(p.stdin.fileno()) # close it without flushing the buffer read_line = p.stdout.readline() - try: - p.stdin.close() - except OSError: - pass + with support.SuppressCrashReport(): + try: + p.stdin.close() + except OSError: + pass p.stdin = None self.assertEqual(p.returncode, 0) self.assertEqual(read_line, expected) @@ -1089,13 +1151,54 @@ def test_leaking_fds_on_error(self): # 1024 times (each call leaked two fds). for i in range(1024): with self.assertRaises(OSError) as c: - subprocess.Popen(['nonexisting_i_hope'], + subprocess.Popen(NONEXISTING_CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # ignore errors that indicate the command was not found if c.exception.errno not in (errno.ENOENT, errno.EACCES): raise c.exception + def test_nonexisting_with_pipes(self): + # bpo-30121: Popen with pipes must close properly pipes on error. + # Previously, os.close() was called with a Windows handle which is not + # a valid file descriptor. + # + # Run the test in a subprocess to control how the CRT reports errors + # and to get stderr content. + try: + import msvcrt + msvcrt.CrtSetReportMode + except (AttributeError, ImportError): + self.skipTest("need msvcrt.CrtSetReportMode") + + code = textwrap.dedent(f""" + import msvcrt + import subprocess + + cmd = {NONEXISTING_CMD!r} + + for report_type in [msvcrt.CRT_WARN, + msvcrt.CRT_ERROR, + msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(report_type, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(report_type, msvcrt.CRTDBG_FILE_STDERR) + + try: + subprocess.Popen([cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError: + pass + """) + cmd = [sys.executable, "-c", code] + proc = subprocess.Popen(cmd, + stderr=subprocess.PIPE, + universal_newlines=True) + with proc: + stderr = proc.communicate()[1] + self.assertEqual(stderr, "") + self.assertEqual(proc.returncode, 0) + @unittest.skipIf(threading is None, "threading required") def test_double_close_on_error(self): # Issue #18851 @@ -1108,7 +1211,7 @@ def open_fds(): t.start() try: with self.assertRaises(EnvironmentError): - subprocess.Popen(['nonexisting_i_hope'], + subprocess.Popen(NONEXISTING_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -1272,6 +1375,18 @@ def test_failed_child_execute_fd_leak(self): fds_after_exception = os.listdir(fd_directory) self.assertEqual(fds_before_popen, fds_after_exception) + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_includes_filename(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.call(['/opt/nonexistent_binary', 'with', 'some', 'args']) + self.assertEqual(c.exception.filename, '/opt/nonexistent_binary') + + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_with_bad_cwd(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.Popen(['exit', '0'], cwd='/some/nonexistent/directory') + self.assertEqual(c.exception.filename, '/some/nonexistent/directory') + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1430,6 +1545,57 @@ def test_exception_bad_args_0(self): else: self.fail("Expected OSError: %s" % desired_exception) + # We mock the __del__ method for Popen in the next two tests + # because it does cleanup based on the pid returned by fork_exec + # along with issuing a resource warning if it still exists. Since + # we don't actually spawn a process in these tests we can forego + # the destructor. An alternative would be to set _child_created to + # False before the destructor is called but there is no easy way + # to do that + class PopenNoDestructor(subprocess.Popen): + def __del__(self): + pass + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_normal(self, fork_exec): + """Test error passing done through errpipe_write in the good case""" + def proper_error(*args): + errpipe_write = args[13] + # Write the hex for the error code EISDIR: 'is a directory' + err_code = '{:x}'.format(errno.EISDIR).encode() + os.write(errpipe_write, b"OSError:" + err_code + b":") + return 0 + + fork_exec.side_effect = proper_error + + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(IsADirectoryError): + self.PopenNoDestructor(["non_existent_command"]) + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_bad_data(self, fork_exec): + """Test error passing done through errpipe_write where its not + in the expected format""" + error_data = b"\xFF\x00\xDE\xAD" + def bad_error(*args): + errpipe_write = args[13] + # Anything can be in the pipe, no assumptions should + # be made about its encoding, so we'll write some + # arbitrary hex bytes to test it out + os.write(errpipe_write, error_data) + return 0 + + fork_exec.side_effect = bad_error + + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(subprocess.SubprocessError) as e: + self.PopenNoDestructor(["non_existent_command"]) + + self.assertIn(repr(error_data), str(e.exception)) + + def test_restore_signals(self): # Code coverage for both values of restore_signals to make sure it # at least does not blow up. @@ -2374,7 +2540,7 @@ def test_leak_fast_process_del_killed(self): # should trigger the wait() of p time.sleep(0.2) with self.assertRaises(OSError) as c: - with subprocess.Popen(['nonexisting_i_hope'], + with subprocess.Popen(NONEXISTING_CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: pass @@ -2421,7 +2587,7 @@ def test_fork_exec(self): with self.assertRaises(TypeError): _posixsubprocess.fork_exec( args, exe_list, - True, [], cwd, env_list, + True, (), cwd, env_list, -1, -1, -1, -1, 1, 2, 3, 4, True, True, func) @@ -2433,6 +2599,16 @@ def test_fork_exec(self): def test_fork_exec_sorted_fd_sanity_check(self): # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + gc_enabled = gc.isenabled() try: gc.enable() @@ -2443,6 +2619,7 @@ def test_fork_exec_sorted_fd_sanity_check(self): (18, 23, 42, 2**63), # Out of range. (5, 4), # Not sorted. (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), ): with self.assertRaises( ValueError, @@ -2504,45 +2681,24 @@ def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): proc.communicate(timeout=999) mock_proc_stdin.close.assert_called_once_with() - _libc_file_extensions = { - 'Linux': 'so.6', - 'Darwin': 'dylib', - } - @unittest.skipIf(not ctypes, 'ctypes module required.') - @unittest.skipIf(platform.uname()[0] not in _libc_file_extensions, - 'Test requires a libc this code can load with ctypes.') - @unittest.skipIf(not sys.executable, 'Test requires sys.executable.') - def test_child_terminated_in_stopped_state(self): + @unittest.skipUnless(_testcapi is not None + and hasattr(_testcapi, 'W_STOPCODE'), + 'need _testcapi.W_STOPCODE') + def test_stopped(self): """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" - PTRACE_TRACEME = 0 # From glibc and MacOS (PT_TRACE_ME). - libc_name = 'libc.' + self._libc_file_extensions[platform.uname()[0]] - libc = ctypes.CDLL(libc_name) - if not hasattr(libc, 'ptrace'): - raise unittest.SkipTest('ptrace() required.') - test_ptrace = subprocess.Popen( - [sys.executable, '-c', """if True: - import ctypes - libc = ctypes.CDLL({libc_name!r}) - libc.ptrace({PTRACE_TRACEME}, 0, 0) - """.format(libc_name=libc_name, PTRACE_TRACEME=PTRACE_TRACEME) - ]) - if test_ptrace.wait() != 0: - raise unittest.SkipTest('ptrace() failed - unable to test.') - child = subprocess.Popen( - [sys.executable, '-c', """if True: - import ctypes - libc = ctypes.CDLL({libc_name!r}) - libc.ptrace({PTRACE_TRACEME}, 0, 0) - libc.printf(ctypes.c_char_p(0xdeadbeef)) # Crash the process. - """.format(libc_name=libc_name, PTRACE_TRACEME=PTRACE_TRACEME) - ]) - try: - returncode = child.wait() - except Exception as e: - child.kill() # Clean up the hung stopped process. - raise e - self.assertNotEqual(0, returncode) - self.assertLess(returncode, 0) # signal death, likely SIGSEGV. + args = [sys.executable, '-c', 'pass'] + proc = subprocess.Popen(args) + + # Wait until the real process completes to avoid zombie process + pid = proc.pid + pid, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + status = _testcapi.W_STOPCODE(3) + with mock.patch('subprocess.os.waitpid', return_value=(pid, status)): + returncode = proc.wait() + + self.assertEqual(returncode, -3) @unittest.skipUnless(mswindows, "Windows specific tests") @@ -2583,6 +2739,15 @@ def test_invalid_args(self): stdout=subprocess.PIPE, close_fds=True) + @support.cpython_only + def test_issue31471(self): + # There shouldn't be an assertion failure in Popen() in case the env + # argument has a bad keys() method. + class BadEnv(dict): + keys = None + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], env=BadEnv()) + def test_close_fds(self): # close file descriptors rc = subprocess.call([sys.executable, "-c", @@ -2814,7 +2979,7 @@ def test_communicate_stdin(self): def test_invalid_args(self): with self.assertRaises((FileNotFoundError, PermissionError)) as c: - with subprocess.Popen(['nonexisting_i_hope'], + with subprocess.Popen(NONEXISTING_CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: pass diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e83a4d6426a8b2..ddd65092312a3b 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -282,17 +282,34 @@ def test_python_is_optimized(self): def test_swap_attr(self): class Obj: - x = 1 + pass obj = Obj() - with support.swap_attr(obj, "x", 5): + obj.x = 1 + with support.swap_attr(obj, "x", 5) as x: self.assertEqual(obj.x, 5) + self.assertEqual(x, 1) self.assertEqual(obj.x, 1) + with support.swap_attr(obj, "y", 5) as y: + self.assertEqual(obj.y, 5) + self.assertIsNone(y) + self.assertFalse(hasattr(obj, 'y')) + with support.swap_attr(obj, "y", 5): + del obj.y + self.assertFalse(hasattr(obj, 'y')) def test_swap_item(self): - D = {"item":1} - with support.swap_item(D, "item", 5): - self.assertEqual(D["item"], 5) - self.assertEqual(D["item"], 1) + D = {"x":1} + with support.swap_item(D, "x", 5) as x: + self.assertEqual(D["x"], 5) + self.assertEqual(x, 1) + self.assertEqual(D["x"], 1) + with support.swap_item(D, "y", 5) as y: + self.assertEqual(D["y"], 5) + self.assertIsNone(y) + self.assertNotIn("y", D) + with support.swap_item(D, "y", 5): + del D["y"] + self.assertNotIn("y", D) class RefClass: attribute1 = None diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index df9ebd40859e21..e151f493d4e36c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -913,13 +913,15 @@ def inner(): return inner check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size('6i13P')) - check(get_cell.__code__, size('6i13P')) + def check_code_size(a, expected_size): + self.assertGreaterEqual(sys.getsizeof(a), expected_size) + check_code_size(get_cell().__code__, size('6i13P')) + check_code_size(get_cell.__code__, size('6i13P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size('6i13P') + 1) + check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n')) # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc03b28abc..fc79055421161c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ def test_extract_directory(self): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ def test_directory_size(self): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ def test_create_existing_taropen(self): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 8ffd1853147e49..db99b75eec6458 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -662,32 +662,44 @@ def setUp(self): @support.bigmemtest(size=INT_MAX + 1, memuse=5, dry_run=False) def test_huge_string_call(self, size): value = ' ' * size - self.assertRaises(OverflowError, self.interp.call, 'set', '_', value) + self.assertRaises(OverflowError, self.interp.call, 'string', 'index', value, 0) @support.cpython_only @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX") - @support.bigmemtest(size=INT_MAX + 1, memuse=9, dry_run=False) + @support.bigmemtest(size=INT_MAX + 1, memuse=2, dry_run=False) def test_huge_string_builtins(self, size): + tk = self.interp.tk value = '1' + ' ' * size - self.assertRaises(OverflowError, self.interp.tk.getint, value) - self.assertRaises(OverflowError, self.interp.tk.getdouble, value) - self.assertRaises(OverflowError, self.interp.tk.getboolean, value) - self.assertRaises(OverflowError, self.interp.eval, value) - self.assertRaises(OverflowError, self.interp.evalfile, value) - self.assertRaises(OverflowError, self.interp.record, value) - self.assertRaises(OverflowError, self.interp.adderrorinfo, value) - self.assertRaises(OverflowError, self.interp.setvar, value, 'x', 'a') - self.assertRaises(OverflowError, self.interp.setvar, 'x', value, 'a') - self.assertRaises(OverflowError, self.interp.unsetvar, value) - self.assertRaises(OverflowError, self.interp.unsetvar, 'x', value) - self.assertRaises(OverflowError, self.interp.adderrorinfo, value) - self.assertRaises(OverflowError, self.interp.exprstring, value) - self.assertRaises(OverflowError, self.interp.exprlong, value) - self.assertRaises(OverflowError, self.interp.exprboolean, value) - self.assertRaises(OverflowError, self.interp.splitlist, value) - self.assertRaises(OverflowError, self.interp.split, value) - self.assertRaises(OverflowError, self.interp.createcommand, value, max) - self.assertRaises(OverflowError, self.interp.deletecommand, value) + self.assertRaises(OverflowError, tk.getint, value) + self.assertRaises(OverflowError, tk.getdouble, value) + self.assertRaises(OverflowError, tk.getboolean, value) + self.assertRaises(OverflowError, tk.eval, value) + self.assertRaises(OverflowError, tk.evalfile, value) + self.assertRaises(OverflowError, tk.record, value) + self.assertRaises(OverflowError, tk.adderrorinfo, value) + self.assertRaises(OverflowError, tk.setvar, value, 'x', 'a') + self.assertRaises(OverflowError, tk.setvar, 'x', value, 'a') + self.assertRaises(OverflowError, tk.unsetvar, value) + self.assertRaises(OverflowError, tk.unsetvar, 'x', value) + self.assertRaises(OverflowError, tk.adderrorinfo, value) + self.assertRaises(OverflowError, tk.exprstring, value) + self.assertRaises(OverflowError, tk.exprlong, value) + self.assertRaises(OverflowError, tk.exprboolean, value) + self.assertRaises(OverflowError, tk.splitlist, value) + self.assertRaises(OverflowError, tk.split, value) + self.assertRaises(OverflowError, tk.createcommand, value, max) + self.assertRaises(OverflowError, tk.deletecommand, value) + + @support.cpython_only + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX") + @support.bigmemtest(size=INT_MAX + 1, memuse=6, dry_run=False) + def test_huge_string_builtins2(self, size): + # These commands require larger memory for possible error messages + tk = self.interp.tk + value = '1' + ' ' * size + self.assertRaises(OverflowError, tk.evalfile, value) + self.assertRaises(OverflowError, tk.unsetvar, value) + self.assertRaises(OverflowError, tk.unsetvar, 'x', value) def setUpModule(): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 51df1ecd7d18e6..710756bde64c04 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -78,7 +78,6 @@ def setUp(self): def tearDown(self): self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] @@ -184,12 +183,15 @@ def test_process_awareness(self): try: pid = os.fork() if not pid: + # child process os.close(read_fd) os.write(write_fd, next(self.r).encode("ascii")) os.close(write_fd) # bypass the normal exit handlers- leave those to # the parent. os._exit(0) + + # parent process parent_value = next(self.r) child_value = os.read(read_fd, len(parent_value)).decode("ascii") finally: @@ -200,6 +202,10 @@ def test_process_awareness(self): os.kill(pid, signal.SIGKILL) except OSError: pass + + # Read the process exit status to avoid zombie process + os.waitpid(pid, 0) + os.close(read_fd) os.close(write_fd) self.assertNotEqual(child_value, parent_value) @@ -273,13 +279,12 @@ def raise_OSError(*args, **kwargs): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - open = io.open def bad_writer(*args, **kwargs): - fp = open(*args, **kwargs) + fp = orig_open(*args, **kwargs) fp.write = raise_OSError return fp - with support.swap_attr(io, "open", bad_writer): + with support.swap_attr(io, "open", bad_writer) as orig_open: # test again with failing write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index ef3059b68674b4..2dd1593eaa0cb1 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -11,6 +11,7 @@ NUMTASKS = 10 NUMTRIPS = 3 +POLL_SLEEP = 0.010 # seconds = 10 ms _print_mutex = thread.allocate_lock() @@ -20,6 +21,7 @@ def verbose_print(arg): with _print_mutex: print(arg) + class BasicThreadTest(unittest.TestCase): def setUp(self): @@ -31,6 +33,9 @@ def setUp(self): self.running = 0 self.next_ident = 0 + key = support.threading_setup() + self.addCleanup(support.threading_cleanup, *key) + class ThreadRunningTests(BasicThreadTest): @@ -110,7 +115,7 @@ def task(): mut.release() thread.start_new_thread(task, ()) while not started: - time.sleep(0.01) + time.sleep(POLL_SLEEP) self.assertEqual(thread._count(), orig + 1) # Allow the task to finish. mut.release() @@ -121,7 +126,7 @@ def task(): wr = weakref.ref(task, lambda _: done.append(None)) del task while not done: - time.sleep(0.01) + time.sleep(POLL_SLEEP) self.assertEqual(thread._count(), orig) def test_save_exception_state_on_error(self): @@ -144,7 +149,7 @@ def mywrite(self, *args): thread.start_new_thread(task, ()) started.acquire() while thread._count() > c: - time.sleep(0.01) + time.sleep(POLL_SLEEP) self.assertIn("Traceback", stderr.getvalue()) @@ -217,28 +222,36 @@ class TestForkInThread(unittest.TestCase): def setUp(self): self.read_fd, self.write_fd = os.pipe() - @unittest.skipIf(sys.platform.startswith('win'), - "This test is only appropriate for POSIX-like systems.") + @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') @support.reap_threads def test_forkinthread(self): + running = True + status = "not set" + def thread1(): - try: - pid = os.fork() # fork in a thread - except RuntimeError: - os._exit(1) # exit the child + nonlocal running, status - if pid == 0: # child + # fork in a thread + pid = os.fork() + if pid == 0: + # child try: os.close(self.read_fd) os.write(self.write_fd, b"OK") finally: os._exit(0) - else: # parent + else: + # parent os.close(self.write_fd) + pid, status = os.waitpid(pid, 0) + running = False thread.start_new_thread(thread1, ()) self.assertEqual(os.read(self.read_fd, 2), b"OK", "Unable to fork() in thread") + while running: + time.sleep(POLL_SLEEP) + self.assertEqual(status, 0) def tearDown(self): try: diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py index 9b2d9a6f19b8b6..f42c900103f0b9 100644 --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -221,7 +221,8 @@ def target(): import random t = threading.Thread(target=target) t.start() - t.join()""" + t.join() + t = None""" sys.path.insert(0, os.curdir) self.addCleanup(sys.path.remove, os.curdir) filename = TESTFN + ".py" @@ -232,6 +233,7 @@ def target(): self.addCleanup(rmtree, '__pycache__') importlib.invalidate_caches() __import__(TESTFN) + del sys.modules[TESTFN] @reap_threads diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2c2914fd6d8969..b42314fdbb20d6 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -170,6 +170,9 @@ def f(mutex): mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) @@ -467,13 +470,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - self.addCleanup(t.join) pid = os.fork() if pid == 0: - os._exit(1 if t.is_alive() else 0) + os._exit(11 if t.is_alive() else 10) else: + t.join() + pid, status = os.waitpid(pid, 0) - self.assertEqual(0, status) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) def test_main_thread(self): main = threading.main_thread() @@ -537,6 +542,35 @@ def f(): self.assertEqual(err, b"") self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + def test_main_thread_during_shutdown(self): + # bpo-31516: current_thread() should still point to the main thread + # at shutdown + code = """if 1: + import gc, threading + + main_thread = threading.current_thread() + assert main_thread is threading.main_thread() # sanity check + + class RefCycle: + def __init__(self): + self.cycle = self + + def __del__(self): + print("GC:", + threading.current_thread() is main_thread, + threading.main_thread() is main_thread, + threading.enumerate() == [main_thread]) + + RefCycle() + gc.collect() # sanity check + x = RefCycle() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode() + self.assertEqual(err, b"") + self.assertEqual(data.splitlines(), + ["GC: True True True"] * 2) + def test_tstate_lock(self): # Test an implementation detail of Thread objects. started = _thread.allocate_lock() @@ -570,6 +604,7 @@ def f(): self.assertFalse(t.is_alive()) # And verify the thread disposed of _tstate_lock. self.assertIsNone(t._tstate_lock) + t.join() def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. @@ -596,6 +631,7 @@ def f(): break time.sleep(0.01) self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + t.join() def test_BoundedSemaphore_limit(self): # BoundedSemaphore should raise ValueError if released too often. @@ -910,6 +946,7 @@ def test_start_thread_again(self): thread = threading.Thread() thread.start() self.assertRaises(RuntimeError, thread.start) + thread.join() def test_joining_current_thread(self): current_thread = threading.current_thread() @@ -923,6 +960,7 @@ def test_daemonize_active_thread(self): thread = threading.Thread() thread.start() self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + thread.join() def test_releasing_unacquired_lock(self): lock = threading.Lock() @@ -1061,6 +1099,8 @@ def run(self): thread.join() self.assertIsNotNone(thread.exc) self.assertIsInstance(thread.exc, RuntimeError) + # explicitly break the reference cycle to not leak a dangling thread + thread.exc = None class TimerTests(BaseTestCase): @@ -1083,6 +1123,8 @@ def test_init_immutable_default_args(self): self.callback_event.wait() self.assertEqual(len(self.callback_args), 2) self.assertEqual(self.callback_args, [((), {}), ((), {})]) + timer1.join() + timer2.join() def _callback_spy(self, *args, **kwargs): self.callback_args.append((args[:], kwargs.copy())) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index f2242126d1826d..7093fc6a2ec172 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -126,6 +126,10 @@ def test_strftime(self): except ValueError: self.fail('conversion specifier: %r failed.' % format) + self.assertRaises(TypeError, time.strftime, b'%S', tt) + # embedded null character + self.assertRaises(ValueError, time.strftime, '%S\0', tt) + def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts # of the time tuple (0 is valid for *all* values). @@ -485,6 +489,10 @@ def test_localtime_failure(self): self.assertRaises(OSError, time.localtime, invalid_time_t) self.assertRaises(OSError, time.ctime, invalid_time_t) + # Issue #26669: check for localtime() failure + self.assertRaises(ValueError, time.localtime, float("nan")) + self.assertRaises(ValueError, time.ctime, float("nan")) + def test_get_clock_info(self): clocks = ['clock', 'perf_counter', 'process_time', 'time'] if hasattr(time, 'monotonic'): @@ -819,6 +827,11 @@ def c_int_filter(secs): lambda secs: secs * SEC_TO_NS, value_filter=c_int_filter) + # test nan + for time_rnd, _ in ROUNDING_MODES: + with self.assertRaises(TypeError): + PyTime_FromSeconds(float('nan')) + def test_FromSecondsObject(self): from _testcapi import PyTime_FromSecondsObject @@ -830,6 +843,11 @@ def test_FromSecondsObject(self): PyTime_FromSecondsObject, lambda ns: self.decimal_round(ns * SEC_TO_NS)) + # test nan + for time_rnd, _ in ROUNDING_MODES: + with self.assertRaises(ValueError): + PyTime_FromSecondsObject(float('nan'), time_rnd) + def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble @@ -843,6 +861,11 @@ def float_converter(ns): float_converter, NS_TO_SEC) + # test nan + for time_rnd, _ in ROUNDING_MODES: + with self.assertRaises(TypeError): + PyTime_AsSecondsDouble(float('nan')) + def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -948,6 +971,11 @@ def test_object_to_timeval(self): self.create_converter(SEC_TO_US), value_filter=self.time_t_filter) + # test nan + for time_rnd, _ in ROUNDING_MODES: + with self.assertRaises(ValueError): + pytime_object_to_timeval(float('nan'), time_rnd) + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec @@ -959,6 +987,11 @@ def test_object_to_timespec(self): self.create_converter(SEC_TO_NS), value_filter=self.time_t_filter) + # test nan + for time_rnd, _ in ROUNDING_MODES: + with self.assertRaises(ValueError): + pytime_object_to_timespec(float('nan'), time_rnd) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5a81a5f11a4adb..10e0ad807798e1 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3,6 +3,7 @@ STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, open as tokenize_open, Untokenizer) from io import BytesIO +import unittest from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py index d7c20e1e5c7206..34df0c5d511904 100644 --- a/Lib/test/test_tools/test_reindent.py +++ b/Lib/test/test_tools/test_reindent.py @@ -7,6 +7,7 @@ import os import unittest from test.support.script_helper import assert_python_ok +from test.support import findfile from test.test_tools import scriptsdir, skip_if_missing @@ -23,6 +24,12 @@ def test_help(self): self.assertEqual(out, b'') self.assertGreater(err, b'') + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 7276bc7ee79079..e4833535890d15 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -344,7 +344,8 @@ def f(): # 2nd last line contains the repetition count self.assertEqual(actual[:-2], expected[:-2]) self.assertRegex(actual[-2], expected[-2]) - self.assertEqual(actual[-1], expected[-1]) + # last line can have additional text appended + self.assertIn(expected[-1], actual[-1]) # Check the recursion count is roughly as expected rec_limit = sys.getrecursionlimit() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7ee606d60..742259b4396b98 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ def test_sys_xoptions_invalid(self): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 382ca03e5ad2b8..d79429faf118f6 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -846,6 +846,28 @@ def __prepare__(*args): self.assertIs(ns, expected_ns) self.assertEqual(len(kwds), 0) + def test_bad___prepare__(self): + # __prepare__() must return a mapping. + class BadMeta(type): + @classmethod + def __prepare__(*args): + return None + with self.assertRaisesRegex(TypeError, + r'^BadMeta\.__prepare__\(\) must ' + r'return a mapping, not NoneType$'): + class Foo(metaclass=BadMeta): + pass + # Also test the case in which the metaclass is not a type. + class BadMeta: + @classmethod + def __prepare__(*args): + return None + with self.assertRaisesRegex(TypeError, + r'^\.__prepare__\(\) must ' + r'return a mapping, not NoneType$'): + class Bar(metaclass=BadMeta()): + pass + def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation new_calls = [] # to check the order of __new__ calls diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fce6b5aaffdf93..a3b2ea7f03e5f2 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3,10 +3,10 @@ import pickle import re import sys -from unittest import TestCase, main, skipUnless, SkipTest +from unittest import TestCase, main, skipUnless, SkipTest, expectedFailure from copy import copy, deepcopy -from typing import Any +from typing import Any, NoReturn from typing import TypeVar, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional @@ -30,6 +30,13 @@ import collections as collections_abc # Fallback for PY3.2. +try: + import mod_generics_cache +except ImportError: + # try to use the builtin one, Python 3.5+ + from test import mod_generics_cache + + class BaseTestCase(TestCase): def assertIsSubclass(self, cls, class_or_tuple, msg=None): @@ -102,10 +109,6 @@ def test_cannot_instantiate(self): with self.assertRaises(TypeError): type(Any)() - def test_cannot_subscript(self): - with self.assertRaises(TypeError): - Any[int] - def test_any_works_with_alias(self): # These expressions must simply not fail. typing.Match[Any] @@ -113,6 +116,40 @@ def test_any_works_with_alias(self): typing.IO[Any] +class NoReturnTests(BaseTestCase): + + def test_noreturn_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, NoReturn) + + def test_noreturn_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, NoReturn) + with self.assertRaises(TypeError): + issubclass(NoReturn, Employee) + + def test_repr(self): + self.assertEqual(repr(NoReturn), 'typing.NoReturn') + + def test_not_generic(self): + with self.assertRaises(TypeError): + NoReturn[int] + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class A(NoReturn): + pass + with self.assertRaises(TypeError): + class A(type(NoReturn)): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + NoReturn() + with self.assertRaises(TypeError): + type(NoReturn)() + + class TypeVarTests(BaseTestCase): def test_basic_plain(self): @@ -190,6 +227,10 @@ def test_bound_errors(self): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, contravariant=True) + class UnionTests(BaseTestCase): @@ -254,6 +295,11 @@ def test_repr(self): self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) u = Union[int, Employee] self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') def test_cannot_subclass(self): with self.assertRaises(TypeError): @@ -304,6 +350,15 @@ def test_union_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Union[int, str]) + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 A = Union[str, Pattern] @@ -401,6 +456,8 @@ def test_callable_wrong_forms(self): Callable[[()], int] with self.assertRaises(TypeError): Callable[[int, 1], 2] + with self.assertRaises(TypeError): + Callable[int] def test_callable_instance_works(self): def f(): @@ -546,15 +603,27 @@ def test_basics(self): Y[str] with self.assertRaises(TypeError): Y[str, str] + self.assertIsSubclass(SimpleMapping[str, int], SimpleMapping) def test_generic_errors(self): T = TypeVar('T') + S = TypeVar('S') with self.assertRaises(TypeError): Generic[T]() + with self.assertRaises(TypeError): + Generic[T][T] + with self.assertRaises(TypeError): + Generic[T][S] with self.assertRaises(TypeError): isinstance([], List[int]) with self.assertRaises(TypeError): issubclass(list, List[int]) + with self.assertRaises(TypeError): + class NewGeneric(Generic): ... + with self.assertRaises(TypeError): + class MyGeneric(Generic[T], Generic[S]): ... + with self.assertRaises(TypeError): + class MyGeneric(List[T], Generic[S]): ... def test_init(self): T = TypeVar('T') @@ -642,6 +711,41 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_subscripted_generics_as_proxies(self): + T = TypeVar('T') + class C(Generic[T]): + x = 'def' + self.assertEqual(C[int].x, 'def') + self.assertEqual(C[C[int]].x, 'def') + C[C[int]].x = 'changed' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[str].x, 'changed') + C[List[str]].z = 'new' + self.assertEqual(C.z, 'new') + self.assertEqual(C[Tuple[int]].z, 'new') + + self.assertEqual(C().x, 'changed') + self.assertEqual(C[Tuple[str]]().z, 'new') + + class D(C[T]): + pass + self.assertEqual(D[int].x, 'changed') + self.assertEqual(D.z, 'new') + D.z = 'from derived z' + D[int].x = 'from derived x' + self.assertEqual(C.x, 'changed') + self.assertEqual(C[int].z, 'new') + self.assertEqual(D.x, 'from derived x') + self.assertEqual(D[str].z, 'from derived z') + + def test_abc_registry_kept(self): + T = TypeVar('T') + class C(Generic[T]): ... + C.register(int) + self.assertIsInstance(1, C) + C[int] + self.assertIsInstance(1, C) + def test_false_subclasses(self): class MyMapping(MutableMapping[str, str]): pass self.assertNotIsInstance({}, MyMapping) @@ -738,6 +842,49 @@ def test_subscript_meta(self): self.assertEqual(Union[T, int][GenericMeta], Union[GenericMeta, int]) self.assertEqual(Callable[..., GenericMeta].__args__, (Ellipsis, GenericMeta)) + def test_generic_hashes(self): + class A(Generic[T]): + ... + + class B(Generic[T]): + class A(Generic[T]): + ... + + self.assertEqual(A, A) + self.assertEqual(mod_generics_cache.A[str], mod_generics_cache.A[str]) + self.assertEqual(B.A, B.A) + self.assertEqual(mod_generics_cache.B.A[B.A[str]], + mod_generics_cache.B.A[B.A[str]]) + + self.assertNotEqual(A, B.A) + self.assertNotEqual(A, mod_generics_cache.A) + self.assertNotEqual(A, mod_generics_cache.B.A) + self.assertNotEqual(B.A, mod_generics_cache.A) + self.assertNotEqual(B.A, mod_generics_cache.B.A) + + self.assertNotEqual(A[str], B.A[str]) + self.assertNotEqual(A[List[Any]], B.A[List[Any]]) + self.assertNotEqual(A[str], mod_generics_cache.A[str]) + self.assertNotEqual(A[str], mod_generics_cache.B.A[str]) + self.assertNotEqual(B.A[int], mod_generics_cache.A[int]) + self.assertNotEqual(B.A[List[Any]], mod_generics_cache.B.A[List[Any]]) + + self.assertNotEqual(Tuple[A[str]], Tuple[B.A[str]]) + self.assertNotEqual(Tuple[A[List[Any]]], Tuple[B.A[List[Any]]]) + self.assertNotEqual(Union[str, A[str]], Union[str, mod_generics_cache.A[str]]) + self.assertNotEqual(Union[A[str], A[str]], + Union[A[str], mod_generics_cache.A[str]]) + self.assertNotEqual(typing.FrozenSet[A[str]], + typing.FrozenSet[mod_generics_cache.B.A[str]]) + + if sys.version_info[:2] > (3, 2): + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -835,6 +982,8 @@ def test_fail_with_bare_generic(self): Tuple[Generic[T]] with self.assertRaises(TypeError): List[typing._Protocol] + with self.assertRaises(TypeError): + isinstance(1, Generic) def test_type_erasure_special(self): T = TypeVar('T') @@ -853,6 +1002,11 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertIs(MyDef[int]().__orig_class__, MyDef[int]) + # ChainMap was added in 3.3 + if sys.version_info >= (3, 3): + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertIs(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) @@ -918,6 +1072,13 @@ class Node(Generic[T]): ... for t in things + [Any]: self.assertEqual(t, copy(t)) self.assertEqual(t, deepcopy(t)) + if sys.version_info >= (3, 3): + # From copy module documentation: + # It does "copy" functions and classes (shallow and deeply), by returning + # the original object unchanged; this is compatible with the way these + # are treated by the pickle module. + self.assertTrue(t is copy(t)) + self.assertTrue(t is deepcopy(t)) def test_weakref_all(self): T = TypeVar('T') @@ -1203,6 +1364,19 @@ def test_forwardref_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, fr) + def test_forwardref_subclass_type_error(self): + fr = typing._ForwardRef('int') + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forward_equality(self): + fr = typing._ForwardRef('int') + self.assertEqual(fr, typing._ForwardRef('int')) + self.assertNotEqual(List['int'], List[int]) + + def test_forward_repr(self): + self.assertEqual(repr(List['int']), "typing.List[_ForwardRef('int')]") + def test_union_forward(self): def foo(a: Union['T']): @@ -1285,6 +1459,15 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) + def test_no_type_check_no_bases(self): + class C: + def meth(self, x: int): ... + @no_type_check + class D(C): + c = C + # verify that @no_type_check never affects bases + self.assertEqual(get_type_hints(C.meth), {'x': int}) + def test_meta_no_type_check(self): @no_type_check_decorator @@ -1379,6 +1562,12 @@ def __anext__(self) -> T_a: return data else: raise StopAsyncIteration + +class ACM: + async def __aenter__(self) -> int: + return 42 + async def __aexit__(self, etype, eval, tb): + return None """ if ASYNCIO: @@ -1389,23 +1578,29 @@ def __anext__(self) -> T_a: else: # fake names for the sake of static analysis asyncio = None - AwaitableWrapper = AsyncIteratorWrapper = object + AwaitableWrapper = AsyncIteratorWrapper = ACM = object PY36 = sys.version_info[:2] >= (3, 6) PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 +from typing import AsyncContextManager class A: y: float class B(A): x: ClassVar[Optional['B']] = None y: int + b: int class CSub(B): z: ClassVar['CSub'] = B() class G(Generic[T]): lst: ClassVar[List[T]] = [] +class NoneAndForward: + parent: 'NoneAndForward' + meaning: None + class CoolEmployee(NamedTuple): name: str cool: int @@ -1419,10 +1614,27 @@ class XMeth(NamedTuple): def double(self): return 2 * self.x -class XMethBad(NamedTuple): +class XRepr(NamedTuple): x: int - def _fields(self): - return 'no chance for this' + y: int = 1 + def __str__(self): + return f'{self.x} -> {self.y}' + def __add__(self, other): + return 0 + +class HasForeignBaseClass(mod_generics_cache.A): + some_xrepr: 'XRepr' + other_a: 'mod_generics_cache.A' + +async def g_with(am: AsyncContextManager[int]): + x: int + async with am as x: + return x + +try: + g_with(ACM()).send(None) +except StopIteration as e: + assert e.args[0] == 42 """ if PY36: @@ -1431,7 +1643,7 @@ def _fields(self): # fake names for the sake of static analysis ann_module = ann_module2 = ann_module3 = None A = B = CSub = G = CoolEmployee = CoolEmployeeWithDefault = object - XMeth = XMethBad = object + XMeth = XRepr = NoneAndForward = object gth = get_type_hints @@ -1453,9 +1665,19 @@ def test_get_type_hints_modules(self): self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) + @skipUnless(PY36, 'Python 3.6 required') + @expectedFailure + def test_get_type_hints_modules_forwardref(self): + # FIXME: This currently exposes a bug in typing. Cached forward references + # don't account for the case where there are multiple types of the same + # name coming from different modules in the same program. + mgc_hints = {'default_a': Optional[mod_generics_cache.A], + 'default_b': Optional[mod_generics_cache.B]} + self.assertEqual(gth(mod_generics_cache), mgc_hints) + @skipUnless(PY36, 'Python 3.6 required') def test_get_type_hints_classes(self): - self.assertEqual(gth(ann_module.C, ann_module.__dict__), + self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) @@ -1466,6 +1688,15 @@ def test_get_type_hints_classes(self): {'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.S), {'x': str, 'y': str}) self.assertEqual(gth(ann_module.foo), {'x': int}) + self.assertEqual(gth(NoneAndForward), + {'parent': NoneAndForward, 'meaning': type(None)}) + self.assertEqual(gth(HasForeignBaseClass), + {'some_xrepr': XRepr, 'other_a': mod_generics_cache.A, + 'some_b': mod_generics_cache.B}) + self.assertEqual(gth(mod_generics_cache.B), + {'my_inner_a1': mod_generics_cache.B.A, + 'my_inner_a2': mod_generics_cache.B.A, + 'my_outer_a': mod_generics_cache.A}) @skipUnless(PY36, 'Python 3.6 required') def test_respect_no_type_check(self): @@ -1482,17 +1713,22 @@ def meth(x: int): ... class Der(ABase): ... self.assertEqual(gth(ABase.meth), {'x': int}) - def test_get_type_hints_for_builins(self): + def test_get_type_hints_for_builtins(self): # Should not fail for built-in classes and functions. self.assertEqual(gth(int), {}) self.assertEqual(gth(type), {}) self.assertEqual(gth(dir), {}) self.assertEqual(gth(len), {}) + self.assertEqual(gth(object.__str__), {}) + self.assertEqual(gth(object().__str__), {}) + self.assertEqual(gth(str.join), {}) def test_previous_behavior(self): def testf(x, y): ... testf.__annotations__['x'] = 'int' self.assertEqual(gth(testf), {'x': int}) + def testg(x: None): ... + self.assertEqual(gth(testg), {'x': type(None)}) def test_get_type_hints_for_object_with_annotations(self): class A: ... @@ -1506,9 +1742,10 @@ def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) self.assertEqual(gth(B, globals()), - {'y': int, 'x': ClassVar[Optional[B]]}) + {'y': int, 'x': ClassVar[Optional[B]], 'b': int}) self.assertEqual(gth(CSub, globals()), - {'z': ClassVar[CSub], 'y': int, 'x': ClassVar[Optional[B]]}) + {'z': ClassVar[CSub], 'y': int, 'b': int, + 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) @@ -1628,6 +1865,11 @@ def test_list(self): def test_deque(self): self.assertIsSubclass(collections.deque, typing.Deque) + class MyDeque(typing.Deque[int]): ... + self.assertIsInstance(MyDeque(), collections.deque) + + def test_counter(self): + self.assertIsSubclass(collections.Counter, typing.Counter) def test_set(self): self.assertIsSubclass(set, typing.Set) @@ -1680,13 +1922,10 @@ class MyDict(typing.Dict[str, int]): self.assertIsSubclass(MyDict, dict) self.assertNotIsSubclass(dict, MyDict) - def test_no_defaultdict_instantiation(self): - with self.assertRaises(TypeError): - typing.DefaultDict() - with self.assertRaises(TypeError): - typing.DefaultDict[KT, VT]() - with self.assertRaises(TypeError): - typing.DefaultDict[str, int]() + def test_defaultdict_instantiation(self): + self.assertIs(type(typing.DefaultDict()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[KT, VT]()), collections.defaultdict) + self.assertIs(type(typing.DefaultDict[str, int]()), collections.defaultdict) def test_defaultdict_subclass(self): @@ -1699,13 +1938,49 @@ class MyDefDict(typing.DefaultDict[str, int]): self.assertIsSubclass(MyDefDict, collections.defaultdict) self.assertNotIsSubclass(collections.defaultdict, MyDefDict) - def test_no_deque_instantiation(self): - with self.assertRaises(TypeError): - typing.Deque() - with self.assertRaises(TypeError): - typing.Deque[T]() - with self.assertRaises(TypeError): - typing.Deque[int]() + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_instantiation(self): + self.assertIs(type(typing.ChainMap()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) + self.assertIs(type(typing.ChainMap[str, int]()), collections.ChainMap) + class CM(typing.ChainMap[KT, VT]): ... + self.assertIs(type(CM[int, str]()), CM) + + @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') + def test_chainmap_subclass(self): + + class MyChainMap(typing.ChainMap[str, int]): + pass + + cm = MyChainMap() + self.assertIsInstance(cm, MyChainMap) + + self.assertIsSubclass(MyChainMap, collections.ChainMap) + self.assertNotIsSubclass(collections.ChainMap, MyChainMap) + + def test_deque_instantiation(self): + self.assertIs(type(typing.Deque()), collections.deque) + self.assertIs(type(typing.Deque[T]()), collections.deque) + self.assertIs(type(typing.Deque[int]()), collections.deque) + class D(typing.Deque[T]): ... + self.assertIs(type(D[int]()), D) + + def test_counter_instantiation(self): + self.assertIs(type(typing.Counter()), collections.Counter) + self.assertIs(type(typing.Counter[T]()), collections.Counter) + self.assertIs(type(typing.Counter[int]()), collections.Counter) + class C(typing.Counter[T]): ... + self.assertIs(type(C[int]()), C) + + def test_counter_subclass_instantiation(self): + + class MyCounter(typing.Counter[int]): + pass + + d = MyCounter() + self.assertIsInstance(d, MyCounter) + self.assertIsInstance(d, typing.Counter) + self.assertIsInstance(d, collections.Counter) def test_no_set_instantiation(self): with self.assertRaises(TypeError): @@ -1929,8 +2204,6 @@ class B: ... class OtherABCTests(BaseTestCase): - @skipUnless(hasattr(typing, 'ContextManager'), - 'requires typing.ContextManager') def test_contextmanager(self): @contextlib.contextmanager def manager(): @@ -1940,6 +2213,24 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + @skipUnless(ASYNCIO, 'Python 3.5 required') + def test_async_contextmanager(self): + class NotACM: + pass + self.assertIsInstance(ACM(), typing.AsyncContextManager) + self.assertNotIsInstance(NotACM(), typing.AsyncContextManager) + @contextlib.contextmanager + def manager(): + yield 42 + + cm = manager() + self.assertNotIsInstance(cm, typing.AsyncContextManager) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + with self.assertRaises(TypeError): + isinstance(42, typing.AsyncContextManager[int]) + with self.assertRaises(TypeError): + typing.AsyncContextManager[int, str] + class TypeTests(BaseTestCase): @@ -2018,6 +2309,14 @@ def test_basics(self): collections.OrderedDict([('name', str), ('id', int)])) self.assertIs(Emp._field_types, Emp.__annotations__) + def test_namedtuple_pyversion(self): + if sys.version_info[:2] < (3, 6): + with self.assertRaises(TypeError): + NamedTuple('Name', one=int, other=str) + with self.assertRaises(TypeError): + class NotYet(NamedTuple): + whatever = 0 + @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) @@ -2055,10 +2354,26 @@ class NonDefaultAfterDefault(NamedTuple): @skipUnless(PY36, 'Python 3.6 required') def test_annotation_usage_with_methods(self): - self.assertEquals(XMeth(1).double(), 2) - self.assertEquals(XMeth(42).x, XMeth(42)[0]) - self.assertEquals(XMethBad(1)._fields, ('x',)) - self.assertEquals(XMethBad(1).__annotations__, {'x': int}) + self.assertEqual(XMeth(1).double(), 2) + self.assertEqual(XMeth(42).x, XMeth(42)[0]) + self.assertEqual(str(XRepr(42)), '42 -> 1') + self.assertEqual(XRepr(1, 2) + XRepr(3), 0) + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad(NamedTuple): + x: int + def _fields(self): + return 'no chance for this' +""") + + with self.assertRaises(AttributeError): + exec(""" +class XMethBad2(NamedTuple): + x: int + def _source(self): + return 'no chance for this as well' +""") @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): @@ -2138,6 +2453,12 @@ def test_basics(self): Pattern[Union[str, bytes]] Match[Union[bytes, str]] + def test_alias_equality(self): + self.assertEqual(Pattern[str], Pattern[str]) + self.assertNotEqual(Pattern[str], Pattern[bytes]) + self.assertNotEqual(Pattern[str], Match[str]) + self.assertNotEqual(Pattern[str], str) + def test_errors(self): with self.assertRaises(TypeError): # Doesn't fit AnyStr. @@ -2152,6 +2473,9 @@ def test_errors(self): with self.assertRaises(TypeError): # We don't support isinstance(). isinstance(42, Pattern[str]) + with self.assertRaises(TypeError): + # We don't support issubclass(). + issubclass(Pattern[bytes], Pattern[str]) def test_repr(self): self.assertEqual(repr(Pattern), 'Pattern[~AnyStr]') @@ -2198,6 +2522,9 @@ def test_all(self): self.assertNotIn('sys', a) # Check that Text is defined. self.assertIn('Text', a) + # Check previously missing classes. + self.assertIn('SupportsBytes', a) + self.assertIn('SupportsComplex', a) if __name__ == '__main__': diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 86ebd45e7b031d..56ab0ad7329908 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1279,6 +1279,13 @@ def __format__(self, spec): self.assertRaises(ValueError, '{}'.format_map, 'a') self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1}) + class BadMapping: + def __getitem__(self, key): + return 1/0 + self.assertRaises(KeyError, '{a}'.format_map, {}) + self.assertRaises(TypeError, '{a}'.format_map, []) + self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) + def test_format_huge_precision(self): format_string = ".{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): @@ -1448,6 +1455,15 @@ def test_formatting_huge_precision(self): with self.assertRaises(ValueError): result = format_string % 2.34 + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + @support.cpython_only def test_formatting_huge_precision_c_limits(self): from _testcapi import INT_MAX diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486e5ab479..fa3757cc94be52 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ def test_proxy_bypass_environment_host_match(self): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ def test_getproxies_environment_prefer_lowercase(self): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ def test_ftp_cache_pruning(self): finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ def test_cafile_and_context(self): "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f87162a62..876fcd4199fb92 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) @@ -830,7 +841,6 @@ def test_file(self): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -838,8 +848,7 @@ def test_file(self): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1414,7 +1423,6 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1690,7 +1698,6 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 70c4c013e9cc71..f83f9ccb786727 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -289,11 +289,15 @@ def setUp(self): def http_server_with_basic_auth_handler(*args, **kwargs): return BasicAuthHandler(*args, **kwargs) self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) - self.addCleanup(self.server.stop) + self.addCleanup(self.stop_server) self.server_url = 'http://127.0.0.1:%s' % self.server.port self.server.start() self.server.ready.wait() + def stop_server(self): + self.server.stop() + self.server = None + def tearDown(self): super(BasicAuthTests, self).tearDown() @@ -339,6 +343,7 @@ def create_fake_proxy_handler(*args, **kwargs): return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs) self.server = LoopbackHttpServerThread(create_fake_proxy_handler) + self.addCleanup(self.stop_server) self.server.start() self.server.ready.wait() proxy_url = "http://127.0.0.1:%d" % self.server.port @@ -347,9 +352,9 @@ def create_fake_proxy_handler(*args, **kwargs): self.opener = urllib.request.build_opener( handler, self.proxy_digest_handler) - def tearDown(self): + def stop_server(self): self.server.stop() - super(ProxyAuthTests, self).tearDown() + self.server = None def test_proxy_with_bad_password_raises_httperror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, @@ -468,13 +473,17 @@ def urlopen(self, url, data=None, **kwargs): f.close() return b"".join(l) + def stop_server(self): + self.server.stop() + self.server = None + def start_server(self, responses=None): if responses is None: responses = [(200, [], b"we don't care")] handler = GetRequestHandler(responses) self.server = LoopbackHttpServerThread(handler) - self.addCleanup(self.server.stop) + self.addCleanup(self.stop_server) self.server.start() self.server.ready.wait() port = self.server.port @@ -664,7 +673,7 @@ def setUpModule(): def tearDownModule(): if threads_key: - support.threading_cleanup(threads_key) + support.threading_cleanup(*threads_key) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index cad83fd5aa6a88..13e2dd5b784205 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -97,8 +97,8 @@ def setUp(self): def test_ftp(self): urls = [ - 'ftp://ftp.debian.org/debian/README', - ('ftp://ftp.debian.org/debian/non-existent-file', + 'ftp://www.pythontest.net/README', + ('ftp://www.pythontest.net/non-existent-file', None, urllib.error.URLError), ] self._test_urls(urls, self._extra_handlers()) @@ -287,7 +287,7 @@ def test_http_timeout(self): self.addCleanup(u.close) self.assertEqual(u.fp.raw._sock.gettimeout(), 120) - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c2b567e0..4103b6c07505d5 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -11,6 +11,7 @@ support.requires('network') + class URLTimeoutTest(unittest.TestCase): # XXX this test doesn't seem to test anything useful. @@ -25,11 +26,11 @@ def tearDown(self): def testURLread(self): with support.transient_internet("www.example.com"): f = urllib.request.urlopen("http://www.example.com/") - x = f.read() + f.read() class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no @@ -188,6 +189,7 @@ def test_data_header(self): def test_reporthook(self): records = [] + def recording_reporthook(blocks, block_size, total_size): records.append((blocks, block_size, total_size)) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c033e30d20..e5f6130e4a075e 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ def test_RFC2368(self): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ def test_RFC2396(self): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ def test_RFC3986(self): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. @@ -755,28 +755,35 @@ def test_default_scheme(self): def test_parse_fragments(self): # Exercise the allow_fragments parameter of urlparse() and urlsplit() tests = ( - ("http:#frag", "path"), - ("//example.net#frag", "path"), - ("index.html#frag", "path"), - (";a=b#frag", "params"), - ("?a=b#frag", "query"), - ("#frag", "path"), + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), ) - for url, attr in tests: + for url, attr, expected_frag in tests: for func in (urllib.parse.urlparse, urllib.parse.urlsplit): if attr == "params" and func is urllib.parse.urlsplit: attr = "path" with self.subTest(url=url, function=func): result = func(url, allow_fragments=False) self.assertEqual(result.fragment, "") - self.assertTrue(getattr(result, attr).endswith("#frag")) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) self.assertEqual(func(url, "", False).fragment, "") result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, "frag") - self.assertFalse(getattr(result, attr).endswith("frag")) - self.assertEqual(func(url, "", True).fragment, "frag") - self.assertEqual(func(url).fragment, "frag") + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -983,6 +990,26 @@ def test_splithost(self): self.assertEqual(splithost('/foo/bar/baz.html'), (None, '/foo/bar/baz.html')) + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + def test_splituser(self): splituser = urllib.parse.splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 0cddf4a2f4b10a..ea81d119160c8e 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -727,10 +727,15 @@ def test_showwarnmsg_missing(self): text = 'del _showwarnmsg test' with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) - del self.module._showwarnmsg - with support.captured_output('stderr') as stream: - self.module.warn(text) - result = stream.getvalue() + + show = self.module._showwarnmsg + try: + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + finally: + self.module._showwarnmsg = show self.assertIn(text, result) def test_showwarning_not_callable(self): @@ -789,6 +794,78 @@ def test_stderr_none(self): self.assertNotIn(b'Warning!', stderr) self.assertNotIn(b'Error', stderr) + def test_issue31285(self): + # warn_explicit() should neither raise a SystemError nor cause an + # assertion failure, in case the return value of get_source() has a + # bad splitlines() method. + def get_bad_loader(splitlines_ret_val): + class BadLoader: + def get_source(self, fullname): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + return BadSource('spam') + return BadLoader() + + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filterwarnings('default', category=UserWarning) + + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'foo', UserWarning, 'bar', 1, + module_globals={'__loader__': get_bad_loader(42), + '__name__': 'foobar'}) + self.assertIn('UserWarning: foo', stderr.getvalue()) + + show = wmod._showwarnmsg + try: + del wmod._showwarnmsg + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals={'__loader__': get_bad_loader([42]), + '__name__': 'foobar'}) + self.assertIn('UserWarning: eggs', stderr.getvalue()) + finally: + wmod._showwarnmsg = show + + @support.cpython_only + def test_issue31411(self): + # warn_explicit() shouldn't raise a SystemError in case + # warnings.onceregistry isn't a dictionary. + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filterwarnings('once') + with support.swap_attr(wmod, 'onceregistry', None): + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1, registry=None) + + @support.cpython_only + def test_issue31416(self): + # warn_explicit() shouldn't cause an assertion failure in case of a + # bad warnings.filters or warnings.defaultaction. + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filters = [(None, None, Warning, None, 0)] + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + wmod.filters = [] + with support.swap_attr(wmod, 'defaultaction', None), \ + self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + @support.cpython_only + def test_issue31566(self): + # warn() shouldn't cause an assertion failure in case of a bad + # __name__ global. + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings('error', category=UserWarning) + with support.swap_item(globals(), '__name__', b'foo'), \ + support.swap_item(globals(), '__file__', None): + self.assertRaises(UserWarning, self.module.warn, 'bar') + class WarningsDisplayTests(BaseTest): diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index 179e069a1ccc1d..21437ef5118cb0 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -98,6 +98,8 @@ def test_errors(self): self.assertRaises(TypeError, winsound.PlaySound, "bad", winsound.SND_MEMORY) self.assertRaises(TypeError, winsound.PlaySound, 1, 0) + # embedded null character + self.assertRaises(ValueError, winsound.PlaySound, 'bad\0', 0) def test_keyword_args(self): safe_PlaySound(flags=winsound.SND_ALIAS, sound="SystemExit") diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1cb8febf..7adfbdfd7a9e94 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -33,6 +33,7 @@ except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") SIMPLE_NS_XMLFILE = findfile("simple-ns.xml", subdir="xmltestdata") +UTF8_BUG_XMLFILE = findfile("expat224_utf8_bug.xml", subdir="xmltestdata") SAMPLE_XML = """\ @@ -1523,6 +1524,7 @@ def test_bug_xmltoolkit62(self): self.assertEqual(t.find('.//paragraph').text, 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') + @unittest.skipIf(sys.gettrace(), "Skips under coverage.") def test_bug_xmltoolkit63(self): # Check reference leak. def xmltoolkit63(): @@ -1723,6 +1725,37 @@ def __eq__(self, other): self.assertIsInstance(e[0].tag, str) self.assertEqual(e[0].tag, 'changed') + def check_expat224_utf8_bug(self, text): + xml = b'' % text + root = ET.XML(xml) + self.assertEqual(root.get('b'), text.decode('utf-8')) + + def test_expat224_utf8_bug(self): + # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. + # Check that Expat 2.2.4 fixed the bug. + # + # Test buffer bounds at odd and even positions. + + text = b'\xc3\xa0' * 1024 + self.check_expat224_utf8_bug(text) + + text = b'x' + b'\xc3\xa0' * 1024 + self.check_expat224_utf8_bug(text) + + def test_expat224_utf8_bug_file(self): + with open(UTF8_BUG_XMLFILE, 'rb') as fp: + raw = fp.read() + root = ET.fromstring(raw) + xmlattr = root.get('b') + + # "Parse" manually the XML file to extract the value of the 'b' + # attribute of the XML element + text = raw.decode('utf-8').strip() + text = text.replace('\r\n', ' ') + text = text[6:-4] + self.assertEqual(root.get('b'), text) + + # -------------------------------------------------------------------- @@ -1879,6 +1912,118 @@ def test_recursive_repr(self): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) @@ -2363,6 +2508,31 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) + def test_builder_lookup_errors(self): + class RaisingBuilder: + def __init__(self, raise_in=None, what=ValueError): + self.raise_in = raise_in + self.what = what + + def __getattr__(self, name): + if name == self.raise_in: + raise self.what(self.raise_in) + def handle(*args): + pass + return handle + + ET.XMLParser(target=RaisingBuilder()) + # cET also checks for 'close' and 'doctype', PyET does it only at need + for event in ('start', 'data', 'end', 'comment', 'pi'): + with self.assertRaisesRegex(ValueError, event): + ET.XMLParser(target=RaisingBuilder(event)) + + ET.XMLParser(target=RaisingBuilder(what=AttributeError)) + for event in ('start', 'data', 'end', 'comment', 'pi'): + parser = ET.XMLParser(target=RaisingBuilder(event, what=AttributeError)) + parser.feed(self.sample1) + self.assertIsNone(parser.close()) + class XMLParserTest(unittest.TestCase): sample1 = b'22' diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 7c60699f91262c..b2200d31f59dbf 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -64,6 +64,26 @@ def test_trashcan(self): del root support.gc_collect() + def test_parser_ref_cycle(self): + # bpo-31499: xmlparser_dealloc() crashed with a segmentation fault when + # xmlparser_gc_clear() was called previously by the garbage collector, + # when the parser was part of a reference cycle. + + def parser_ref_cycle(): + parser = cET.XMLParser() + # Create a reference cycle using an exception to keep the frame + # alive, so the parser will be destroyed by the garbage collector + try: + raise ValueError + except ValueError as exc: + err = exc + + # Create a parser part of reference cycle + parser_ref_cycle() + # Trigger an explicit garbage collection to break the reference cycle + # and so destroy the parser + support.gc_collect() + @unittest.skipUnless(cET, 'requires _elementtree') class TestAliasWorking(unittest.TestCase): diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e3df0689..f9bf304010e1c7 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -328,6 +328,10 @@ def do_POST(self): self.handled = True self.close_connection = False + def log_message(self, format, *args): + # don't clobber sys.stderr + pass + def run_server(): server.socket.settimeout(float(1)) # Don't hang if client fails server.handle_request() # First request and attempt at second @@ -343,6 +347,94 @@ def run_server(): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -667,7 +759,9 @@ def setUp(self): self.evt = threading.Event() # start server thread to handle requests serv_args = (self.evt, self.request_count, self.requestHandler) - threading.Thread(target=self.threadFunc, args=serv_args).start() + thread = threading.Thread(target=self.threadFunc, args=serv_args) + thread.start() + self.addCleanup(thread.join) # wait for the server to be ready self.evt.wait() @@ -1119,7 +1213,9 @@ def setUp(self): self.evt = threading.Event() # start server thread to handle requests serv_args = (self.evt, 1) - threading.Thread(target=http_server, args=serv_args).start() + thread = threading.Thread(target=http_server, args=serv_args) + thread.start() + self.addCleanup(thread.join) # wait for the server to be ready self.evt.wait() @@ -1312,7 +1408,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20e2bcf0d..d09ad96fc8baaa 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ def test_open(self): for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -727,6 +734,48 @@ class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, compression = zipfile.ZIP_LZMA +class AbstractWriterTests: + + def tearDown(self): + unlink(TESTFN2) + + def test_close_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + w.close() + self.assertTrue(w.closed) + self.assertEqual(zipf.read('test'), data) + + def test_write_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + self.assertRaises(ValueError, w.write, b'') + self.assertEqual(zipf.read('test'), data) + +class StoredWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib +class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2 +class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma +class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + + class PyZipFileTests(unittest.TestCase): def assertCompiledIn(self, name, namelist): if name + 'o' not in namelist: @@ -906,22 +955,56 @@ def test_write_pyfile_bad_syntax(self): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +1012,50 @@ def test_extract(self): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1295,8 @@ def test_is_zip_erroneous_file(self): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2142,26 @@ def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 7ddbc509f9bb33..daa5138751b929 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -521,6 +521,23 @@ def testGetData(self): z.close() os.remove(TEMP_ZIP) + def test_issue31291(self): + # There shouldn't be an assertion failure in get_data(). + class FunnyStr(str): + def replace(self, old, new): + return 42 + z = ZipFile(TEMP_ZIP, "w") + try: + name = "test31291.dat" + data = b'foo' + z.writestr(name, data) + z.close() + zi = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(data, zi.get_data(FunnyStr(name))) + finally: + z.close() + os.remove(TEMP_ZIP) + def testImporterAttr(self): src = """if 1: # indent hack def get_file(): diff --git a/Lib/test/xmltestdata/expat224_utf8_bug.xml b/Lib/test/xmltestdata/expat224_utf8_bug.xml new file mode 100644 index 00000000000000..d66a8e6b50f93b --- /dev/null +++ b/Lib/test/xmltestdata/expat224_utf8_bug.xml @@ -0,0 +1,2 @@ + diff --git a/Lib/threading.py b/Lib/threading.py index 4829ff426e0bfd..bb2556544b0781 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1182,8 +1182,8 @@ def run(self): self.function(*self.args, **self.kwargs) self.finished.set() + # Special thread class to represent the main thread -# This is garbage collected through an exit handler class _MainThread(Thread): @@ -1217,6 +1217,10 @@ def __init__(self): def _stop(self): pass + def is_alive(self): + assert not self._is_stopped and self._started.is_set() + return True + def join(self, timeout=None): assert False, "cannot join a dummy thread" @@ -1289,7 +1293,6 @@ def _shutdown(): while t: t.join() t = _pickSomeNonDaemonThread() - _main_thread._delete() def _pickSomeNonDaemonThread(): for t in enumerate(): diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100644 new mode 100755 index 2770efa35a0160..8eea766b4c344b --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -208,7 +208,7 @@ def repeat(self, repeat=default_repeat, number=default_number): return r def autorange(self, callback=None): - """Return the number of loops so that total time >= 0.2. + """Return the number of loops and time taken so that total time >= 0.2. Calls the timeit method with *number* set to successive powers of ten (10, 100, 1000, ...) up to a maximum of one billion, until diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index ee2415da72a01e..deea791831ed66 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1004,7 +1004,7 @@ def winfo_ismapped(self): return self.tk.getint( self.tk.call('winfo', 'ismapped', self._w)) def winfo_manager(self): - """Return the window mananger name for this widget.""" + """Return the window manager name for this widget.""" return self.tk.call('winfo', 'manager', self._w) def winfo_name(self): """Return the name of this widget.""" @@ -1679,7 +1679,7 @@ def image_names(self): return self.tk.splitlist(self.tk.call('image', 'names')) def image_types(self): - """Return a list of all available image types (e.g. phote bitmap).""" + """Return a list of all available image types (e.g. photo bitmap).""" return self.tk.splitlist(self.tk.call('image', 'types')) @@ -1818,7 +1818,7 @@ def wm_focusmodel(self, model=None): return self.tk.call('wm', 'focusmodel', self._w, model) focusmodel = wm_focusmodel def wm_forget(self, window): # new in Tk 8.5 - """The window will be unmappend from the screen and will no longer + """The window will be unmapped from the screen and will no longer be managed by wm. toplevel windows will be treated like frame windows once they are no longer managed by wm, however, the menu option configuration will be remembered and the menus will return @@ -2527,7 +2527,7 @@ def find_closest(self, x, y, halo=None, start=None): """Return item which is closest to pixel at X, Y. If several match take the top-most. All items closer than HALO are considered overlapping (all are - closests). If START is specified the next below this tag is taken.""" + closest). If START is specified the next below this tag is taken.""" return self.find('closest', x, y, halo, start) def find_enclosed(self, x1, y1, x2, y2): """Return all items in rectangle defined @@ -2587,7 +2587,7 @@ def postscript(self, cnf={}, **kw): """Print the contents of the canvas to a postscript file. Valid options: colormap, colormode, file, fontmap, height, pageanchor, pageheight, pagewidth, pagex, pagey, - rotate, witdh, x, y.""" + rotate, width, x, y.""" return self.tk.call((self._w, 'postscript') + self._options(cnf, kw)) def tag_raise(self, *args): @@ -3522,7 +3522,7 @@ def height(self): return self.tk.getint( self.tk.call('image', 'height', self.name)) def type(self): - """Return the type of the imgage, e.g. "photo" or "bitmap".""" + """Return the type of the image, e.g. "photo" or "bitmap".""" return self.tk.call('image', 'type', self.name) def width(self): """Return the width of the image.""" diff --git a/Lib/tkinter/test/test_ttk/test_extensions.py b/Lib/tkinter/test/test_ttk/test_extensions.py index 218b27fc30ea1b..a45f882bb00d48 100644 --- a/Lib/tkinter/test/test_ttk/test_extensions.py +++ b/Lib/tkinter/test/test_ttk/test_extensions.py @@ -291,6 +291,31 @@ def cb_test(item): optmenu.destroy() + def test_unique_radiobuttons(self): + # check that radiobuttons are unique across instances (bpo25684) + items = ('a', 'b', 'c') + default = 'a' + optmenu = ttk.OptionMenu(self.root, self.textvar, default, *items) + textvar2 = tkinter.StringVar(self.root) + optmenu2 = ttk.OptionMenu(self.root, textvar2, default, *items) + optmenu.pack() + optmenu.wait_visibility() + optmenu2.pack() + optmenu2.wait_visibility() + optmenu['menu'].invoke(1) + optmenu2['menu'].invoke(2) + optmenu_stringvar_name = optmenu['menu'].entrycget(0, 'variable') + optmenu2_stringvar_name = optmenu2['menu'].entrycget(0, 'variable') + self.assertNotEqual(optmenu_stringvar_name, + optmenu2_stringvar_name) + self.assertEqual(self.root.tk.globalgetvar(optmenu_stringvar_name), + items[1]) + self.assertEqual(self.root.tk.globalgetvar(optmenu2_stringvar_name), + items[2]) + + optmenu.destroy() + optmenu2.destroy() + tests_gui = (LabeledScaleTest, OptionMenuTest) diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index 26766a8a1fe75b..ab0db2878e13df 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -1556,7 +1556,7 @@ def test_selection(self): self.tv.selection_toggle((c1, c3)) self.assertEqual(self.tv.selection(), (c3, item2)) - if sys.version_info >= (3, 7): + if sys.version_info >= (3, 8): import warnings warnings.warn( 'Deprecated API of Treeview.selection() should be removed') diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c474e60713f75e..635b500579ec69 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -1404,13 +1404,13 @@ def selection(self, selop=_sentinel, items=None): import warnings warnings.warn( "The selop=None argument of selection() is deprecated " - "and will be removed in Python 3.7", + "and will be removed in Python 3.8", DeprecationWarning, 3) elif selop in ('set', 'add', 'remove', 'toggle'): import warnings warnings.warn( "The selop argument of selection() is deprecated " - "and will be removed in Python 3.7, " + "and will be removed in Python 3.8, " "use selection_%s() instead" % (selop,), DeprecationWarning, 3) else: @@ -1543,11 +1543,12 @@ def destroy(self): try: self._variable.trace_vdelete('w', self.__tracecb) except AttributeError: - # widget has been destroyed already pass else: del self._variable - Frame.destroy(self) + super().destroy() + self.label = None + self.scale = None def _adjust(self, *args): @@ -1638,7 +1639,8 @@ def set_menu(self, default=None, *values): menu.delete(0, 'end') for val in values: menu.add_radiobutton(label=val, - command=tkinter._setit(self._variable, val, self._callback)) + command=tkinter._setit(self._variable, val, self._callback), + variable=self._variable) if default: self._variable.set(default) @@ -1646,5 +1648,8 @@ def set_menu(self, default=None, *values): def destroy(self): """Destroy this widget and its associated variable.""" - del self._variable - Menubutton.destroy(self) + try: + del self._variable + except AttributeError: + pass + super().destroy() diff --git a/Lib/typing.py b/Lib/typing.py index c9e341753769e0..c00a3a10e1fae1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,6 +10,14 @@ import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +if sys.version_info[:2] >= (3, 6): + import _collections_abc # Needed for private function _check_methods # noqa +try: + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType +except ImportError: + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) # Please keep __all__ alphabetized within each category. @@ -31,6 +39,7 @@ # for 'Generic' and ABCs below. 'ByteString', 'Container', + 'ContextManager', 'Hashable', 'ItemsView', 'Iterable', @@ -51,17 +60,20 @@ # AsyncIterable, # Coroutine, # Collection, - # ContextManager, # AsyncGenerator, + # AsyncContextManager # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', 'SupportsFloat', 'SupportsInt', 'SupportsRound', # Concrete collection types. + 'Counter', 'Deque', 'Dict', 'DefaultDict', @@ -364,7 +376,7 @@ def _type_check(arg, msg): if ( type(arg).__name__ in ('_Union', '_Optional') and not getattr(arg, '__origin__', None) or - isinstance(arg, TypingMeta) and _gorg(arg) in (Generic, _Protocol) + isinstance(arg, TypingMeta) and arg._gorg in (Generic, _Protocol) ): raise TypeError("Plain %s is not valid as type argument" % arg) return arg @@ -413,6 +425,31 @@ def __subclasscheck__(self, cls): Any = _Any(_root=True) +class _NoReturn(_FinalTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("NoReturn cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("NoReturn cannot be used with issubclass().") + + +NoReturn = _NoReturn(_root=True) + + class TypeVar(_TypingBase, _root=True): """Type variable. @@ -812,29 +849,6 @@ def __getitem__(self, arg): Optional = _Optional(_root=True) -def _gorg(a): - """Return the farthest origin of a generic class (internal helper).""" - assert isinstance(a, GenericMeta) - while a.__origin__ is not None: - a = a.__origin__ - return a - - -def _geqv(a, b): - """Return whether two generic classes are equivalent (internal helper). - - The intention is to consider generic class X and any of its - parameterized forms (X[T], X[int], etc.) as equivalent. - - However, X is not equivalent to a subclass of X. - - The relation is reflexive, symmetric and transitive. - """ - assert isinstance(a, GenericMeta) and isinstance(b, GenericMeta) - # Reduce each to its origin. - return _gorg(a) is _gorg(b) - - def _next_in_mro(cls): """Helper for Generic.__new__. @@ -844,24 +858,11 @@ def _next_in_mro(cls): next_in_mro = object # Look for the last occurrence of Generic or Generic[...]. for i, c in enumerate(cls.__mro__[:-1]): - if isinstance(c, GenericMeta) and _gorg(c) is Generic: + if isinstance(c, GenericMeta) and c._gorg is Generic: next_in_mro = cls.__mro__[i + 1] return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -872,7 +873,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +887,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -969,21 +968,21 @@ def __new__(cls, name, bases, namespace, initial_bases = bases if extra is not None and type(extra) is abc.ABCMeta and extra not in bases: bases = (extra,) + bases - bases = tuple(_gorg(b) if isinstance(b, GenericMeta) else b for b in bases) + bases = tuple(b._gorg if isinstance(b, GenericMeta) else b for b in bases) # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) - + super(GenericMeta, self).__setattr__('_gorg', + self if not origin else origin._gorg) self.__parameters__ = tvars # Be prepared that GenericMeta will be subclassed by TupleMeta # and CallableMeta, those two allow ..., (), or [] in __args___. self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -994,20 +993,56 @@ def __new__(cls, name, bases, namespace, # with issubclass() and isinstance() in the same way as their # collections.abc counterparts (e.g., isinstance([], Iterable)). if ( - # allow overriding '__subclasshook__' not in namespace and extra or - hasattr(self.__subclasshook__, '__name__') and - self.__subclasshook__.__name__ == '__extrahook__' + # allow overriding + getattr(self.__subclasshook__, '__name__', '') == '__extrahook__' ): self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ - self.__tree_hash__ = hash(self._subs_tree()) if origin else hash((self.__name__,)) + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return self._gorg._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return self._gorg._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1048,7 +1083,7 @@ def _subs_tree(self, tvars=None, args=None): if self.__origin__ is None: return self tree_args = _subs_tree(self, tvars, args) - return (_gorg(self),) + tuple(tree_args) + return (self._gorg,) + tuple(tree_args) def __eq__(self, other): if not isinstance(other, GenericMeta): @@ -1064,7 +1099,7 @@ def __hash__(self): def __getitem__(self, params): if not isinstance(params, tuple): params = (params,) - if not params and not _gorg(self) is Tuple: + if not params and self._gorg is not Tuple: raise TypeError( "Parameter list to %s[...] cannot be empty" % _qualname(self)) msg = "Parameters to generic types must be types." @@ -1095,8 +1130,10 @@ def __getitem__(self, params): _check_generic(self, params) tvars = _type_vars(params) args = params + + prepend = (self,) if self.__origin__ is None else () return self.__class__(self.__name__, - self.__bases__, + prepend + self.__bases__, _no_slots_copy(self.__dict__), tvars=tvars, args=args, @@ -1104,6 +1141,17 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the @@ -1118,6 +1166,16 @@ def __copy__(self): self.__parameters__, self.__args__, self.__origin__, self.__extra__, self.__orig_bases__) + def __setattr__(self, attr, value): + # We consider all the subscripted generics as proxies for original class + if ( + attr.startswith('__') and attr.endswith('__') or + attr.startswith('_abc_') + ): + super(GenericMeta, self).__setattr__(attr, value) + else: + super(GenericMeta, self._gorg).__setattr__(attr, value) + # Prevent checks for Generic to crash when defining Generic. Generic = None @@ -1129,7 +1187,7 @@ def _generic_new(base_cls, cls, *args, **kwds): if cls.__origin__ is None: return base_cls.__new__(cls) else: - origin = _gorg(cls) + origin = cls._gorg obj = base_cls.__new__(origin) try: obj.__orig_class__ = cls @@ -1163,7 +1221,7 @@ def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Generic): + if cls._gorg is Generic: raise TypeError("Type Generic cannot be instantiated; " "it can be used only as a base class") return _generic_new(cls.__next_in_mro__, cls, *args, **kwds) @@ -1185,7 +1243,7 @@ class TupleMeta(GenericMeta): @_tp_cache def __getitem__(self, parameters): - if self.__origin__ is not None or not _geqv(self, Tuple): + if self.__origin__ is not None or self._gorg is not Tuple: # Normal generic rules apply if this is not the first subscription # or a subscription of a subclass. return super().__getitem__(parameters) @@ -1227,7 +1285,7 @@ class Tuple(tuple, extra=tuple, metaclass=TupleMeta): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Tuple): + if cls._gorg is Tuple: raise TypeError("Type Tuple cannot be instantiated; " "use tuple() instead") return _generic_new(tuple, cls, *args, **kwds) @@ -1242,7 +1300,7 @@ def __repr__(self): return self._tree_repr(self._subs_tree()) def _tree_repr(self, tree): - if _gorg(self) is not Callable: + if self._gorg is not Callable: return super()._tree_repr(tree) # For actual Callable (not its subclass) we override # super()._tree_repr() for nice formatting. @@ -1262,7 +1320,7 @@ def __getitem__(self, parameters): with hashable arguments to improve speed. """ - if self.__origin__ is not None or not _geqv(self, Callable): + if self.__origin__ is not None or self._gorg is not Callable: return super().__getitem__(parameters) if not isinstance(parameters, tuple) or len(parameters) != 2: raise TypeError("Callable must be used as " @@ -1304,7 +1362,7 @@ class Callable(extra=collections_abc.Callable, metaclass=CallableMeta): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Callable): + if cls._gorg is Callable: raise TypeError("Type Callable cannot be instantiated; " "use a non-abstract subclass instead") return _generic_new(cls.__next_in_mro__, cls, *args, **kwds) @@ -1398,6 +1456,11 @@ def _get_defaults(func): return res +_allowed_types = (types.FunctionType, types.BuiltinFunctionType, + types.MethodType, types.ModuleType, + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) + + def get_type_hints(obj, globalns=None, localns=None): """Return type hints for an object. @@ -1418,8 +1481,9 @@ def get_type_hints(obj, globalns=None, localns=None): search order is locals first, then globals. - If no dict arguments are passed, an attempt is made to use the - globals from obj, and these are also used as the locals. If the - object does not appear to have globals, an exception is raised. + globals from obj (or the respective module's globals for classes), + and these are also used as the locals. If the object does not appear + to have globals, an empty dictionary is used. - If one dict argument is passed, it is used for both globals and locals. @@ -1430,34 +1494,37 @@ def get_type_hints(obj, globalns=None, localns=None): if getattr(obj, '__no_type_check__', None): return {} - if globalns is None: - globalns = getattr(obj, '__globals__', {}) - if localns is None: - localns = globalns - elif localns is None: - localns = globalns # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + if globalns is None: + base_globals = sys.modules[base.__module__].__dict__ + else: + base_globals = globalns ann = base.__dict__.get('__annotations__', {}) for name, value in ann.items(): if value is None: value = type(None) if isinstance(value, str): value = _ForwardRef(value) - value = _eval_type(value, globalns, localns) + value = _eval_type(value, base_globals, localns) hints[name] = value return hints + + if globalns is None: + if isinstance(obj, types.ModuleType): + globalns = obj.__dict__ + else: + globalns = getattr(obj, '__globals__', {}) + if localns is None: + localns = globalns + elif localns is None: + localns = globalns hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. - if ( - isinstance(obj, types.FunctionType) or - isinstance(obj, types.BuiltinFunctionType) or - isinstance(obj, types.MethodType) or - isinstance(obj, types.ModuleType) - ): + if isinstance(obj, _allowed_types): return {} else: raise TypeError('{!r} is not a module, class, method, ' @@ -1488,7 +1555,7 @@ def no_type_check(arg): if isinstance(arg, type): arg_attrs = arg.__dict__.copy() for attr, val in arg.__dict__.items(): - if val in arg.__bases__: + if val in arg.__bases__ + (arg,): arg_attrs.pop(attr) for obj in arg_attrs.values(): if isinstance(obj, types.FunctionType): @@ -1607,6 +1674,7 @@ def _get_protocol_attrs(self): attr != '__annotations__' and attr != '__weakref__' and attr != '_is_protocol' and + attr != '_gorg' and attr != '__dict__' and attr != '__args__' and attr != '__slots__' and @@ -1812,7 +1880,7 @@ class List(list, MutableSequence[T], extra=list): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, List): + if cls._gorg is List: raise TypeError("Type List cannot be instantiated; " "use list() instead") return _generic_new(list, cls, *args, **kwds) @@ -1823,9 +1891,8 @@ class Deque(collections.deque, MutableSequence[T], extra=collections.deque): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Deque): - raise TypeError("Type Deque cannot be instantiated; " - "use deque() instead") + if cls._gorg is Deque: + return collections.deque(*args, **kwds) return _generic_new(collections.deque, cls, *args, **kwds) @@ -1834,7 +1901,7 @@ class Set(set, MutableSet[T], extra=set): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Set): + if cls._gorg is Set: raise TypeError("Type Set cannot be instantiated; " "use set() instead") return _generic_new(set, cls, *args, **kwds) @@ -1844,7 +1911,7 @@ class FrozenSet(frozenset, AbstractSet[T_co], extra=frozenset): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, FrozenSet): + if cls._gorg is FrozenSet: raise TypeError("Type FrozenSet cannot be instantiated; " "use frozenset() instead") return _generic_new(frozenset, cls, *args, **kwds) @@ -1873,7 +1940,61 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): if hasattr(contextlib, 'AbstractContextManager'): class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): __slots__ = () - __all__.append('ContextManager') +else: + class ContextManager(Generic[T_co]): + __slots__ = () + + def __enter__(self): + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is ContextManager: + # In Python 3.6+, it is possible to set a method to None to + # explicitly indicate that the class does not implement an ABC + # (https://bugs.python.org/issue25958), but we do not support + # that pattern here because this fallback class is only used + # in Python 3.5 and earlier. + if (any("__enter__" in B.__dict__ for B in C.__mro__) and + any("__exit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + + +if hasattr(contextlib, 'AbstractAsyncContextManager'): + class AsyncContextManager(Generic[T_co], + extra=contextlib.AbstractAsyncContextManager): + __slots__ = () + + __all__.append('AsyncContextManager') +elif sys.version_info[:2] >= (3, 5): + exec(""" +class AsyncContextManager(Generic[T_co]): + __slots__ = () + + async def __aenter__(self): + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncContextManager: + if sys.version_info[:2] >= (3, 6): + return _collections_abc._check_methods(C, "__aenter__", "__aexit__") + if (any("__aenter__" in B.__dict__ for B in C.__mro__) and + any("__aexit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + +__all__.append('AsyncContextManager') +""") class Dict(dict, MutableMapping[KT, VT], extra=dict): @@ -1881,7 +2002,7 @@ class Dict(dict, MutableMapping[KT, VT], extra=dict): __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Dict): + if cls._gorg is Dict: raise TypeError("Type Dict cannot be instantiated; " "use dict() instead") return _generic_new(dict, cls, *args, **kwds) @@ -1893,12 +2014,36 @@ class DefaultDict(collections.defaultdict, MutableMapping[KT, VT], __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, DefaultDict): - raise TypeError("Type DefaultDict cannot be instantiated; " - "use collections.defaultdict() instead") + if cls._gorg is DefaultDict: + return collections.defaultdict(*args, **kwds) return _generic_new(collections.defaultdict, cls, *args, **kwds) +class Counter(collections.Counter, Dict[T, int], extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is Counter: + return collections.Counter(*args, **kwds) + return _generic_new(collections.Counter, cls, *args, **kwds) + + +if hasattr(collections, 'ChainMap'): + # ChainMap only exists in 3.3+ + __all__.append('ChainMap') + + class ChainMap(collections.ChainMap, MutableMapping[KT, VT], + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is ChainMap: + return collections.ChainMap(*args, **kwds) + return _generic_new(collections.ChainMap, cls, *args, **kwds) + + # Determine what base class to use for Generator. if hasattr(collections_abc, 'Generator'): # Sufficiently recent versions of 3.5 have a Generator ABC. @@ -1913,7 +2058,7 @@ class Generator(Iterator[T_co], Generic[T_co, T_contra, V_co], __slots__ = () def __new__(cls, *args, **kwds): - if _geqv(cls, Generator): + if cls._gorg is Generator: raise TypeError("Type Generator cannot be instantiated; " "create a subclass instead") return _generic_new(_G_base, cls, *args, **kwds) @@ -1975,6 +2120,13 @@ def _make_nmtuple(name, types): _PY36 = sys.version_info[:2] >= (3, 6) +# attributes prohibited to set in NamedTuple class syntax +_prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', + '_fields', '_field_defaults', '_field_types', + '_make', '_replace', '_asdict', '_source') + +_special = ('__module__', '__name__', '__qualname__', '__annotations__') + class NamedTupleMeta(type): @@ -2002,7 +2154,9 @@ def __new__(cls, typename, bases, ns): nm_tpl._field_defaults = defaults_dict # update from user namespace without overriding special namedtuple attributes for key in ns: - if not hasattr(nm_tpl, key): + if key in _prohibited: + raise AttributeError("Cannot overwrite NamedTuple attribute " + key) + elif key not in _special and key not in nm_tpl._fields: setattr(nm_tpl, key, ns[key]) return nm_tpl diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index b523f73ff22784..f4dbc52852dc13 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -153,28 +153,32 @@ def handle(self, name, args, kwargs): If args is not empty, call a callable passing positional and keyword arguments. """ - if not _is_subtype(self.expected, self._base_type): - raise TypeError('%s() arg 1 must be %s' % - (name, self._base_type_str)) - if args and args[0] is None: - warnings.warn("callable is None", - DeprecationWarning, 3) - args = () - if not args: - self.msg = kwargs.pop('msg', None) - if kwargs: - warnings.warn('%r is an invalid keyword argument for ' - 'this function' % next(iter(kwargs)), - DeprecationWarning, 3) - return self - - callable_obj, *args = args try: - self.obj_name = callable_obj.__name__ - except AttributeError: - self.obj_name = str(callable_obj) - with self: - callable_obj(*args, **kwargs) + if not _is_subtype(self.expected, self._base_type): + raise TypeError('%s() arg 1 must be %s' % + (name, self._base_type_str)) + if args and args[0] is None: + warnings.warn("callable is None", + DeprecationWarning, 3) + args = () + if not args: + self.msg = kwargs.pop('msg', None) + if kwargs: + warnings.warn('%r is an invalid keyword argument for ' + 'this function' % next(iter(kwargs)), + DeprecationWarning, 3) + return self + + callable_obj, *args = args + try: + self.obj_name = callable_obj.__name__ + except AttributeError: + self.obj_name = str(callable_obj) + with self: + callable_obj(*args, **kwargs) + finally: + # bpo-23890: manually break a reference cycle + self = None class _AssertRaisesContext(_AssertRaisesBaseContext): @@ -725,7 +729,11 @@ def assertRaises(self, expected_exception, *args, **kwargs): self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(expected_exception, self) - return context.handle('assertRaises', args, kwargs) + try: + return context.handle('assertRaises', args, kwargs) + finally: + # bpo-23890: manually break a reference cycle + context = None def assertWarns(self, expected_warning, *args, **kwargs): """Fail unless a warning of class warnClass is triggered diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 09fefe11649bd3..807604f08dfd14 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -14,6 +14,7 @@ %(prog)s test_module - run tests from test_module %(prog)s module.TestClass - run tests from module.TestClass %(prog)s module.Class.test_method - run specified test method + %(prog)s path/to/test_file.py - run tests from test_file.py """ MODULE_EXAMPLES = """\ diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6b38362341f16..6be7bb4f065e81 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -165,7 +165,7 @@ def _set_signature(mock, original, instance=False): skipfirst = isinstance(original, type) result = _get_signature_object(original, instance, skipfirst) if result is None: - return + return mock func, sig = result def checksig(*args, **kwargs): sig.bind(*args, **kwargs) @@ -815,8 +815,8 @@ def _error_message(): def assert_called_once_with(_mock_self, *args, **kwargs): - """assert that the mock was called exactly once and with the specified - arguments.""" + """assert that the mock was called exactly once and that that call was + with the specified arguments.""" self = _mock_self if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times." % diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index 8f752b8ae0d0e2..b84959150542bd 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1273,6 +1273,19 @@ def testAssertRaisesNoExceptionType(self): with self.assertRaises(TypeError): self.assertRaises((ValueError, object)) + def testAssertRaisesRefcount(self): + # bpo-23890: assertRaises() must not keep objects alive longer + # than expected + def func() : + try: + raise ValueError + except ValueError: + raise ValueError + + refcount = sys.getrefcount(func) + self.assertRaises(ValueError, func) + self.assertEqual(refcount, sys.getrefcount(func)) + def testAssertRaisesRegex(self): class ExceptionMock(Exception): pass diff --git a/Lib/unittest/test/test_discovery.py b/Lib/unittest/test/test_discovery.py index 1996a8ee5d3680..48d8fe95bce261 100644 --- a/Lib/unittest/test/test_discovery.py +++ b/Lib/unittest/test/test_discovery.py @@ -528,6 +528,9 @@ def _get_module_from_name(name): pickle.loads(pickle.dumps(test, proto)) def test_discover_with_module_that_raises_SkipTest_on_import(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + loader = unittest.TestLoader() def _get_module_from_name(name): @@ -548,6 +551,9 @@ def _get_module_from_name(name): pickle.loads(pickle.dumps(suite, proto)) def test_discover_with_init_module_that_raises_SkipTest_on_import(self): + if not unittest.BaseTestSuite._cleanup: + raise unittest.SkipTest("Suite cleanup is disabled") + vfs = {abspath('/foo'): ['my_package'], abspath('/foo/my_package'): ['__init__.py', 'test_module.py']} self.setup_import_issue_package_tests(vfs) diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/unittest/test/testmock/testhelpers.py index d5f9e7c1d69204..7919482ae99c7f 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/unittest/test/testmock/testhelpers.py @@ -1,3 +1,5 @@ +import time +import types import unittest from unittest.mock import ( @@ -307,18 +309,9 @@ def test_two_args_call(self): self.assertEqual(args, other_args) def test_call_with_name(self): - self.assertEqual( - 'foo', - _Call((), 'foo')[0], - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), ), )[0] - ) - self.assertEqual( - '', - _Call((('bar', 'barz'), {'hello': 'world'}), )[0] - ) + self.assertEqual(_Call((), 'foo')[0], 'foo') + self.assertEqual(_Call((('bar', 'barz'),),)[0], '') + self.assertEqual(_Call((('bar', 'barz'), {'hello': 'world'}),)[0], '') class SpecSignatureTest(unittest.TestCase): @@ -865,6 +858,19 @@ def check_data_descriptor(mock_attr): check_data_descriptor(foo.desc) + def test_autospec_on_bound_builtin_function(self): + meth = types.MethodType(time.ctime, time.time()) + self.assertIsInstance(meth(), str) + mocked = create_autospec(meth) + + # no signature, so no spec to check against + mocked() + mocked.assert_called_once_with() + mocked.reset_mock() + mocked(4, 5, 6) + mocked.assert_called_once_with(4, 5, 6) + + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d16188b8..851515bc307d62 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a08d7e77..3cab2d13d5fb56 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -38,29 +38,37 @@ "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' @@ -612,6 +620,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +642,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +655,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] @@ -932,7 +944,7 @@ def splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3d0e4bf5..96921440edea11 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ def open_local_file(self, req): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1658,14 +1657,10 @@ def pathname2url(pathname): of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index c78a4efa153fc4..fff0765af53b82 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -3,13 +3,13 @@ deactivate () { # reset old environment variables - if [ -n "$_OLD_VIRTUAL_PATH" ] ; then - PATH="$_OLD_VIRTUAL_PATH" + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" export PATH unset _OLD_VIRTUAL_PATH fi - if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then - PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" export PYTHONHOME unset _OLD_VIRTUAL_PYTHONHOME fi @@ -17,12 +17,12 @@ deactivate () { # This should detect bash and zsh, which have a hash command that must # be called to get it to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected - if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then hash -r fi - if [ -n "$_OLD_VIRTUAL_PS1" ] ; then - PS1="$_OLD_VIRTUAL_PS1" + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" export PS1 unset _OLD_VIRTUAL_PS1 fi @@ -47,15 +47,15 @@ export PATH # unset PYTHONHOME if set # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) # could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "$PYTHONHOME" ] ; then - _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" unset PYTHONHOME fi -if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then - _OLD_VIRTUAL_PS1="$PS1" +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" if [ "x__VENV_PROMPT__" != x ] ; then - PS1="__VENV_PROMPT__$PS1" + PS1="__VENV_PROMPT__${PS1:-}" else if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then # special case for Aspen magic directories @@ -71,6 +71,6 @@ fi # This should detect bash and zsh, which have a hash command that must # be called to get it to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then hash -r fi diff --git a/Lib/venv/scripts/nt/Activate.ps1 b/Lib/venv/scripts/nt/Activate.ps1 index 85646c89a81cb3..bf60869e552e97 100644 --- a/Lib/venv/scripts/nt/Activate.ps1 +++ b/Lib/venv/scripts/nt/Activate.ps1 @@ -1,51 +1,51 @@ -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - if (Test-Path function:_OLD_VIRTUAL_PROMPT) { - copy-item function:_OLD_VIRTUAL_PROMPT function:prompt - remove-item function:_OLD_VIRTUAL_PROMPT - } - - if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) { - copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME - remove-item env:_OLD_VIRTUAL_PYTHONHOME - } - - if (Test-Path env:_OLD_VIRTUAL_PATH) { - copy-item env:_OLD_VIRTUAL_PATH env:PATH - remove-item env:_OLD_VIRTUAL_PATH - } - - if (Test-Path env:VIRTUAL_ENV) { - remove-item env:VIRTUAL_ENV - } - - if (!$NonDestructive) { - # Self destruct! - remove-item function:deactivate - } -} - -deactivate -nondestructive - -$env:VIRTUAL_ENV="__VENV_DIR__" - -if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) { - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT {""} - copy-item function:prompt function:_OLD_VIRTUAL_PROMPT - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green '__VENV_PROMPT__' - _OLD_VIRTUAL_PROMPT - } -} - -# Clear PYTHONHOME -if (Test-Path env:PYTHONHOME) { - copy-item env:PYTHONHOME env:_OLD_VIRTUAL_PYTHONHOME - remove-item env:PYTHONHOME -} - -# Add the venv to the PATH -copy-item env:PATH env:_OLD_VIRTUAL_PATH -$env:PATH = "$env:VIRTUAL_ENV\__VENV_BIN_NAME__;$env:PATH" +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + if (Test-Path function:_OLD_VIRTUAL_PROMPT) { + copy-item function:_OLD_VIRTUAL_PROMPT function:prompt + remove-item function:_OLD_VIRTUAL_PROMPT + } + + if (Test-Path env:_OLD_VIRTUAL_PYTHONHOME) { + copy-item env:_OLD_VIRTUAL_PYTHONHOME env:PYTHONHOME + remove-item env:_OLD_VIRTUAL_PYTHONHOME + } + + if (Test-Path env:_OLD_VIRTUAL_PATH) { + copy-item env:_OLD_VIRTUAL_PATH env:PATH + remove-item env:_OLD_VIRTUAL_PATH + } + + if (Test-Path env:VIRTUAL_ENV) { + remove-item env:VIRTUAL_ENV + } + + if (!$NonDestructive) { + # Self destruct! + remove-item function:deactivate + } +} + +deactivate -nondestructive + +$env:VIRTUAL_ENV="__VENV_DIR__" + +if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) { + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT {""} + copy-item function:prompt function:_OLD_VIRTUAL_PROMPT + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green '__VENV_PROMPT__' + _OLD_VIRTUAL_PROMPT + } +} + +# Clear PYTHONHOME +if (Test-Path env:PYTHONHOME) { + copy-item env:PYTHONHOME env:_OLD_VIRTUAL_PYTHONHOME + remove-item env:PYTHONHOME +} + +# Add the venv to the PATH +copy-item env:PATH env:_OLD_VIRTUAL_PATH +$env:PATH = "$env:VIRTUAL_ENV\__VENV_BIN_NAME__;$env:PATH" diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 9eab147f314d3b..d76ca1359627ee 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -1,32 +1,32 @@ -@echo off -set "VIRTUAL_ENV=__VENV_DIR__" - -if not defined PROMPT ( - set "PROMPT=$P$G" -) - -if defined _OLD_VIRTUAL_PROMPT ( - set "PROMPT=%_OLD_VIRTUAL_PROMPT%" -) - -if defined _OLD_VIRTUAL_PYTHONHOME ( - set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" -) - -set "_OLD_VIRTUAL_PROMPT=%PROMPT%" -set "PROMPT=__VENV_PROMPT__%PROMPT%" - -if defined PYTHONHOME ( - set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%" - set PYTHONHOME= -) - -if defined _OLD_VIRTUAL_PATH ( - set "PATH=%_OLD_VIRTUAL_PATH%" -) else ( - set "_OLD_VIRTUAL_PATH=%PATH%" -) - -set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" - -:END +@echo off +set "VIRTUAL_ENV=__VENV_DIR__" + +if not defined PROMPT ( + set "PROMPT=$P$G" +) + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" +) + +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=__VENV_PROMPT__%PROMPT%" + +if defined PYTHONHOME ( + set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%" + set PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) else ( + set "_OLD_VIRTUAL_PATH=%PATH%" +) + +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" + +:END diff --git a/Lib/venv/scripts/nt/deactivate.bat b/Lib/venv/scripts/nt/deactivate.bat index 313c0791173682..1205c618686fbb 100644 --- a/Lib/venv/scripts/nt/deactivate.bat +++ b/Lib/venv/scripts/nt/deactivate.bat @@ -1,21 +1,21 @@ -@echo off - -if defined _OLD_VIRTUAL_PROMPT ( - set "PROMPT=%_OLD_VIRTUAL_PROMPT%" -) -set _OLD_VIRTUAL_PROMPT= - -if defined _OLD_VIRTUAL_PYTHONHOME ( - set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" - set _OLD_VIRTUAL_PYTHONHOME= -) - -if defined _OLD_VIRTUAL_PATH ( - set "PATH=%_OLD_VIRTUAL_PATH%" -) - -set _OLD_VIRTUAL_PATH= - -set VIRTUAL_ENV= - -:END +@echo off + +if defined _OLD_VIRTUAL_PROMPT ( + set "PROMPT=%_OLD_VIRTUAL_PROMPT%" +) +set _OLD_VIRTUAL_PROMPT= + +if defined _OLD_VIRTUAL_PYTHONHOME ( + set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%" + set _OLD_VIRTUAL_PYTHONHOME= +) + +if defined _OLD_VIRTUAL_PATH ( + set "PATH=%_OLD_VIRTUAL_PATH%" +) + +set _OLD_VIRTUAL_PATH= + +set VIRTUAL_ENV= + +:END diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 98b5ca953983c7..421358fa5bc7f0 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -105,9 +105,16 @@ def parse(self, source): source = saxutils.prepare_input_source(source) self._source = source - self.reset() - self._cont_handler.setDocumentLocator(ExpatLocator(self)) - xmlreader.IncrementalParser.parse(self, source) + try: + self.reset() + self._cont_handler.setDocumentLocator(ExpatLocator(self)) + xmlreader.IncrementalParser.parse(self, source) + except: + # bpo-30264: Close the source on error to not leak resources: + # xml.sax.parse() doesn't give access to the underlying parser + # to the caller + self._close_source() + raise def prepareParser(self, source): if source.getSystemId() is not None: @@ -213,6 +220,17 @@ def feed(self, data, isFinal = 0): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def _close_source(self): + source = self._source + try: + file = source.getCharacterStream() + if file is not None: + file.close() + finally: + file = source.getByteStream() + if file is not None: + file.close() + def close(self): if (self._entity_stack or self._parser is None or isinstance(self._parser, _ClosedParser)): @@ -232,14 +250,7 @@ def close(self): parser.ErrorColumnNumber = self._parser.ErrorColumnNumber parser.ErrorLineNumber = self._parser.ErrorLineNumber self._parser = parser - try: - file = self._source.getCharacterStream() - if file is not None: - file.close() - finally: - file = self._source.getByteStream() - if file is not None: - file.close() + self._close_source() def _reset_cont_handler(self): self._parser.ProcessingInstructionHandler = \ diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfddd84816f..3e0dca027f0686 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -264,10 +264,14 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): except: # report exception back to server exc_type, exc_value, exc_tb = sys.exc_info() - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none, - ) + try: + response = dumps( + Fault(1, "%s:%s" % (exc_type, exc_value)), + encoding=self.encoding, allow_none=self.allow_none, + ) + finally: + # Break reference cycle + exc_type = exc_value = exc_tb = None return response.encode(self.encoding, 'xmlcharrefreplace') @@ -359,10 +363,14 @@ def system_multicall(self, call_list): ) except: exc_type, exc_value, exc_tb = sys.exc_info() - results.append( - {'faultCode' : 1, - 'faultString' : "%s:%s" % (exc_type, exc_value)} - ) + try: + results.append( + {'faultCode' : 1, + 'faultString' : "%s:%s" % (exc_type, exc_value)} + ) + finally: + # Break reference cycle + exc_type = exc_value = exc_tb = None return results def _dispatch(self, method, params): @@ -386,31 +394,36 @@ def _dispatch(self, method, params): not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. @@ -619,10 +632,14 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): # (each dispatcher should have handled their own # exceptions) exc_type, exc_value = sys.exc_info()[:2] - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none) - response = response.encode(self.encoding, 'xmlcharrefreplace') + try: + response = dumps( + Fault(1, "%s:%s" % (exc_type, exc_value)), + encoding=self.encoding, allow_none=self.allow_none) + response = response.encode(self.encoding, 'xmlcharrefreplace') + finally: + # Break reference cycle + exc_type = exc_value = None return response class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher): diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43ce1cc848..9164f8ab086a16 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ def from_file(cls, filename, arcname=None): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -979,6 +981,8 @@ def writable(self): return True def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) @@ -989,6 +993,8 @@ def write(self, data): return nbytes def close(self): + if self.closed: + return super().close() # Flush any data from the compressor, and update header info if self._compressor: @@ -1070,6 +1076,8 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1102,7 +1110,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self._start_disk = 0 try: self.start_dir = self.fp.tell() except (AttributeError, OSError): @@ -1128,7 +1135,7 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self.start_dir = self._start_disk = self.fp.tell() + self.start_dir = self.fp.tell() else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1172,18 +1179,17 @@ def _RealGetContents(self): offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd if endrec[_ECD_SIGNATURE] == stringEndArchive64: # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: - inferred = self._start_disk + offset_cd - print("given, inferred, offset", offset_cd, inferred, self._start_disk) + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk + self.start_dir = offset_cd + concat fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) @@ -1223,7 +1229,7 @@ def _RealGetContents(self): t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk + x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1470,11 +1476,10 @@ def extract(self, member, path=None, pwd=None): as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1492,13 @@ def extractall(self, path=None, members=None, pwd=None): if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1519,9 @@ def _extract_member(self, member, targetpath, pwd): """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1687,10 +1700,11 @@ def _write_end_record(self): file_size = zinfo.file_size compress_size = zinfo.compress_size - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset extra_data = zinfo.extra min_version = 0 @@ -1737,7 +1751,7 @@ def _write_end_record(self): # Write end-of-zip-archive record centDirCount = len(self.filelist) centDirSize = pos2 - self.start_dir - centDirOffset = self.start_dir - self._start_disk + centDirOffset = self.start_dir requires_zip64 = None if centDirCount > ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" @@ -1801,6 +1815,7 @@ def writepy(self, pathname, basename="", filterfunc=None): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 8dfd092d08b512..4cb7e3214cd40a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of hg and a third-party version of +requires an installed third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), hg, and for the documentation +(2.5 is needed for Python parser updates) and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2j", - url="https://www.openssl.org/source/openssl-1.0.2j.tar.gz", - checksum='96322138f0b69e61b7212bc53d5e912b', + name="OpenSSL 1.0.2k", + url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", + checksum='f965fc0bf01bf882b31314b61391ae65', patches=[ "openssl_sdk_makedepend.patch", ], @@ -635,9 +635,8 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to hg and to sphinx-build. - # You may have to create links in /usr/bin for them. - runCommand('hg --version') + # Ensure we have access to sphinx-build. + # You may have to create a link in /usr/bin for it. runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1089,10 +1088,10 @@ def buildPythonDocs(): docdir = os.path.join(rootDir, 'pydocs') curDir = os.getcwd() os.chdir(buildDir) - # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9 runCommand('make clean') - # Assume sphinx-build is on our PATH, checked in checkEnvironment - runCommand('make html') + # Create virtual environment for docs builds with blurb and sphinx + runCommand('make venv') + runCommand('make html PYTHON=venv/bin/python') os.chdir(curDir) if not os.path.exists(docdir): os.mkdir(docdir) @@ -1142,11 +1141,25 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - print("Running make touch") - runCommand("make touch") - - print("Running make") - runCommand("make") + # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS + # and, if defined, append its value to the make command. This allows + # us to pass in version control tags, like GITTAG, to a build from a + # tarball rather than from a vcs checkout, thus eliminating the need + # to have a working copy of the vcs program on the build machine. + # + # A typical use might be: + # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ + # GITVERSION='echo 123456789a' \ + # GITTAG='echo v3.6.0' \ + # GITBRANCH='echo 3.6'" + + make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") + if make_extras: + make_cmd = "make " + make_extras + else: + make_cmd = "make" + print("Running " + make_cmd) + runCommand(make_cmd) print("Running make install") runCommand("make install DESTDIR=%s"%( diff --git a/Mac/BuildScript/openssl_sdk_makedepend.patch b/Mac/BuildScript/openssl_sdk_makedepend.patch index e22d67e4e45143..0caac0a64c1ec6 100644 --- a/Mac/BuildScript/openssl_sdk_makedepend.patch +++ b/Mac/BuildScript/openssl_sdk_makedepend.patch @@ -1,6 +1,6 @@ # HG changeset patch # -# using openssl 1.0.2j +# using openssl 1.0.2k # # - support building with an OS X SDK diff --git a/Mac/README b/Mac/README index 07f09fa8366528..d4194747ebb05c 100644 --- a/Mac/README +++ b/Mac/README @@ -98,7 +98,7 @@ In general, universal builds depend on specific features provided by the Apple-supplied compilers and other build tools included in Apple's Xcode development tools. You should install Xcode and the command line tools component appropriate for the OS X release you are running on. See the -Python Developer's Guide (http://docs.python.org/devguide/setup.html) +Python Developer's Guide (https://devguide.python.org/setup/) for more information. 2.1 Flavors of universal binaries @@ -355,4 +355,4 @@ Resources * http://www.python.org/community/sigs/current/pythonmac-sig/ - * http://docs.python.org/devguide/ \ No newline at end of file + * https://devguide.python.org/ diff --git a/Makefile.pre.in b/Makefile.pre.in index 93aae9124726bd..6a0118f7871420 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,9 +41,9 @@ RANLIB= @RANLIB@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -HGVERSION= @HGVERSION@ -HGTAG= @HGTAG@ -HGBRANCH= @HGBRANCH@ +GITVERSION= @GITVERSION@ +GITTAG= @GITTAG@ +GITBRANCH= @GITBRANCH@ PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ @@ -107,6 +107,8 @@ ARFLAGS= @ARFLAGS@ CFLAGSFORSHARED=@CFLAGSFORSHARED@ # C flags used for building the interpreter object files PY_CORE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE +# Strict or non-strict aliasing flags used to compile dtoa.c, see above +CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Machine-dependent subdirectories @@ -227,7 +229,8 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ +PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ +UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -271,11 +274,6 @@ IO_OBJS= \ Modules/_io/stringio.o ########################################################################## -# Grammar -GRAMMAR_H= Include/graminit.h -GRAMMAR_C= Python/graminit.c -GRAMMAR_INPUT= $(srcdir)/Grammar/Grammar - LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ @@ -314,38 +312,9 @@ PARSER_HEADERS= \ PGENOBJS= $(POBJS) $(PGOBJS) -########################################################################## -# opcode.h generation -OPCODE_H_DIR= $(srcdir)/Include -OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py -OPCODE_H= $(OPCODE_H_DIR)/opcode.h -OPCODE_H_GEN= $(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H) - -########################################################################## -# AST -AST_H_DIR= Include -AST_H= $(AST_H_DIR)/Python-ast.h -AST_C_DIR= Python -AST_C= $(AST_C_DIR)/Python-ast.c -AST_ASDL= $(srcdir)/Parser/Python.asdl - -ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py -# Note that a build now requires Python to exist before the build starts. -# Use "hg touch" to fix up screwed up file mtimes in a checkout. -ASDLGEN= $(PYTHON_FOR_GEN) $(srcdir)/Parser/asdl_c.py - ########################################################################## # Python -OPCODETARGETS_H= \ - Python/opcode_targets.h - -OPCODETARGETGEN= \ - $(srcdir)/Python/makeopcodetargets.py - -OPCODETARGETGEN_FILES= \ - $(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py - PYTHON_OBJS= \ Python/_warnings.o \ Python/Python-ast.o \ @@ -545,7 +514,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -coverage-report: +# Force regeneration of parser and importlib +coverage-report: regen-grammar regen-importlib : # force rebuilding of parser and importlib @touch $(GRAMMAR_INPUT) @touch $(srcdir)/Lib/importlib/_bootstrap.py @@ -721,14 +691,26 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -Python/importlib_external.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib Python/marshal.c +.PHONY: regen-importlib +regen-importlib: Programs/_freeze_importlib + # Regenerate Python/importlib_external.h + # from Lib/importlib/_bootstrap_external.py using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h - -Python/importlib.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib Python/marshal.c + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Python/importlib_external.h.new + $(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new + # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py + # using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Python/importlib.h.new + $(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new + +############################################################################ +# Regenerate all generated files + +regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib ############################################################################ # Special rules for object files @@ -740,9 +722,9 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(MODOBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ - -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \ - -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \ - -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile @@ -787,15 +769,20 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(IO_OBJS): $(IO_H) -$(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) - @$(MKDIR_P) Include - $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) -$(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) - touch $(GRAMMAR_C) - $(PGEN): $(PGENOBJS) $(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) +.PHONY: regen-grammar +regen-grammar: $(PGEN) + # Regenerate Include/graminit.h and Python/graminit.c + # from Grammar/Grammar using pgen + @$(MKDIR_P) Include + $(PGEN) $(srcdir)/Grammar/Grammar \ + $(srcdir)/Include/graminit.h.new \ + $(srcdir)/Python/graminit.c.new + $(UPDATE_FILE) $(srcdir)/Include/graminit.h $(srcdir)/Include/graminit.h.new + $(UPDATE_FILE) $(srcdir)/Python/graminit.c $(srcdir)/Python/graminit.c.new + Parser/grammar.o: $(srcdir)/Parser/grammar.c \ $(srcdir)/Include/token.h \ $(srcdir)/Include/grammar.h @@ -807,18 +794,31 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c Parser/pgenmain.o: $(srcdir)/Include/parsetok.h -$(AST_H): $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_H_DIR) - $(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL) - -$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_C_DIR) - $(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL) - -$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT) - $(OPCODE_H_GEN) - -Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H) +.PHONY=regen-ast +regen-ast: + # Regenerate Include/Python-ast.h using Parser/asdl_c.py -h + $(MKDIR_P) $(srcdir)/Include + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -h $(srcdir)/Include/Python-ast.h.new \ + $(srcdir)/Parser/Python.asdl + $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new + # Regenerate Python/Python-ast.c using Parser/asdl_c.py -c + $(MKDIR_P) $(srcdir)/Python + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -c $(srcdir)/Python/Python-ast.c.new \ + $(srcdir)/Parser/Python.asdl + $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new + +.PHONY: regen-opcode +regen-opcode: + # Regenerate Include/opcode.h from Lib/opcode.py + # using Tools/scripts/generate_opcode_h.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \ + $(srcdir)/Lib/opcode.py \ + $(srcdir)/Include/opcode.h.new + $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new + +Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -868,17 +868,23 @@ Objects/odictobject.o: $(srcdir)/Objects/dict-common.h Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) - $(PYTHON_FOR_GEN) $(OPCODETARGETGEN) $(OPCODETARGETS_H) +.PHONY: regen-opcode-targets +regen-opcode-targets: + # Regenerate Python/opcode_targets.h from Lib/opcode.py + # using Python/makeopcodetargets.py + $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ + $(srcdir)/Python/opcode_targets.h.new + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new -Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h -Python/frozen.o: Python/importlib.h Python/importlib_external.h +Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d + $(MKDIR_P) Include $(DTRACE) $(DFLAGS) -o $@ -h -s $< : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp @@ -888,8 +894,15 @@ Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc -Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py - $(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc + +.PHONY: regen-typeslots +regen-typeslots: + # Regenerate Objects/typeslots.inc from Include/typeslotsh + # using Objects/typeslots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ + < $(srcdir)/Include/typeslots.h \ + $(srcdir)/Objects/typeslots.inc.new + $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new ############################################################################ # Header files @@ -942,7 +955,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/node.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ - $(OPCODE_H) \ + $(srcdir)/Include/opcode.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -985,7 +998,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/weakrefobject.h \ pyconfig.h \ $(PARSER_HEADERS) \ - $(AST_H) \ + $(srcdir)/Include/Python-ast.h \ $(DTRACE_HEADERS) $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -998,9 +1011,11 @@ TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py TESTTIMEOUT= 1200 +.PHONY: test testall testuniversal buildbottest pythoninfo + # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: all platform +test: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1010,7 +1025,7 @@ test: all platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: all platform +testall: @DEF_MAKE_RULE@ platform -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1019,7 +1034,7 @@ testall: all platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: all platform +testuniversal: @DEF_MAKE_RULE@ platform if [ `arch` != 'i386' ];then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1036,15 +1051,25 @@ buildbottest: build_all platform fi $(TESTRUNNER) -j 1 -u all -W --slowest --timeout=$(TESTTIMEOUT) $(TESTOPTS) +pythoninfo: build_all + $(RUNSHARED) ./$(BUILDPYTHON) -m test.pythoninfo + QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ test_mailbox test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: all platform +quicktest: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(QUICKTESTOPTS) +# SSL tests +.PHONY: multisslcompile multissltest +multisslcompile: build_all + $(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py --compile-only + +multissltest: build_all + $(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ @@ -1203,6 +1228,8 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/test_import/data \ test/test_import/data/circular_imports \ test/test_import/data/circular_imports/subpkg \ + test/test_import/data/package \ + test/test_import/data/package2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ @@ -1379,7 +1406,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: all python-config +libainstall: @DEF_MAKE_RULE@ python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -1535,6 +1562,13 @@ config.status: $(srcdir)/configure .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +# bpo-30104: dtoa.c uses union to cast double to unsigned long[2]. clang 4.0 +# with -O2 or higher and strict aliasing miscompiles the ratio() function +# causing rounding issues. Compile dtoa.c using -fno-strict-aliasing on clang. +# https://bugs.llvm.org//show_bug.cgi?id=31928 +Python/dtoa.o: Python/dtoa.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< + # Run reindent on the library reindent: ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib @@ -1545,9 +1579,12 @@ recheck: $(SHELL) config.status --recheck $(SHELL) config.status -# Rebuild the configure script from configure.ac; also rebuild pyconfig.h.in +# Regenerate configure and pyconfig.h.in +.PHONY: autoconf autoconf: + # Regenerate the configure script from configure.ac using autoconf (cd $(srcdir); autoconf -Wall) + # Regenerate pyconfig.h.in from configure.ac using autoheader (cd $(srcdir); autoheader -Wall) # Create a tags file for vi @@ -1564,14 +1601,6 @@ TAGS:: etags Include/*.h; \ for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done -# This fixes up the mtimes of checked-in generated files, assuming that they -# only *appear* to be outdated because of checkout order. -# This is run while preparing a source release tarball, and can be run manually -# to avoid bootstrap issues. -touch: - cd $(srcdir); \ - hg --config extensions.touch=Tools/hg/hgtouch.py touch -v - # Sanitation targets -- clean leaves libraries, executables and tags # files, which clobber removes as well pycremoval: @@ -1626,7 +1655,8 @@ distclean: clobber done -rm -f core Makefile Makefile.pre config.status \ Modules/Setup Modules/Setup.local Modules/Setup.config \ - Modules/ld_so_aix Modules/python.exp Misc/python.pc + Modules/ld_so_aix Modules/python.exp Misc/python.pc \ + Misc/python-config.sh -rm -f python*-gdb.py # Issue #28258: set LC_ALL to avoid issues with Estonian locale. # Expansion is performed here by shell (spawned by make) itself before @@ -1639,7 +1669,7 @@ distclean: clobber -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) -smelly: all +smelly: @DEF_MAKE_RULE@ nm -p $(LIBRARY) | \ sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ @@ -1676,7 +1706,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: all +patchcheck: @DEF_MAKE_RULE@ $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py # Dependencies @@ -1689,7 +1719,7 @@ Python/thread.o: @THREADHEADERS@ .PHONY: maninstall libinstall inclinstall libainstall sharedinstall .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools -.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean +.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean .PHONY: smelly funny patchcheck touch altmaninstall commoninstall .PHONY: gdbhooks diff --git a/Misc/ACKS b/Misc/ACKS index a6b3df12ff230c..83336eca5b9d25 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -56,6 +56,7 @@ Ankur Ankan Heidi Annexstad Ramchandra Apte Éric Araujo +Alexandru Ardelean Alicia Arlen Jeffrey Armstrong Jason Asbahr @@ -128,6 +129,7 @@ Andrew Bennetts Andy Bensky Bennett Benson Ezra Berch +Stuart Berg Michel Van den Bergh Julian Berman Brice Berna @@ -269,6 +271,7 @@ Albert Chin-A-Young Adal Chiriliuc Matt Chisholm Lita Cho +Sayan Chowdhury Anders Chrigström Tom Christiansen Renee Chu @@ -307,6 +310,7 @@ Garrett Cooper Greg Copeland Ian Cordasco Aldo Cortesi +Mircea Cosbuc David Costanzo Scott Cotton Greg Couch @@ -347,6 +351,7 @@ A. Jesse Jiryu Davis Merlijn van Deen John DeGood Ned Deily +Jim DeLaHunt Vincent Delft Arnaud Delobelle Konrad Delong @@ -540,6 +545,7 @@ Tim Graham Kim Gräsman Nathaniel Gray Eddy De Greef +Duane Griffin Grant Griffin Andrea Griffini Duncan Grisby @@ -549,6 +555,8 @@ Eric Groo Daniel Andrade Groppe Dag Gruneau Filip Gruszczyński +Andrii Grynenko +Grzegorz Grzywacz Thomas Guettler Yuyang Guo Anuj Gupta @@ -584,6 +592,7 @@ David Harrigan Brian Harring Jonathan Hartley Travis B. Hartwell +Shane Harvey Larry Hastings Tim Hatch Shane Hathaway @@ -611,11 +620,13 @@ Thomas Herve Bernhard Herzog Magnus L. Hetland Raymond Hettinger +Lisa Hewus Fresh Kevan Heydon Wouter van Heyst Kelsey Hightower Jason Hildebrand Aaron Hill +Joel Hillacre Richie Hindle Konrad Hinsen David Hobley @@ -768,11 +779,13 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes @@ -791,6 +804,7 @@ Kim Knapp Lenny Kneler Pat Knight Jeff Knupp +Nicholas Kobald Kubilay Kocak Greg Kochanski Manvisha Kodali @@ -956,6 +970,7 @@ David Marek Doug Marien Sven Marnach Alex Martelli +Dennis Mårtensson Anthony Martin Owen Martin Sidney San Martín @@ -1056,6 +1071,7 @@ R. David Murray Matti Mäki Jörg Müller Kaushik N +Dong-hee Na Dale Nagata John Nagle Takahiro Nakayama @@ -1065,11 +1081,13 @@ Vilmos Nebehaj Fredrik Nehr Tony Nelson Trent Nelson +Osvaldo Santana Neto Chad Netzer Max Neunhöffer Anthon van der Neut George Neville-Neil Hieu Nguyen +Nam Nguyen Johannes Nicolai Samuel Nicolary Jonathan Niehof @@ -1101,6 +1119,7 @@ Milan Oberkirch Pascal Oberndoerfer Jeffrey Ollie Adam Olsen +Bryan Olson Grant Olson Koray Oner Piet van Oostrum @@ -1109,7 +1128,9 @@ Jason Orendorff Douglas Orr William Orr Michele Orrù +Tomáš Orsava Oleg Oshmyan +Denis Osipov Denis S. Otkidach Peter Otten Michael Otteneder @@ -1151,6 +1172,7 @@ Berker Peksag Andreas Pelme Steven Pemberton Bo Peng +Bruno "Polaco" Penteado Santiago Peresón George Peristerakis Thomas Perl @@ -1179,6 +1201,7 @@ Steve Piercy Jim St. Pierre Dan Pierson Martijn Pieters +Antoine Pietri Anand B. Pillai François Pinard Tom Pinckney @@ -1320,6 +1343,7 @@ Chris Ryland Bernt Røskar Brenna Constantina S. Matthieu S +Cheryl Sabella Patrick Sabin Sébastien Sablé Amit Saha @@ -1369,6 +1393,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Žiga Seilnacht Yury Selivanov Fred Sells @@ -1469,6 +1494,7 @@ Daniel Stokes Michael Stone Serhiy Storchaka Ken Stox +Charalampos Stratakis Dan Stromberg Donald Stufft Daniel Stutzbach @@ -1511,6 +1537,7 @@ Martin Teichmann Gustavo Temple Mikhail Terekhov Victor Terrón +Pablo Galindo Richard M. Tew Tobias Thelen Févry Thibault @@ -1561,6 +1588,7 @@ Doobee R. Tzeck Eren Türkay Lionel Ulmer Adnan Umer +Utkarsh Upadhyay Roger Upole Daniel Urban Michael Urman @@ -1571,6 +1599,7 @@ Andi Vajda Case Van Horsen John Mark Vandenberg Kyle VanderBeek +Eric N. Vander Weele Andrew Vant Atul Varma Dmitry Vasiliev @@ -1648,6 +1677,7 @@ Jakub Wilk Gerald S. Williams Jason Williams John Williams +Roy Williams Sue Williams Carol Willing Steven Willis @@ -1669,6 +1699,7 @@ John Wiseman Chris Withers Stefan Witzel Irek Wlizlo +Charles Wohlganger David Wolever Klaus-Juergen Wolf Dan Wolfe @@ -1706,6 +1737,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS deleted file mode 100644 index d79ff2450defaa..00000000000000 --- a/Misc/NEWS +++ /dev/null @@ -1,8415 +0,0 @@ -+++++++++++ -Python News -+++++++++++ - -What's New in Python 3.6.1 release candidate 1? -=============================================== - -*Release date: XXXX-XX-XX* - -Core and Builtins ------------------ - -- Issue #29319: Prevent RunMainFromImporter overwriting sys.path[0]. - -- Issue #29337: Fixed possible BytesWarning when compare the code objects. - Warnings could be emitted at compile time. - -- Issue #29327: Fixed a crash when pass the iterable keyword argument to - sorted(). - -- Issue #29034: Fix memory leak and use-after-free in os module (path_converter). - -- Issue #29159: Fix regression in bytes(x) when x.__index__() raises Exception. - -- Issue #28932: Do not include if it does not exist. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #26919: On Android, operating system data is now always encoded/decoded - to/from UTF-8, instead of the locale encoding to avoid inconsistencies with - os.fsencode() and os.fsdecode() which are already using UTF-8. - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug triggerable by a monkey-patched len() function. - -- Issue #28739: f-string expressions are no longer accepted as docstrings and - by ast.literal_eval() even if they do not include expressions. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28918: Fix the cross compilation of xxlimited when Python has been - built with Py_DEBUG defined. - -- Issue #28731: Optimize _PyDict_NewPresized() to create correct size dict. - Improve speed of dict literal with constant keys up to 30%. - -Extension Modules ------------------ - -- Issue #29169: Update zlib to 1.2.11. - -Library -------- - -- Issue #29100: Fix datetime.fromtimestamp() regression introduced in Python - 3.6.0: check minimum and maximum years. - -- Issue #29519: Fix weakref spewing exceptions during interpreter shutdown - when used with a rare combination of multiprocessing and custom codecs. - -- Issue #29416: Prevent infinite loop in pathlib.Path.mkdir - -- Issue #29444: Fixed out-of-bounds buffer access in the group() method of - the match object. Based on patch by WGH. - -- Issue #29335: Fix subprocess.Popen.wait() when the child process has - exited to a stopped instead of terminated state (ex: when under ptrace). - -- Issue #29290: Fix a regression in argparse that help messages would wrap at - non-breaking spaces. - -- Issue #28735: Fixed the comparison of mock.MagickMock with mock.ANY. - -- Issue #29316: Restore the provisional status of typing module, add - corresponding note to documentation. Patch by Ivan L. - -- Issue #29219: Fixed infinite recursion in the repr of uninitialized - ctypes.CDLL instances. - -- Issue #29011: Fix an important omission by adding Deque to the typing module. - -- Issue #28969: Fixed race condition in C implementation of functools.lru_cache. - KeyError could be raised when cached function with full cache was - simultaneously called from differen threads with the same uncached arguments. - -- Issue #29142: In urllib.request, suffixes in no_proxy environment variable with - leading dots could match related hostnames again (e.g. .b.c matches a.b.c). - Patch by Milan Oberkirch. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- Issue #29203: functools.lru_cache() now respects PEP 468 and preserves - the order of keyword arguments. f(a=1, b=2) is now cached separately - from f(b=2, a=1) since both calls could potentially give different results. - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #29085: Allow random.Random.seed() to use high quality OS randomness - rather than the pid and time. - -- Issue #29061: Fixed bug in secrets.randbelow() which would hang when given - a negative input. Patch by Brendan Donegan. - -- Issue #29079: Prevent infinite loop in pathlib.resolve() on Windows - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #29119: Fix weakrefs in the pure python version of - collections.OrderedDict move_to_end() method. - Contributed by Andra Bogildea. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #29055: Neaten-up empty population error on random.choice() - by suppressing the upstream exception. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when passing a sequence that - doesn't own its elements as limits. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. - -- Issue #26937: The chown() method of the tarfile.TarFile class does not fail - now when the grp module cannot be imported, as for example on Android - platforms. - -Windows -------- - -- Issue #29326: Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) - -- Issue #28164: Correctly handle special console filenames (patch by Eryk Sun) - -- Issue #29409: Implement PEP 529 for io.FileIO (Patch by Eryk Sun) - -- Issue #29392: Prevent crash when passing invalid arguments into msvcrt module. - -- Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun) - -- Issue #28896: Deprecate WindowsRegistryFinder and disable it by default. - -C API ------ - -- Issue #27867: Function PySlice_GetIndicesEx() is replaced with a macro if - Py_LIMITED_API is not set or set to the value between 0x03050400 - and 0x03060000 (not including) or 0x03060100 or higher. - -- Issue #29083: Fixed the declaration of some public API functions. - PyArg_VaParse() and PyArg_VaParseTupleAndKeywords() were not available in - limited API. PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and - Py_BuildValue() were not available in limited API of version < 3.3 when - PY_SSIZE_T_CLEAN is defined. - -- Issue #29058: All stable API extensions added after Python 3.2 are now - available only when Py_LIMITED_API is set to the PY_VERSION_HEX value of - the minimum Python version supporting this API. - -Documentation -------------- - -- Issue #26355: Add canonical header link on each page to corresponding major - version of the documentation. Patch by Matthias Bussonnier. - -- Issue #29349: Fix Python 2 syntax in code for building the documentation. - -Tests ------ - -- Issue #28950: Disallow -j0 to be combined with -T/-l in regrtest - command line arguments. - -- Issue #28683: Fix the tests that bind() a unix socket and raise - PermissionError on Android for a non-root user. - -- Issue #26939: Add the support.setswitchinterval() function to fix - test_functools hanging on the Android armv7 qemu emulator. - -Build ------ - -- Issue #26851: Set Android compilation and link flags. - -- Issue #28768: Fix implicit declaration of function _setmode. Patch by - Masayuki Yamamoto - -- Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat - -- Issue #23903: Added missed names to PC/python3.def. - -- Issue #28762: lockf() is available on Android API level 24, but the F_LOCK - macro is not defined in android-ndk-r13. - -- Issue #28538: Fix the compilation error that occurs because if_nameindex() is - available on Android API level 24, but the if_nameindex structure is not - defined. - -- Issue #20211: Do not add the directory for installing C header files and the - directory for installing object code libraries to the cross compilation - search paths. Original patch by Thomas Petazzoni. - -- Issue #28849: Do not define sys.implementation._multiarch on Android. - - -What's New in Python 3.6.0? -=========================== - -*Release date: 2016-12-23* - -- No changes since release candidate 2 - - -What's New in Python 3.6.0 release candidate 2? -=============================================== - -*Release date: 2016-12-16* - -Core and Builtins ------------------ - -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. Patch written by INADA - Naoki. - -- Issue #28990: Fix asyncio SSL hanging if connection is closed before - handshake is completed. (Patch by HoHo-Ho) - -Tools/Demos ------------ - -- Issue #28770: Fix python-gdb.py for fastcalls. - -Windows -------- - -- Issue #28896: Deprecate WindowsRegistryFinder. - -Build ------ - -- Issue #28898: Prevent gdb build errors due to HAVE_LONG_LONG redefinition. - - -What's New in Python 3.6.0 release candidate 1? -=============================================== - -*Release date: 2016-12-06* - -Core and Builtins ------------------ - -- Issue #23722: Rather than silently producing a class that doesn't support - zero-argument ``super()`` in methods, failing to pass the new - ``__classcell__`` namespace entry up to ``type.__new__`` now results in a - ``DeprecationWarning`` and a class that supports zero-argument ``super()``. - -- Issue #28797: Modifying the class __dict__ inside the __set_name__ method of - a descriptor that is used inside that class no longer prevents calling the - __set_name__ method of other descriptors. - -- Issue #28782: Fix a bug in the implementation ``yield from`` when checking - if the next instruction is YIELD_FROM. Regression introduced by WORDCODE - (issue #26647). - -Library -------- - -- Issue #27030: Unknown escapes in re.sub() replacement template are allowed - again. But they still are deprecated and will be disabled in 3.7. - -- Issue #28835: Fix a regression introduced in warnings.catch_warnings(): - call warnings.showwarning() if it was overriden inside the context manager. - -- Issue #27172: To assist with upgrades from 2.7, the previously documented - deprecation of ``inspect.getfullargspec()`` has been reversed. This decision - may be revisited again after the Python 2.7 branch is no longer officially - supported. - -- Issue #26273: Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and - :data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by - Omar Sandoval. - -- Issue #24142: Reading a corrupt config file left configparser in an - invalid state. Original patch by Florian Höch. - -- Issue #28843: Fix asyncio C Task to handle exceptions __traceback__. - -C API ------ - -- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. - -Documentation -------------- - -- Issue #23722: The data model reference and the porting section in the What's - New guide now cover the additional ``__classcell__`` handling needed for - custom metaclasses to fully support PEP 487 and zero-argument ``super()``. - -Tools/Demos ------------ - -- Issue #28023: Fix python-gdb.py didn't support new dict implementation. - - -What's New in Python 3.6.0 beta 4? -================================== - -*Release date: 2016-11-21* - -Core and Builtins ------------------ - -- Issue #28532: Show sys.version when -V option is supplied twice. - -- Issue #27100: The with-statement now checks for __enter__ before it - checks for __exit__. This gives less confusing error messages when - both methods are missing. Patch by Jonathan Ellington. - -- Issue #28746: Fix the set_inheritable() file descriptor method on platforms - that do not have the ioctl FIOCLEX and FIONCLEX commands. - -- Issue #26920: Fix not getting the locale's charset upon initializing the - interpreter, on platforms that do not have langinfo. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28665: Improve speed of the STORE_DEREF opcode by 40%. - -- Issue #28583: PyDict_SetDefault didn't combine split table when needed. - Patch by Xiang Zhang. - -- Issue #27243: Change PendingDeprecationWarning -> DeprecationWarning. - As it was agreed in the issue, __aiter__ returning an awaitable - should result in PendingDeprecationWarning in 3.5 and in - DeprecationWarning in 3.6. - -- Issue #26182: Fix a refleak in code that raises DeprecationWarning. - -- Issue #28721: Fix asynchronous generators aclose() and athrow() to - handle StopAsyncIteration propagation properly. - -Library -------- - -- Issue #28752: Restored the __reduce__() methods of datetime objects. - -- Issue #28727: Regular expression patterns, _sre.SRE_Pattern objects created - by re.compile(), become comparable (only x==y and x!=y operators). This - change should fix the issue #18383: don't duplicate warning filters when the - warnings module is reloaded (thing usually only done in unit tests). - -- Issue #20572: The subprocess.Popen.wait method's undocumented - endtime parameter now raises a DeprecationWarning. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. - Patch by Vajrasky Kok - -- Issue #28563: Fixed possible DoS and arbitrary code execution when handle - plural form selections in the gettext module. The expression parser now - supports exact syntax supported by GNU gettext. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #28600: Optimize loop.call_soon. - -- Issue #28613: Fix get_event_loop() return the current loop if - called from coroutines/callbacks. - -- Issue #28634: Fix asyncio.isfuture() to support unittest.Mock. - -- Issue #26081: Fix refleak in _asyncio.Future.__iter__().throw. - -- Issue #28639: Fix inspect.isawaitable to always return bool - Patch by Justin Mayfield. - -- Issue #28652: Make loop methods reject socket kinds they do not support. - -- Issue #28653: Fix a refleak in functools.lru_cache. - -- Issue #28703: Fix asyncio.iscoroutinefunction to handle Mock objects. - -- Issue #28704: Fix create_unix_server to support Path-like objects - (PEP 519). - -- Issue #28720: Add collections.abc.AsyncGenerator. - -Documentation -------------- - -- Issue #28513: Documented command-line interface of zipfile. - -Tests ------ - -- Issue #28666: Now test.support.rmtree is able to remove unwritable or - unreadable directories. - -- Issue #23839: Various caches now are cleared before running every test file. - -Build ------ - -- Issue #10656: Fix out-of-tree building on AIX. Patch by Tristan Carel and - Michael Haubenwallner. - -- Issue #26359: Rename --with-optimiations to --enable-optimizations. - -- Issue #28676: Prevent missing 'getentropy' declaration warning on macOS. - Patch by Gareth Rees. - - -What's New in Python 3.6.0 beta 3? -================================== - -*Release date: 2016-10-31* - -Core and Builtins ------------------ - -- Issue #28128: Deprecation warning for invalid str and byte escape - sequences now prints better information about where the error - occurs. Patch by Serhiy Storchaka and Eric Smith. - -- Issue #28509: dict.update() no longer allocate unnecessary large memory. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused - keeping unreachable code. - -- Issue #28214: Improved exception reporting for problematic __set_name__ - attributes. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28471: Fix "Python memory allocator called without holding the GIL" - crash in socket.setblocking. - -Library -------- - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #18844: The various ways of specifying weights for random.choices() - now produce the same result sequences. - -- Issue #28255: calendar.TextCalendar().prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #28430: Fix iterator of C implemented asyncio.Future doesn't accept - non-None value is passed to it.send(val). - -- Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer adds entry "./" to ZIP archive. - -- Issue #25953: re.sub() now raises an error for invalid numerical group - reference in replacement template even if the pattern is not found in - the string. Error message for invalid group reference now includes the - group index and the position of the reference. - Based on patch by SilentGhost. - -- Issue #18219: Optimize csv.DictWriter for large number of columns. - Patch by Mariatta Wijaya. - -- Issue #28448: Fix C implemented asyncio.Future didn't work on Windows. - -- Issue #28480: Fix error building socket module when multithreading is - disabled. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #28492: Fix how StopIteration exception is raised in _asyncio.Future. - -- Issue #28500: Fix asyncio to handle async gens GC from another thread. - -- Issue #26923: Fix asyncio.Gather to refuse being cancelled once all - children are done. - Patch by Johannes Ebke. - -- Issue #26796: Don't configure the number of workers for default - threadpool executor. - Initial patch by Hans Lawrenz. - -- Issue #28544: Implement asyncio.Task in C. - -Windows -------- - -- Issue #28522: Fixes mishandled buffer reallocation in getpathp.c - -Build ------ - -- Issue #28444: Fix missing extensions modules when cross compiling. - -- Issue #28208: Update Windows build and OS X installers to use SQLite 3.14.2. - -- Issue #28248: Update Windows build and OS X installers to use OpenSSL 1.0.2j. - -Tests ------ - -- Issue #26944: Fix test_posix for Android where 'id -G' is entirely wrong or - missing the effective gid. - -- Issue #28409: regrtest: fix the parser of command line arguments. - - -What's New in Python 3.6.0 beta 2? -================================== - -*Release date: 2016-10-10* - -Core and Builtins ------------------ - -- Issue #28183: Optimize and cleanup dict iteration. - -- Issue #26081: Added C implementation of asyncio.Future. - Original patch by Yury Selivanov. - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: Creating instances of range_iterator by calling range_iterator - type now is deprecated. Patch by Oren Milman. - -- Issue #28376: The constructor of range_iterator now checks that step is not 0. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28201: Dict reduces possibility of 2nd conflict in hash table when - hashes have same lower bits. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in complex(1.0, {2:3}) error message. - Patch by Soumya Sharma. - -- Issue #28086: Single var-positional argument of tuple subtype was passed - unscathed to the C-defined function. Now it is converted to exact tuple. - -- Issue #28214: Now __set_name__ is looked up on the class instead of the - instance. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28192: Don't import readline in isolated mode. - -- Upgrade internal unicode databases to Unicode version 9.0.0. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #28126: Replace Py_MEMCPY with memcpy(). Visual Studio can properly - optimize memcpy(). - -- Issue #28120: Fix dict.pop() for splitted dictionary when trying to remove a - "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. - -- Issue #26182: Raise DeprecationWarning when async and await keywords are - used as variable/attribute/class/function name. - -Library -------- - -- Issue #27998: Fixed bytes path support in os.scandir() on Windows. - Patch by Eryk Sun. - -- Issue #28317: The disassembler now decodes FORMAT_VALUE argument. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28380: unittest.mock Mock autospec functions now properly support - assert_called, assert_not_called, and assert_called_once. - -- Issue #27181 remove statistics.geometric_mean and defer until 3.7. - -- Issue #28229: lzma module now supports pathlib. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28225: bz2 module now supports pathlib. Initial patch by Ethan Furman. - -- Issue #28227: gzip now supports pathlib. Patch by Ethan Furman. - -- Issue #27358: Optimized merging var-keyword arguments and improved error - message when passing a non-mapping as a var-keyword argument. - -- Issue #28257: Improved error message when passing a non-iterable as - a var-positional argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Issue #28228: imghdr now supports pathlib. - -- Issue #28226: compileall now supports pathlib. - -- Issue #28314: Fix function declaration (C flags) for the getiterator() method - of xml.etree.ElementTree.Element. - -- Issue #28148: Stop using localtime() and gmtime() in the time - module. - - Introduced platform independent _PyTime_localtime API that is - similar to POSIX localtime_r, but available on all platforms. Patch - by Ed Schouten. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18844: random.choices() now has k as a keyword-only argument - to improve the readability of common cases and come into line - with the signature used in other languages. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - Added the master parameter in the DisplayStyle constructor. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27778: Fix a memory leak in os.getrandom() when the getrandom() is - interrupted by a signal and a signal handler raises a Python exception. - -- Issue #28200: Fix memory leak on Windows in the os module (fix - path_converter() function). - -- Issue #25400: RobotFileParser now correctly returns default values for - crawl_delay and request_rate. Initial patch by Peter Wirtz. - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #22493: Warning message emitted by using inline flags in the middle of - regular expression now contains a (truncated) regex pattern. - Patch by Tim Graham. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. - Patch by Eryk Sun. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28368: Refuse monitoring processes if the child watcher has no - loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by Коренберг Марк. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by Коренберг Марк. - -- Issue #27972: Prohibit Tasks to await on themselves. - -Windows -------- - -- Issue #28402: Adds signed catalog files for stdlib on Windows. - -- Issue #28333: Enables Unicode for ps1/ps2 and input() prompts. (Patch by - Eryk Sun) - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #28161: Opening CON for write access fails - -- Issue #28162: WindowsConsoleIO readall() fails if first line starts with - Ctrl+Z - -- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to - _open_osfhandle - -- Issue #28164: _PyIO_get_console_type fails for various paths - -- Issue #28137: Renames Windows path file to ._pth - -- Issue #28138: Windows ._pth file should allow import site - -C API ------ - -- Issue #28426: Deprecated undocumented functions PyUnicode_AsEncodedObject(), - PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and - PyUnicode_AsEncodedUnicode(). - -Build ------ - -- Issue #28258: Fixed build with Estonian locale (python-config and distclean - targets in Makefile). Patch by Arfrever Frehtes Taifersar Arahesis. - -- Issue #26661: setup.py now detects system libffi with multiarch wrapper. - -- Issue #15819: Remove redundant include search directory option for building - outside the source tree. - -Tests ------ - -- Issue #28217: Adds _testconsole module to test console input. - - -What's New in Python 3.6.0 beta 1? -================================== - -*Release date: 2016-09-12* - -Core and Builtins ------------------ - -- Issue #23722: The __class__ cell used by zero-argument super() is now - initialized from type.__new__ rather than __build_class__, so class methods - relying on that will now work correctly when called from metaclass methods - during class creation. Patch by Martin Teichmann. - -- Issue #25221: Fix corrupted result from PyLong_FromLong(0) when Python - is compiled with NSMALLPOSINTS = 0. - -- Issue #27080: Implement formatting support for PEP 515. Initial patch - by Chris Angelico. - -- Issue #27199: In tarfile, expose copyfileobj bufsize to improve throughput. - Patch by Jason Fried. - -- Issue #27948: In f-strings, only allow backslashes inside the braces - (where the expressions are). This is a breaking change from the 3.6 - alpha releases, where backslashes are allowed anywhere in an - f-string. Also, require that expressions inside f-strings be - enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. - -- Issue #28046: Remove platform-specific directories from sys.path. - -- Issue #28071: Add early-out for differencing from an empty set. - -- Issue #25758: Prevents zipimport from unnecessarily encoding a filename - (patch by Eryk Sun) - -- Issue #25856: The __module__ attribute of extension classes and functions - now is interned. This leads to more compact pickle data with protocol 4. - -- Issue #27213: Rework CALL_FUNCTION* opcodes to produce shorter and more - efficient bytecode. Patch by Demur Rumed, design by Serhiy Storchaka, - reviewed by Serhiy Storchaka and Victor Stinner. - -- Issue #26331: Implement tokenizing support for PEP 515. Patch by Georg Brandl. - -- Issue #27999: Make "global after use" a SyntaxError, and ditto for nonlocal. - Patch by Ivan Levkivskyi. - -- Issue #28003: Implement PEP 525 -- Asynchronous Generators. - -- Issue #27985: Implement PEP 526 -- Syntax for Variable Annotations. - Patch by Ivan Levkivskyi. - -- Issue #26058: Add a new private version to the builtin dict type, incremented - at each dictionary creation and at each dictionary change. Implementation of - the PEP 509. - -- Issue #27364: A backslash-character pair that is not a valid escape sequence - now generates a DeprecationWarning. Patch by Emanuel Barry. - -- Issue #27350: `dict` implementation is changed like PyPy. It is more compact - and preserves insertion order. - (Concept developed by Raymond Hettinger and patch by Inada Naoki.) - -- Issue #27911: Remove unnecessary error checks in - ``exec_builtin_or_dynamic()``. - -- Issue #27078: Added BUILD_STRING opcode. Optimized f-strings evaluation. - -- Issue #17884: Python now requires systems with inttypes.h and stdint.h - -- Issue #27961: Require platforms to support ``long long``. Python hasn't - compiled without ``long long`` for years, so this is basically a formality. - -- Issue #27355: Removed support for Windows CE. It was never finished, - and Windows CE is no longer a relevant platform for Python. - -- Implement PEP 523. - -- Issue #27870: A left shift of zero by a large integer no longer attempts - to allocate large amounts of memory. - -- Issue #25402: In int-to-decimal-string conversion, improve the estimate - of the intermediate memory required, and remove an unnecessarily strict - overflow check. Patch by Serhiy Storchaka. - -- Issue #27214: In long_invert, be more careful about modifying object - returned by long_add, and remove an unnecessary check for small longs. - Thanks Oren Milman for analysis and patch. - -- Issue #27506: Support passing the bytes/bytearray.translate() "delete" - argument by keyword. - -- Issue #27812: Properly clear out a generator's frame's backreference to the - generator to prevent crashes in frame.clear(). - -- Issue #27811: Fix a crash when a coroutine that has not been awaited is - finalized with warnings-as-errors enabled. - -- Issue #27587: Fix another issue found by PVS-Studio: Null pointer check - after use of 'def' in _PyState_AddModule(). - Initial patch by Christian Heimes. - -- Issue #27792: The modulo operation applied to ``bool`` and other - ``int`` subclasses now always returns an ``int``. Previously - the return type depended on the input values. Patch by Xiang Zhang. - -- Issue #26984: int() now always returns an instance of exact int. - -- Issue #25604: Fix a minor bug in integer true division; this bug could - potentially have caused off-by-one-ulp results on platforms with - unreliable ldexp implementations. - -- Issue #24254: Make class definition namespace ordered by default. - -- Issue #27662: Fix an overflow check in ``List_New``: the original code was - checking against ``Py_SIZE_MAX`` instead of the correct upper bound of - ``Py_SSIZE_T_MAX``. Patch by Xiang Zhang. - -- Issue #27782: Multi-phase extension module import now correctly allows the - ``m_methods`` field to be used to add module level functions to instances - of non-module types returned from ``Py_create_mod``. Patch by Xiang Zhang. - -- Issue #27936: The round() function accepted a second None argument - for some types but not for others. Fixed the inconsistency by - accepting None for all numeric types. - -- Issue #27487: Warn if a submodule argument to "python -m" or - runpy.run_module() is found in sys.modules after parent packages are - imported, but before the submodule is executed. - -- Issue #27157: Make only type() itself accept the one-argument form. - Patch by Eryk Sun and Emanuel Barry. - -- Issue #27558: Fix a SystemError in the implementation of "raise" statement. - In a brand new thread, raise a RuntimeError since there is no active - exception to reraise. Patch written by Xiang Zhang. - -- Issue #28008: Implement PEP 530 -- asynchronous comprehensions. - -- Issue #27942: Fix memory leak in codeobject.c - -Library -------- - -- Issue #28732: Fix crash in os.spawnv() with no elements in args - -- Issue #28485: Always raise ValueError for negative - compileall.compile_dir(workers=...) parameter, even when multithreading is - unavailable. - -- Issue #28037: Use sqlite3_get_autocommit() instead of setting - Connection->inTransaction manually. - -- Issue #25283: Attributes tm_gmtoff and tm_zone are now available on - all platforms in the return values of time.localtime() and - time.gmtime(). - -- Issue #24454: Regular expression match object groups are now - accessible using __getitem__. "mo[x]" is equivalent to - "mo.group(x)". - -- Issue #10740: sqlite3 no longer implicitly commit an open transaction - before DDL statements. - -- Issue #17941: Add a *module* parameter to collections.namedtuple(). - -- Issue #22493: Inline flags now should be used only at the start of the - regular expression. Deprecation warning is emitted if uses them in the - middle of the regular expression. - -- Issue #26885: xmlrpc now supports unmarshalling additional data types used - by Apache XML-RPC implementation for numerics and None. - -- Issue #28070: Fixed parsing inline verbose flag in regular expressions. - -- Issue #19500: Add client-side SSL session resumption to the ssl module. - -- Issue #28022: Deprecate ssl-related arguments in favor of SSLContext. The - deprecation include manual creation of SSLSocket and certfile/keyfile - (or similar) in ftplib, httplib, imaplib, smtplib, poplib and urllib. - -- Issue #28043: SSLContext has improved default settings: OP_NO_SSLv2, - OP_NO_SSLv3, OP_NO_COMPRESSION, OP_CIPHER_SERVER_PREFERENCE, - OP_SINGLE_DH_USE, OP_SINGLE_ECDH_USE and HIGH ciphers without MD5. - -- Issue #24693: Changed some RuntimeError's in the zipfile module to more - appropriate types. Improved some error messages and debugging output. - -- Issue #17909: ``json.load`` and ``json.loads`` now support binary input - encoded as UTF-8, UTF-16 or UTF-32. Patch by Serhiy Storchaka. - -- Issue #27137: the pure Python fallback implementation of ``functools.partial`` - now matches the behaviour of its accelerated C counterpart for subclassing, - pickling and text representation purposes. Patch by Emanuel Barry and - Serhiy Storchaka. - -- Fix possible integer overflows and crashes in the mmap module with unusual - usage patterns. - -- Issue #1703178: Fix the ability to pass the --link-objects option to the - distutils build_ext command. - -- Issue #28019: itertools.count() no longer rounds non-integer step in range - between 1.0 and 2.0 to 1. - -- Issue #18401: Pdb now supports the 'readrc' keyword argument to control - whether .pdbrc files should be read. Patch by Martin Matusiak and - Sam Kimbrel. - -- Issue #25969: Update the lib2to3 grammar to handle the unpacking - generalizations added in 3.5. - -- Issue #14977: mailcap now respects the order of the lines in the mailcap - files ("first match"), as required by RFC 1542. Patch by Michael Lazar. - -- Issue #28082: Convert re flag constants to IntFlag. - -- Issue #28025: Convert all ssl module constants to IntEnum and IntFlags. - SSLContext properties now return flags and enums. - -- Issue #23591: Add Flag, IntFlag, and auto() to enum module. - -- Issue #433028: Added support of modifier spans in regular expressions. - -- Issue #24594: Validates persist parameter when opening MSI database - -- Issue #17582: xml.etree.ElementTree nows preserves whitespaces in attributes - (Patch by Duane Griffin. Reviewed and approved by Stefan Behnel.) - -- Issue #28047: Fixed calculation of line length used for the base64 CTE - in the new email policies. - -- Issue #27576: Fix call order in OrderedDict.__init__(). - -- email.generator.DecodedGenerator now supports the policy keyword. - -- Issue #28027: Remove undocumented modules from ``Lib/plat-*``: IN, CDROM, - DLFCN, TYPES, CDIO, and STROPTS. - -- Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). - Patch by Claude Paroz. - -- Issue #24277: The new email API is no longer provisional, and the docs - have been reorganized and rewritten to emphasize the new API. - -- Issue #22450: urllib now includes an "Accept: */*" header among the - default headers. This makes the results of REST API requests more - consistent and predictable especially when proxy servers are involved. - -- lib2to3.pgen3.driver.load_grammar() now creates a stable cache file - between runs given the same Grammar.txt input regardless of the hash - randomization setting. - -- Issue #28005: Allow ImportErrors in encoding implementation to propagate. - -- Issue #26667: Support path-like objects in importlib.util. - -- Issue #27570: Avoid zero-length memcpy() etc calls with null source - pointers in the "ctypes" and "array" modules. - -- Issue #22233: Break email header lines *only* on the RFC specified CR and LF - characters, not on arbitrary unicode line breaks. This also fixes a bug in - HTTP header parsing. - -- Issue 27331: The email.mime classes now all accept an optional policy keyword. - -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. - -- Issue #16113: Add SHA-3 and SHAKE support to hashlib module. - -- Eliminate a tautological-pointer-compare warning in _scproxy.c. - -- Issue #27776: The :func:`os.urandom` function does now block on Linux 3.17 - and newer until the system urandom entropy pool is initialized to increase - the security. This change is part of the :pep:`524`. - -- Issue #27778: Expose the Linux ``getrandom()`` syscall as a new - :func:`os.getrandom` function. This change is part of the :pep:`524`. - -- Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name - fields in X.509 certs. - -- Issue #18844: Add random.choices(). - -- Issue #25761: Improved error reporting about truncated pickle data in - C implementation of unpickler. UnpicklingError is now raised instead of - AttributeError and ValueError in some cases. - -- Issue #26798: Add BLAKE2 (blake2b and blake2s) to hashlib. - -- Issue #26032: Optimized globbing in pathlib by using os.scandir(); it is now - about 1.5--4 times faster. - -- Issue #25596: Optimized glob() and iglob() functions in the - glob module; they are now about 3--6 times faster. - -- Issue #27928: Add scrypt (password-based key derivation function) to - hashlib module (requires OpenSSL 1.1.0). - -- Issue #27850: Remove 3DES from ssl module's default cipher list to counter - measure sweet32 attack (CVE-2016-2183). - -- Issue #27766: Add ChaCha20 Poly1305 to ssl module's default ciper list. - (Required OpenSSL 1.1.0 or LibreSSL). - -- Issue #25387: Check return value of winsound.MessageBeep. - -- Issue #27866: Add SSLContext.get_ciphers() method to get a list of all - enabled ciphers. - -- Issue #27744: Add AF_ALG (Linux Kernel crypto) to socket module. - -- Issue #26470: Port ssl and hashlib module to OpenSSL 1.1.0. - -- Issue #11620: Fix support for SND_MEMORY in winsound.PlaySound. Based on a - patch by Tim Lesher. - -- Issue #11734: Add support for IEEE 754 half-precision floats to the - struct module. Based on a patch by Eli Stevens. - -- Issue #27919: Deprecated ``extra_path`` distribution option in distutils - packaging. - -- Issue #23229: Add new ``cmath`` constants: ``cmath.inf`` and ``cmath.nan`` to - match ``math.inf`` and ``math.nan``, and also ``cmath.infj`` and - ``cmath.nanj`` to match the format used by complex repr. - -- Issue #27842: The csv.DictReader now returns rows of type OrderedDict. - (Contributed by Steve Holden.) - -- Remove support for passing a file descriptor to os.access. It never worked but - previously didn't raise. - -- Issue #12885: Fix error when distutils encounters symlink. - -- Issue #27881: Fixed possible bugs when setting sqlite3.Connection.isolation_level. - Based on patch by Xiang Zhang. - -- Issue #27861: Fixed a crash in sqlite3.Connection.cursor() when a factory - creates not a cursor. Patch by Xiang Zhang. - -- Issue #19884: Avoid spurious output on OS X with Gnu Readline. - -- Issue #27706: Restore deterministic behavior of random.Random().seed() - for string seeds using seeding version 1. Allows sequences of calls - to random() to exactly match those obtained in Python 2. - Patch by Nofar Schnider. - -- Issue #10513: Fix a regression in Connection.commit(). Statements should - not be reset after a commit. - -- Issue #12319: Chunked transfer encoding support added to - http.client.HTTPConnection requests. The - urllib.request.AbstractHTTPHandler class does not enforce a Content-Length - header any more. If a HTTP request has a file or iterable body, but no - Content-Length header, the library now falls back to use chunked transfer- - encoding. - -- A new version of typing.py from https://github.com/python/typing: - - Collection (only for 3.6) (Issue #27598) - - Add FrozenSet to __all__ (upstream #261) - - fix crash in _get_type_vars() (upstream #259) - - Remove the dict constraint in ForwardRef._eval_type (upstream #252) - -- Issue #27832: Make ``_normalize`` parameter to ``Fraction`` constuctor - keyword-only, so that ``Fraction(2, 3, 4)`` now raises ``TypeError``. - -- Issue #27539: Fix unnormalised ``Fraction.__pow__`` result in the case - of negative exponent and negative base. - -- Issue #21718: cursor.description is now available for queries using CTEs. - -- Issue #27819: In distutils sdists, simply produce the "gztar" (gzipped tar - format) distributions on all platforms unless "formats" is supplied. - -- Issue #2466: posixpath.ismount now correctly recognizes mount points which - the user does not have permission to access. - -- Issue #9998: On Linux, ctypes.util.find_library now looks in LD_LIBRARY_PATH - for shared libraries. - -- Issue #27573: exit message for code.interact is now configurable. - -- Issue #27930: Improved behaviour of logging.handlers.QueueListener. - Thanks to Paulo Andrade and Petr Viktorin for the analysis and patch. - -- Issue #6766: Distributed reference counting added to multiprocessing - to support nesting of shared values / proxy objects. - -- Issue #21201: Improves readability of multiprocessing error message. Thanks - to Wojciech Walczak for patch. - -- asyncio: Add set_protocol / get_protocol to Transports. - -- Issue #27456: asyncio: Set TCP_NODELAY by default. - -IDLE ----- - -- Issue #15308: Add 'interrupt execution' (^C) to Shell menu. - Patch by Roger Serwy, updated by Bayard Randel. - -- Issue #27922: Stop IDLE tests from 'flashing' gui widgets on the screen. - -- Issue #27891: Consistently group and sort imports within idlelib modules. - -- Issue #17642: add larger font sizes for classroom projection. - -- Add version to title of IDLE help window. - -- Issue #25564: In section on IDLE -- console differences, mention that - using exec means that __builtins__ is defined for each statement. - -- Issue #27821: Fix 3.6.0a3 regression that prevented custom key sets - from being selected when no custom theme was defined. - -C API ------ - -- Issue #26900: Excluded underscored names and other private API from limited API. - -- Issue #26027: Add support for path-like objects in PyUnicode_FSConverter() & - PyUnicode_FSDecoder(). - -Tests ------ - -- Issue #27427: Additional tests for the math module. Patch by Francisco Couzo. - -- Issue #27953: Skip math and cmath tests that fail on OS X 10.4 due to a - poor libm implementation of tan. - -- Issue #26040: Improve test_math and test_cmath coverage and rigour. Patch by - Jeff Allen. - -- Issue #27787: Call gc.collect() before checking each test for "dangling - threads", since the dangling threads are weak references. - -Build ------ - -- Issue #27566: Fix clean target in freeze makefile (patch by Lisa Roach) - -- Issue #27705: Update message in validate_ucrtbase.py - -- Issue #27976: Deprecate building _ctypes with the bundled copy of libffi on - non-OSX UNIX platforms. - -- Issue #27983: Cause lack of llvm-profdata tool when using clang as - required for PGO linking to be a configure time error rather than - make time when --with-optimizations is enabled. Also improve our - ability to find the llvm-profdata tool on MacOS and some Linuxes. - -- Issue #21590: Support for DTrace and SystemTap probes. - -- Issue #26307: The profile-opt build now applies PGO to the built-in modules. - -- Issue #26359: Add the --with-optimizations flag to turn on LTO and PGO build - support when available. - -- Issue #27917: Set platform triplets for Android builds. - -- Issue #25825: Update references to the $(LIBPL) installation path on AIX. - This path was changed in 3.2a4. - -- Update OS X installer to use SQLite 3.14.1 and XZ 5.2.2. - -- Issue #21122: Fix LTO builds on OS X. - -- Issue #17128: Build OS X installer with a private copy of OpenSSL. - Also provide a sample Install Certificates command script to install a - set of root certificates from the third-party certifi module. - -Tools/Demos ------------ - -- Issue #27952: Get Tools/scripts/fixcid.py working with Python 3 and the - current "re" module, avoid invalid Python backslash escapes, and fix a bug - parsing escaped C quote signs. - -Windows -------- - -- Issue #28065: Update xz dependency to 5.2.2 and build it from source. - -- Issue #25144: Ensures TargetDir is set before continuing with custom - install. - -- Issue #1602: Windows console doesn't input or print Unicode (PEP 528) - -- Issue #27781: Change file system encoding on Windows to UTF-8 (PEP 529) - -- Issue #27731: Opt-out of MAX_PATH on Windows 10 - -- Issue #6135: Adds encoding and errors parameters to subprocess. - -- Issue #27959: Adds oem encoding, alias ansi to mbcs, move aliasmbcs to - codec lookup. - -- Issue #27982: The functions of the winsound module now accept keyword - arguments. - -- Issue #20366: Build full text search support into SQLite on Windows. - -- Issue #27756: Adds new icons for Python files and processes on Windows. - Designs by Cherry Wang. - -- Issue #27883: Update sqlite to 3.14.1.0 on Windows. - - -What's New in Python 3.6.0 alpha 4? -=================================== - -*Release date: 2016-08-15* - -Core and Builtins ------------------ - -- Issue #27704: Optimized creating bytes and bytearray from byte-like objects - and iterables. Speed up to 3 times for short objects. Original patch by - Naoki Inada. - -- Issue #26823: Large sections of repeated lines in tracebacks are now - abbreviated as "[Previous line repeated {count} more times]" by the builtin - traceback rendering. Patch by Emanuel Barry. - -- Issue #27574: Decreased an overhead of parsing keyword arguments in functions - implemented with using Argument Clinic. - -- Issue #22557: Now importing already imported modules is up to 2.5 times - faster. - -- Issue #17596: Include to help with Min GW building. - -- Issue #17599: On Windows, rename the privately defined REPARSE_DATA_BUFFER - structure to avoid conflicting with the definition from Min GW. - -- Issue #27507: Add integer overflow check in bytearray.extend(). Patch by - Xiang Zhang. - -- Issue #27581: Don't rely on wrapping for overflow check in - PySequence_Tuple(). Patch by Xiang Zhang. - -- Issue #1621: Avoid signed integer overflow in list and tuple operations. - Patch by Xiang Zhang. - -- Issue #27419: Standard __import__() no longer look up "__import__" in globals - or builtins for importing submodules or "from import". Fixed a crash if - raise a warning about unabling to resolve package from __spec__ or - __package__. - -- Issue #27083: Respect the PYTHONCASEOK environment variable under Windows. - -- Issue #27514: Make having too many statically nested blocks a SyntaxError - instead of SystemError. - -- Issue #27366: Implemented PEP 487 (Simpler customization of class creation). - Upon subclassing, the __init_subclass__ classmethod is called on the base - class. Descriptors are initialized with __set_name__ after class creation. - -Library -------- - -- Issue #26027, #27524: Add PEP 519/__fspath__() support to the os and os.path - modules. Includes code from Jelle Zijlstra. - -- Issue #27598: Add Collections to collections.abc. - Patch by Ivan Levkivskyi, docs by Neil Girdhar. - -- Issue #25958: Support "anti-registration" of special methods from - various ABCs, like __hash__, __iter__ or __len__. All these (and - several more) can be set to None in an implementation class and the - behavior will be as if the method is not defined at all. - (Previously, this mechanism existed only for __hash__, to make - mutable classes unhashable.) Code contributed by Andrew Barnert and - Ivan Levkivskyi. - -- Issue #16764: Support keyword arguments to zlib.decompress(). Patch by - Xiang Zhang. - -- Issue #27736: Prevent segfault after interpreter re-initialization due - to ref count problem introduced in code for Issue #27038 in 3.6.0a3. - Patch by Xiang Zhang. - -- Issue #25628: The *verbose* and *rename* parameters for - collections.namedtuple are now keyword-only. - -- Issue #12345: Add mathematical constant tau to math and cmath. See also - PEP 628. - -- Issue #26823: traceback.StackSummary.format now abbreviates large sections of - repeated lines as "[Previous line repeated {count} more times]" (this change - then further affects other traceback display operations in the module). Patch - by Emanuel Barry. - -- Issue #27664: Add to concurrent.futures.thread.ThreadPoolExecutor() - the ability to specify a thread name prefix. - -- Issue #27181: Add geometric_mean and harmonic_mean to statistics module. - -- Issue #27573: code.interact now prints an message when exiting. - -- Issue #6422: Add autorange method to timeit.Timer objects. - -- Issue #27773: Correct some memory management errors server_hostname in - _ssl.wrap_socket(). - -- Issue #26750: unittest.mock.create_autospec() now works properly for - subclasses of property() and other data descriptors. Removes the never - publicly used, never documented unittest.mock.DescriptorTypes tuple. - -- Issue #26754: Undocumented support of general bytes-like objects - as path in compile() and similar functions is now deprecated. - -- Issue #26800: Undocumented support of general bytes-like objects - as paths in os functions is now deprecated. - -- Issue #26981: Add _order_ compatibility shim to enum.Enum for - Python 2/3 code bases. - -- Issue #27661: Added tzinfo keyword argument to datetime.combine. - -- In the curses module, raise an error if window.getstr() or window.instr() is - passed a negative value. - -- Issue #27783: Fix possible usage of uninitialized memory in - operator.methodcaller. - -- Issue #27774: Fix possible Py_DECREF on unowned object in _sre. - -- Issue #27760: Fix possible integer overflow in binascii.b2a_qp. - -- Issue #27758: Fix possible integer overflow in the _csv module for large - record lengths. - -- Issue #27568: Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the - HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates - that the script is in CGI mode. - -- Issue #7063: Remove dead code from the "array" module's slice handling. - Patch by Chuck. - -- Issue #27656: Do not assume sched.h defines any SCHED_* constants. - -- Issue #27130: In the "zlib" module, fix handling of large buffers - (typically 4 GiB) when compressing and decompressing. Previously, inputs - were limited to 4 GiB, and compression and decompression operations did not - properly handle results of 4 GiB. - -- Issue #24773: Implemented PEP 495 (Local Time Disambiguation). - -- Expose the EPOLLEXCLUSIVE constant (when it is defined) in the select module. - -- Issue #27567: Expose the EPOLLRDHUP and POLLRDHUP constants in the select - module. - -- Issue #1621: Avoid signed int negation overflow in the "audioop" module. - -- Issue #27533: Release GIL in nt._isdir - -- Issue #17711: Fixed unpickling by the persistent ID with protocol 0. - Original patch by Alexandre Vassalotti. - -- Issue #27522: Avoid an unintentional reference cycle in email.feedparser. - -- Issue #27512: Fix a segfault when os.fspath() called an __fspath__() method - that raised an exception. Patch by Xiang Zhang. - -IDLE ----- - -- Issue #27714: text_textview and test_autocomplete now pass when re-run - in the same process. This occurs when test_idle fails when run with the - -w option but without -jn. Fix warning from test_config. - -- Issue #27621: Put query response validation error messages in the query - box itself instead of in a separate massagebox. Redo tests to match. - Add Mac OSX refinements. Original patch by Mark Roseman. - -- Issue #27620: Escape key now closes Query box as cancelled. - -- Issue #27609: IDLE: tab after initial whitespace should tab, not - autocomplete. This fixes problem with writing docstrings at least - twice indented. - -- Issue #27609: Explicitly return None when there are also non-None - returns. In a few cases, reverse a condition and eliminate a return. - -- Issue #25507: IDLE no longer runs buggy code because of its tkinter imports. - Users must include the same imports required to run directly in Python. - -- Issue #27173: Add 'IDLE Modern Unix' to the built-in key sets. - Make the default key set depend on the platform. - Add tests for the changes to the config module. - -- Issue #27452: add line counter and crc to IDLE configHandler test dump. - -Tests ------ - -- Issue #25805: Skip a test in test_pkgutil as needed that doesn't work when - ``__name__ == __main__``. Patch by SilentGhost. - -- Issue #27472: Add test.support.unix_shell as the path to the default shell. - -- Issue #27369: In test_pyexpat, avoid testing an error message detail that - changed in Expat 2.2.0. - -- Issue #27594: Prevent assertion error when running test_ast with coverage - enabled: ensure code object has a valid first line number. - Patch suggested by Ivan Levkivskyi. - -Windows -------- - -- Issue #27647: Update bundled Tcl/Tk to 8.6.6. - -- Issue #27610: Adds PEP 514 metadata to Windows installer - -- Issue #27469: Adds a shell extension to the launcher so that drag and drop - works correctly. - -- Issue #27309: Enables proper Windows styles in python[w].exe manifest. - -Build ------ - -- Issue #27713: Suppress spurious build warnings when updating importlib's - bootstrap files. Patch by Xiang Zhang - -- Issue #25825: Correct the references to Modules/python.exp, which is - required on AIX. The references were accidentally changed in 3.5.0a1. - -- Issue #27453: CPP invocation in configure must use CPPFLAGS. Patch by - Chi Hsuan Yen. - -- Issue #27641: The configure script now inserts comments into the makefile - to prevent the pgen and _freeze_importlib executables from being cross- - compiled. - -- Issue #26662: Set PYTHON_FOR_GEN in configure as the Python program to be - used for file generation during the build. - -- Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X. - Also update FreedBSD version checks for the original ctype UTF-8 workaround. - - -What's New in Python 3.6.0 alpha 3? -=================================== - -*Release date: 2016-07-11* - -Core and Builtins ------------------ - -- Issue #27473: Fixed possible integer overflow in bytes and bytearray - concatenations. Patch by Xiang Zhang. - -- Issue #23034: The output of a special Python build with defined COUNT_ALLOCS, - SHOW_ALLOC_COUNT or SHOW_TRACK_COUNT macros is now off by default. It can - be re-enabled using the "-X showalloccount" option. It now outputs to stderr - instead of stdout. - -- Issue #27443: __length_hint__() of bytearray iterators no longer return a - negative integer for a resized bytearray. - -- Issue #27007: The fromhex() class methods of bytes and bytearray subclasses - now return an instance of corresponding subclass. - -Library -------- - -- Issue #26844: Fix error message for imp.find_module() to refer to 'path' - instead of 'name'. Patch by Lev Maximov. - -- Issue #23804: Fix SSL zero-length recv() calls to not block and not raise - an error about unclean EOF. - -- Issue #27466: Change time format returned by http.cookie.time2netscape, - confirming the netscape cookie format and making it consistent with - documentation. - -- Issue #21708: Deprecated dbm.dumb behavior that differs from common dbm - behavior: creating a database in 'r' and 'w' modes and modifying a database - in 'r' mode. - -- Issue #26721: Change the socketserver.StreamRequestHandler.wfile attribute - to implement BufferedIOBase. In particular, the write() method no longer - does partial writes. - -- Issue #22115: Added methods trace_add, trace_remove and trace_info in the - tkinter.Variable class. They replace old methods trace_variable, trace, - trace_vdelete and trace_vinfo that use obsolete Tcl commands and might - not work in future versions of Tcl. Fixed old tracing methods: - trace_vdelete() with wrong mode no longer break tracing, trace_vinfo() now - always returns a list of pairs of strings, tracing in the "u" mode now works. - -- Issue #26243: Only the level argument to zlib.compress() is keyword argument - now. The first argument is positional-only. - -- Issue #27038: Expose the DirEntry type as os.DirEntry. Code patch by - Jelle Zijlstra. - -- Issue #27186: Update os.fspath()/PyOS_FSPath() to check the return value of - __fspath__() to be either str or bytes. - -- Issue #18726: All optional parameters of the dump(), dumps(), - load() and loads() functions and JSONEncoder and JSONDecoder class - constructors in the json module are now keyword-only. - -- Issue #27319: Methods selection_set(), selection_add(), selection_remove() - and selection_toggle() of ttk.TreeView now allow passing multiple items as - multiple arguments instead of passing them as a tuple. Deprecated - undocumented ability of calling the selection() method with arguments. - -- Issue #27079: Fixed curses.ascii functions isblank(), iscntrl() and ispunct(). - -- Issue #27294: Numerical state in the repr for Tkinter event objects is now - represented as a combination of known flags. - -- Issue #27177: Match objects in the re module now support index-like objects - as group indices. Based on patches by Jeroen Demeyer and Xiang Zhang. - -- Issue #26754: Some functions (compile() etc) accepted a filename argument - encoded as an iterable of integers. Now only strings and byte-like objects - are accepted. - -- Issue #26536: socket.ioctl now supports SIO_LOOPBACK_FAST_PATH. Patch by - Daniel Stokes. - -- Issue #27048: Prevents distutils failing on Windows when environment - variables contain non-ASCII characters - -- Issue #27330: Fixed possible leaks in the ctypes module. - -- Issue #27238: Got rid of bare excepts in the turtle module. Original patch - by Jelle Zijlstra. - -- Issue #27122: When an exception is raised within the context being managed - by a contextlib.ExitStack() and one of the exit stack generators - catches and raises it in a chain, do not re-raise the original exception - when exiting, let the new chained one through. This avoids the PEP 479 - bug described in issue25782. - -- [Security] Issue #27278: Fix os.urandom() implementation using getrandom() on - Linux. Truncate size to INT_MAX and loop until we collected enough random - bytes, instead of casting a directly Py_ssize_t to int. - -- Issue #16864: sqlite3.Cursor.lastrowid now supports REPLACE statement. - Initial patch by Alex LordThorsen. - -- Issue #26386: Fixed ttk.TreeView selection operations with item id's - containing spaces. - -- Issue #8637: Honor a pager set by the env var MANPAGER (in preference to - one set by the env var PAGER). - -- [Security] Issue #22636: Avoid shell injection problems with - ctypes.util.find_library(). - -- Issue #16182: Fix various functions in the "readline" module to use the - locale encoding, and fix get_begidx() and get_endidx() to return code point - indexes. - -- Issue #27392: Add loop.connect_accepted_socket(). - Patch by Jim Fulton. - -IDLE ----- - -- Issue #27477: IDLE search dialogs now use ttk widgets. - -- Issue #27173: Add 'IDLE Modern Unix' to the built-in key sets. - Make the default key set depend on the platform. - Add tests for the changes to the config module. - -- Issue #27452: make command line "idle-test> python test_help.py" work. - __file__ is relative when python is started in the file's directory. - -- Issue #27452: add line counter and crc to IDLE configHandler test dump. - -- Issue #27380: IDLE: add query.py with base Query dialog and ttk widgets. - Module had subclasses SectionName, ModuleName, and HelpSource, which are - used to get information from users by configdialog and file =>Load Module. - Each subclass has itw own validity checks. Using ModuleName allows users - to edit bad module names instead of starting over. - Add tests and delete the two files combined into the new one. - -- Issue #27372: Test_idle no longer changes the locale. - -- Issue #27365: Allow non-ascii chars in IDLE NEWS.txt, for contributor names. - -- Issue #27245: IDLE: Cleanly delete custom themes and key bindings. - Previously, when IDLE was started from a console or by import, a cascade - of warnings was emitted. Patch by Serhiy Storchaka. - -- Issue #24137: Run IDLE, test_idle, and htest with tkinter default root - disabled. Fix code and tests that fail with this restriction. Fix htests to - not create a second and redundant root and mainloop. - -- Issue #27310: Fix IDLE.app failure to launch on OS X due to vestigial import. - -C API ------ - -- Issue #26754: PyUnicode_FSDecoder() accepted a filename argument encoded as - an iterable of integers. Now only strings and byte-like objects are accepted. - -Build ------ - -- Issue #28066: Fix the logic that searches build directories for generated - include files when building outside the source tree. - -- Issue #27442: Expose the Android API level that python was built against, in - sysconfig.get_config_vars() as 'ANDROID_API_LEVEL'. - -- Issue #27434: The interpreter that runs the cross-build, found in PATH, must - now be of the same feature version (e.g. 3.6) as the source being built. - -- Issue #26930: Update Windows builds to use OpenSSL 1.0.2h. - -- Issue #23968: Rename the platform directory from plat-$(MACHDEP) to - plat-$(PLATFORM_TRIPLET). - Rename the config directory (LIBPL) from config-$(LDVERSION) to - config-$(LDVERSION)-$(PLATFORM_TRIPLET). - Install the platform specifc _sysconfigdata module into the platform - directory and rename it to include the ABIFLAGS. - -- Don't use largefile support for GNU/Hurd. - -Tools/Demos ------------ - -- Issue #27332: Fixed the type of the first argument of module-level functions - generated by Argument Clinic. Patch by Petr Viktorin. - -- Issue #27418: Fixed Tools/importbench/importbench.py. - -Documentation -------------- - -- Issue #19489: Moved the search box from the sidebar to the header and footer - of each page. Patch by Ammar Askar. - -- Issue #27285: Update documentation to reflect the deprecation of ``pyvenv`` - and normalize on the term "virtual environment". Patch by Steve Piercy. - -Tests ------ - -- Issue #27027: Added test.support.is_android that is True when this is an - Android build. - - -What's New in Python 3.6.0 alpha 2? -=================================== - -*Release date: 2016-06-13* - -Core and Builtins ------------------ - -- Issue #27095: Simplified MAKE_FUNCTION and removed MAKE_CLOSURE opcodes. - Patch by Demur Rumed. - -- Issue #27190: Raise NotSupportedError if sqlite3 is older than 3.3.1. - Patch by Dave Sawyer. - -- Issue #27286: Fixed compiling BUILD_MAP_UNPACK_WITH_CALL opcode. Calling - function with generalized unpacking (PEP 448) and conflicting keyword names - could cause undefined behavior. - -- Issue #27140: Added BUILD_CONST_KEY_MAP opcode. - -- Issue #27186: Add support for os.PathLike objects to open() (part of PEP 519). - -- Issue #27066: Fixed SystemError if a custom opener (for open()) returns a - negative number without setting an exception. - -- Issue #26983: float() now always return an instance of exact float. - The deprecation warning is emitted if __float__ returns an instance of - a strict subclass of float. In a future versions of Python this can - be an error. - -- Issue #27097: Python interpreter is now about 7% faster due to optimized - instruction decoding. Based on patch by Demur Rumed. - -- Issue #26647: Python interpreter now uses 16-bit wordcode instead of bytecode. - Patch by Demur Rumed. - -- Issue #23275: Allow assigning to an empty target list in round brackets: - () = iterable. - -- Issue #27243: Update the __aiter__ protocol: instead of returning - an awaitable that resolves to an asynchronous iterator, the asynchronous - iterator should be returned directly. Doing the former will trigger a - PendingDeprecationWarning. - - -Library -------- - -- Comment out socket (SO_REUSEPORT) and posix (O_SHLOCK, O_EXLOCK) constants - exposed on the API which are not implemented on GNU/Hurd. They would not - work at runtime anyway. - -- Issue #27025: Generated names for Tkinter widgets are now more meanful - and recognizirable. - -- Issue #25455: Fixed crashes in repr of recursive ElementTree.Element and - functools.partial objects. - -- Issue #27294: Improved repr for Tkinter event objects. - -- Issue #20508: Improve exception message of IPv{4,6}Network.__getitem__. - Patch by Gareth Rees. - -- [Security] Issue #26556: Update expat to 2.1.1, fixes CVE-2015-1283. - -- [Security] Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. - Reported by Team Oststrom. - -- Issue #21386: Implement missing IPv4Address.is_global property. It was - documented since 07a5610bae9d. Initial patch by Roger Luethi. - -- Issue #27029: Removed deprecated support of universal newlines mode from - ZipFile.open(). - -- Issue #27030: Unknown escapes consisting of ``'\'`` and an ASCII letter in - regular expressions now are errors. The re.LOCALE flag now can be used - only with bytes patterns. - -- Issue #27186: Add os.PathLike support to DirEntry (part of PEP 519). - Initial patch by Jelle Zijlstra. - -- Issue #20900: distutils register command now decodes HTTP responses - correctly. Initial patch by ingrid. - -- Issue #27186: Add os.PathLike support to pathlib, removing its provisional - status (part of PEP 519). Initial patch by Dusty Phillips. - -- Issue #27186: Add support for os.PathLike objects to os.fsencode() and - os.fsdecode() (part of PEP 519). - -- Issue #27186: Introduce os.PathLike and os.fspath() (part of PEP 519). - -- A new version of typing.py provides several new classes and - features: @overload outside stubs, Reversible, DefaultDict, Text, - ContextManager, Type[], NewType(), TYPE_CHECKING, and numerous bug - fixes (note that some of the new features are not yet implemented in - mypy or other static analyzers). Also classes for PEP 492 - (Awaitable, AsyncIterable, AsyncIterator) have been added (in fact - they made it into 3.5.1 but were never mentioned). - -- Issue #25738: Stop http.server.BaseHTTPRequestHandler.send_error() from - sending a message body for 205 Reset Content. Also, don't send Content - header fields in responses that don't have a body. Patch by Susumu - Koshiba. - -- Issue #21313: Fix the "platform" module to tolerate when sys.version - contains truncated build information. - -- [Security] Issue #26839: On Linux, :func:`os.urandom` now calls - ``getrandom()`` with ``GRND_NONBLOCK`` to fall back on reading - ``/dev/urandom`` if the urandom entropy pool is not initialized yet. Patch - written by Colm Buckley. - -- Issue #23883: Added missing APIs to __all__ to match the documented APIs - for the following modules: cgi, mailbox, mimetypes, plistlib and smtpd. - Patches by Jacek Kołodziej. - -- Issue #27164: In the zlib module, allow decompressing raw Deflate streams - with a predefined zdict. Based on patch by Xiang Zhang. - -- Issue #24291: Fix wsgiref.simple_server.WSGIRequestHandler to completely - write data to the client. Previously it could do partial writes and - truncate data. Also, wsgiref.handler.ServerHandler can now handle stdout - doing partial writes, but this is deprecated. - -- Issue #21272: Use _sysconfigdata.py to initialize distutils.sysconfig. - -- Issue #19611: :mod:`inspect` now reports the implicit ``.0`` parameters - generated by the compiler for comprehension and generator expression scopes - as if they were positional-only parameters called ``implicit0``. - Patch by Jelle Zijlstra. - -- Issue #26809: Add ``__all__`` to :mod:`string`. Patch by Emanuel Barry. - -- Issue #26373: subprocess.Popen.communicate now correctly ignores - BrokenPipeError when the child process dies before .communicate() - is called in more/all circumstances. - -- signal, socket, and ssl module IntEnum constant name lookups now return a - consistent name for values having multiple names. Ex: signal.Signals(6) - now refers to itself as signal.SIGALRM rather than flipping between that - and signal.SIGIOT based on the interpreter's hash randomization seed. - -- Issue #27167: Clarify the subprocess.CalledProcessError error message text - when the child process died due to a signal. - -- Issue #25931: Don't define socketserver.Forking* names on platforms such - as Windows that do not support os.fork(). - -- Issue #21776: distutils.upload now correctly handles HTTPError. - Initial patch by Claudiu Popa. - -- Issue #26526: Replace custom parse tree validation in the parser - module with a simple DFA validator. - -- Issue #27114: Fix SSLContext._load_windows_store_certs fails with - PermissionError - -- Issue #18383: Avoid creating duplicate filters when using filterwarnings - and simplefilter. Based on patch by Alex Shkop. - -- Issue #23026: winreg.QueryValueEx() now return an integer for REG_QWORD type. - -- Issue #26741: subprocess.Popen destructor now emits a ResourceWarning warning - if the child process is still running. - -- Issue #27056: Optimize pickle.load() and pickle.loads(), up to 10% faster - to deserialize a lot of small objects. - -- Issue #21271: New keyword only parameters in reset_mock call. - -IDLE ----- - -- Issue #5124: Paste with text selected now replaces the selection on X11. - This matches how paste works on Windows, Mac, most modern Linux apps, - and ttk widgets. Original patch by Serhiy Storchaka. - -- Issue #24750: Switch all scrollbars in IDLE to ttk versions. - Where needed, minimal tests are added to cover changes. - -- Issue #24759: IDLE requires tk 8.5 and availability ttk widgets. - Delete now unneeded tk version tests and code for older versions. - Add test for IDLE syntax colorizoer. - -- Issue #27239: idlelib.macosx.isXyzTk functions initialize as needed. - -- Issue #27262: move Aqua unbinding code, which enable context menus, to maxosx. - -- Issue #24759: Make clear in idlelib.idle_test.__init__ that the directory - is a private implementation of test.test_idle and tool for maintainers. - -- Issue #27196: Stop 'ThemeChanged' warnings when running IDLE tests. - These persisted after other warnings were suppressed in #20567. - Apply Serhiy Storchaka's update_idletasks solution to four test files. - Record this additional advice in idle_test/README.txt - -- Issue #20567: Revise idle_test/README.txt with advice about avoiding - tk warning messages from tests. Apply advice to several IDLE tests. - -- Issue #24225: Update idlelib/README.txt with new file names - and event handlers. - -- Issue #27156: Remove obsolete code not used by IDLE. Replacements: - 1. help.txt, replaced by help.html, is out-of-date and should not be used. - Its dedicated viewer has be replaced by the html viewer in help.py. - 2. ``import idlever; I = idlever.IDLE_VERSION`` is the same as - ``import sys; I = version[:version.index(' ')]`` - 3. After ``ob = stackviewer.VariablesTreeItem(*args)``, - ``ob.keys() == list(ob.object.keys)``. - 4. In macosc, runningAsOSXAPP == isAquaTk; idCarbonAquaTk == isCarbonTk - -- Issue #27117: Make colorizer htest and turtledemo work with dark themes. - Move code for configuring text widget colors to a new function. - -- Issue #24225: Rename many `idlelib/*.py` and `idle_test/test_*.py` files. - Edit files to replace old names with new names when the old name - referred to the module rather than the class it contained. - See the issue and IDLE section in What's New in 3.6 for more. - -- Issue #26673: When tk reports font size as 0, change to size 10. - Such fonts on Linux prevented the configuration dialog from opening. - -- Issue #21939: Add test for IDLE's percolator. - Original patch by Saimadhav Heblikar. - -- Issue #21676: Add test for IDLE's replace dialog. - Original patch by Saimadhav Heblikar. - -- Issue #18410: Add test for IDLE's search dialog. - Original patch by Westley Martínez. - -- Issue #21703: Add test for undo delegator. Patch mostly by - Saimadhav Heblikar . - -- Issue #27044: Add ConfigDialog.remove_var_callbacks to stop memory leaks. - -- Issue #23977: Add more asserts to test_delegator. - -Documentation -------------- - -- Issue #16484: Change the default PYTHONDOCS URL to "https:", and fix the - resulting links to use lowercase. Patch by Sean Rodman, test by Kaushik - Nadikuditi. - -- Issue #24136: Document the new PEP 448 unpacking syntax of 3.5. - -- Issue #22558: Add remaining doc links to source code for Python-coded modules. - Patch by Yoni Lavi. - -Tests ------ - -- Issue #25285: regrtest now uses subprocesses when the -j1 command line option - is used: each test file runs in a fresh child process. Before, the -j1 option - was ignored. - -- Issue #25285: Tools/buildbot/test.bat script now uses -j1 by default to run - each test file in fresh child process. - -Windows -------- - -- Issue #27064: The py.exe launcher now defaults to Python 3. - The Windows launcher ``py.exe`` no longer prefers an installed - Python 2 version over Python 3 by default when used interactively. - -Build ------ - -- Issue #27229: Fix the cross-compiling pgen rule for in-tree builds. Patch - by Xavier de Gaye. - -- Issue #26930: Update OS X 10.5+ 32-bit-only installer to build - and link with OpenSSL 1.0.2h. - -Misc ----- - -- Issue #17500, and https://github.com/python/pythondotorg/issues/945: Remove - unused and outdated icons. - -C API ------ - -- Issue #27186: Add the PyOS_FSPath() function (part of PEP 519). - -- Issue #26282: PyArg_ParseTupleAndKeywords() now supports positional-only - parameters. - -Tools/Demos ------------ - -- Issue #26282: Argument Clinic now supports positional-only and keyword - parameters in the same function. - - -What's New in Python 3.6.0 alpha 1? -=================================== - -Release date: 2016-05-16 - -Core and Builtins ------------------ - -- Issue #20041: Fixed TypeError when frame.f_trace is set to None. - Patch by Xavier de Gaye. - -- Issue #26168: Fixed possible refleaks in failing Py_BuildValue() with the "N" - format unit. - -- Issue #26991: Fix possible refleak when creating a function with annotations. - -- Issue #27039: Fixed bytearray.remove() for values greater than 127. Based on - patch by Joe Jevnik. - -- Issue #23640: int.from_bytes() no longer bypasses constructors for subclasses. - -- Issue #27005: Optimized the float.fromhex() class method for exact float. - It is now 2 times faster. - -- Issue #18531: Single var-keyword argument of dict subtype was passed - unscathed to the C-defined function. Now it is converted to exact dict. - -- Issue #26811: gc.get_objects() no longer contains a broken tuple with NULL - pointer. - -- Issue #20120: Use RawConfigParser for .pypirc parsing, - removing support for interpolation unintentionally added - with move to Python 3. Behavior no longer does any - interpolation in .pypirc files, matching behavior in Python - 2.7 and Setuptools 19.0. - -- Issue #26249: Memory functions of the :c:func:`PyMem_Malloc` domain - (:c:data:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc allocator ` - rather than system :c:func:`malloc`. Applications calling - :c:func:`PyMem_Malloc` without holding the GIL can now crash: use - ``PYTHONMALLOC=debug`` environment variable to validate the usage of memory - allocators in your application. - -- Issue #26802: Optimize function calls only using unpacking like - ``func(*tuple)`` (no other positional argument, no keyword): avoid copying - the tuple. Patch written by Joe Jevnik. - -- Issue #26659: Make the builtin slice type support cycle collection. - -- Issue #26718: super.__init__ no longer leaks memory if called multiple times. - NOTE: A direct call of super.__init__ is not endorsed! - -- Issue #27138: Fix the doc comment for FileFinder.find_spec(). - -- Issue #27147: Mention PEP 420 in the importlib docs. - -- Issue #25339: PYTHONIOENCODING now has priority over locale in setting the - error handler for stdin and stdout. - -- Issue #26494: Fixed crash on iterating exhausting iterators. - Affected classes are generic sequence iterators, iterators of str, bytes, - bytearray, list, tuple, set, frozenset, dict, OrderedDict, corresponding - views and os.scandir() iterator. - -- Issue #26574: Optimize ``bytes.replace(b'', b'.')`` and - ``bytearray.replace(b'', b'.')``. Patch written by Josh Snider. - -- Issue #26581: If coding cookie is specified multiple times on a line in - Python source code file, only the first one is taken to account. - -- Issue #19711: Add tests for reloading namespace packages. - -- Issue #21099: Switch applicable importlib tests to use PEP 451 API. - -- Issue #26563: Debug hooks on Python memory allocators now raise a fatal - error if functions of the :c:func:`PyMem_Malloc` family are called without - holding the GIL. - -- Issue #26564: On error, the debug hooks on Python memory allocators now use - the :mod:`tracemalloc` module to get the traceback where a memory block was - allocated. - -- Issue #26558: The debug hooks on Python memory allocator - :c:func:`PyObject_Malloc` now detect when functions are called without - holding the GIL. - -- Issue #26516: Add :envvar:`PYTHONMALLOC` environment variable to set the - Python memory allocators and/or install debug hooks. - -- Issue #26516: The :c:func:`PyMem_SetupDebugHooks` function can now also be - used on Python compiled in release mode. - -- Issue #26516: The :envvar:`PYTHONMALLOCSTATS` environment variable can now - also be used on Python compiled in release mode. It now has no effect if - set to an empty string. - -- Issue #26516: In debug mode, debug hooks are now also installed on Python - memory allocators when Python is configured without pymalloc. - -- Issue #26464: Fix str.translate() when string is ASCII and first replacements - removes character, but next replacement uses a non-ASCII character or a - string longer than 1 character. Regression introduced in Python 3.5.0. - -- Issue #22836: Ensure exception reports from PyErr_Display() and - PyErr_WriteUnraisable() are sensible even when formatting them produces - secondary errors. This affects the reports produced by - sys.__excepthook__() and when __del__() raises an exception. - -- Issue #26302: Correct behavior to reject comma as a legal character for - cookie names. - -- Issue #26136: Upgrade the warning when a generator raises StopIteration - from PendingDeprecationWarning to DeprecationWarning. Patch by Anish - Shah. - -- Issue #26204: The compiler now ignores all constant statements: bytes, str, - int, float, complex, name constants (None, False, True), Ellipsis - and ast.Constant; not only str and int. For example, ``1.0`` is now ignored - in ``def f(): 1.0``. - -- Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and - Daniel Urban. - -- Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used - by external AST optimizers, but the compiler does not emit directly such - node. - -- Issue #23601: Sped-up allocation of dict key objects by using Python's - small object allocator. (Contributed by Julian Taylor.) - -- Issue #18018: Import raises ImportError instead of SystemError if a relative - import is attempted without a known parent package. - -- Issue #25843: When compiling code, don't merge constants if they are equal - but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` - is now correctly compiled to two different functions: ``f1()`` returns ``1`` - (``int``) and ``f2()`` returns ``1.0`` (``float``), even if ``1`` and ``1.0`` - are equal. - -- Issue #26107: The format of the ``co_lnotab`` attribute of code objects - changes to support negative line number delta. - -- Issue #26154: Add a new private _PyThreadState_UncheckedGet() function to get - the current Python thread state, but don't issue a fatal error if it is NULL. - This new function must be used instead of accessing directly the - _PyThreadState_Current variable. The variable is no more exposed since - Python 3.5.1 to hide the exact implementation of atomic C types, to avoid - compiler issues. - -- Issue #25791: If __package__ != __spec__.parent or if neither __package__ or - __spec__ are defined then ImportWarning is raised. - -- Issue #22995: [UPDATE] Comment out the one of the pickleability tests in - _PyObject_GetState() due to regressions observed in Cython-based projects. - -- Issue #25961: Disallowed null characters in the type name. - -- Issue #25973: Fix segfault when an invalid nonlocal statement binds a name - starting with two underscores. - -- Issue #22995: Instances of extension types with a state that aren't - subclasses of list or dict and haven't implemented any pickle-related - methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, - or __getstate__), can no longer be pickled. Including memoryview. - -- Issue #20440: Massive replacing unsafe attribute setting code with special - macro Py_SETREF. - -- Issue #25766: Special method __bytes__() now works in str subclasses. - -- Issue #25421: __sizeof__ methods of builtin types now use dynamic basic size. - This allows sys.getsize() to work correctly with their subclasses with - __slots__ defined. - -- Issue #25709: Fixed problem with in-place string concatenation and utf-8 - cache. - -- Issue #5319: New Py_FinalizeEx() API allowing Python to set an exit status - of 120 on failure to flush buffered streams. - -- Issue #25485: telnetlib.Telnet is now a context manager. - -- Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside - __getattr__. - -- Issue #24731: Fixed crash on converting objects with special methods - __bytes__, __trunc__, and __float__ returning instances of subclasses of - bytes, int, and float to subclasses of bytes, int, and float correspondingly. - -- Issue #25630: Fix a possible segfault during argument parsing in functions - that accept filesystem paths. - -- Issue #23564: Fixed a partially broken sanity check in the _posixsubprocess - internals regarding how fds_to_pass were passed to the child. The bug had - no actual impact as subprocess.py already avoided it. - -- Issue #25388: Fixed tokenizer crash when processing undecodable source code - with a null byte. - -- Issue #25462: The hash of the key now is calculated only once in most - operations in C implementation of OrderedDict. - -- Issue #22995: Default implementation of __reduce__ and __reduce_ex__ now - rejects builtin types with not defined __new__. - -- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec() - and eval() are passed bytes-like objects. These objects are not - necessarily terminated by a null byte, but the functions assumed they were. - -- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node - when compiling AST from Python objects. - -- Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that - was mutated by direct calls of dict methods. - -- Issue #25449: Iterating OrderedDict with keys with unstable hash now raises - KeyError in C implementations as well as in Python implementation. - -- Issue #25395: Fixed crash when highly nested OrderedDict structures were - garbage collected. - -- Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now - between 2x and 3.5x faster. - -- Issue #25399: Optimize bytearray % args using the new private _PyBytesWriter - API. Formatting is now between 2.5 and 5 times faster. - -- Issue #25274: sys.setrecursionlimit() now raises a RecursionError if the new - recursion limit is too low depending at the current recursion depth. Modify - also the "lower-water mark" formula to make it monotonic. This mark is used - to decide when the overflowed flag of the thread state is reset. - -- Issue #24402: Fix input() to prompt to the redirected stdout when - sys.stdout.fileno() fails. - -- Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API. - Formatting is now up to 2 times faster. - -- Issue #24806: Prevent builtin types that are not allowed to be subclassed from - being subclassed through multiple inheritance. - -- Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error - handlers: ``ignore``, ``replace`` and ``surrogateescape``. - -- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data. - -- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error - handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. - Patch co-written with Serhiy Storchaka. - -- Issue #25280: Import trace messages emitted in verbose (-v) mode are no - longer formatted twice. - -- Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` - error handler: the encoders are now up to 3 times as fast. Initial patch - written by Serhiy Storchaka. - -- Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the - getrandom() function instead of the getentropy() function. The getentropy() - function is blocking to generate very good quality entropy, os.urandom() - doesn't need such high-quality entropy. - -- Issue #9232: Modify Python's grammar to allow trailing commas in the - argument list of a function declaration. For example, "def f(\*, a = - 3,): pass" is now legal. Patch from Mark Dickinson. - -- Issue #24965: Implement PEP 498 "Literal String Interpolation". This - allows you to embed expressions inside f-strings, which are - converted to normal strings at run time. Given x=3, then - f'value={x}' == 'value=3'. Patch by Eric V. Smith. - -- Issue #26478: Fix semantic bugs when using binary operators with dictionary - views and tuples. - -- Issue #26171: Fix possible integer overflow and heap corruption in - zipimporter.get_data(). - -- Issue #25660: Fix TAB key behaviour in REPL with readline. - -- Issue #26288: Optimize PyLong_AsDouble. - -- Issues #26289 and #26315: Optimize floor and modulo division for - single-digit longs. Microbenchmarks show 2-2.5x improvement. Built-in - 'divmod' function is now also ~10% faster. - -- Issue #25887: Raise a RuntimeError when a coroutine object is awaited - more than once. - -Library -------- - -- Issue #27057: Fix os.set_inheritable() on Android, ioctl() is blocked by - SELinux and fails with EACCESS. The function now falls back to fcntl(). - Patch written by Michał Bednarski. - -- Issue #27014: Fix infinite recursion using typing.py. Thanks to Kalle Tuure! - -- Issue #27031: Removed dummy methods in Tkinter widget classes: tk_menuBar() - and tk_bindForTraversal(). - -- Issue #14132: Fix urllib.request redirect handling when the target only has - a query string. Original fix by Ján Janech. - -- Issue #17214: The "urllib.request" module now percent-encodes non-ASCII - bytes found in redirect target URLs. Some servers send Location header - fields with non-ASCII bytes, but "http.client" requires the request target - to be ASCII-encodable, otherwise a UnicodeEncodeError is raised. Based on - patch by Christian Heimes. - -- Issue #27033: The default value of the decode_data parameter for - smtpd.SMTPChannel and smtpd.SMTPServer constructors is changed to False. - -- Issue #27034: Removed deprecated class asynchat.fifo. - -- Issue #26870: Added readline.set_auto_history(), which can stop entries - being automatically added to the history list. Based on patch by Tyler - Crompton. - -- Issue #26039: zipfile.ZipFile.open() can now be used to write data into a ZIP - file, as well as for extracting data. Patch by Thomas Kluyver. - -- Issue #26892: Honor debuglevel flag in urllib.request.HTTPHandler. Patch - contributed by Chi Hsuan Yen. - -- Issue #22274: In the subprocess module, allow stderr to be redirected to - stdout even when stdout is not redirected. Patch by Akira Li. - -- Issue #26807: mock_open 'files' no longer error on readline at end of file. - Patch from Yolanda Robla. - -- Issue #25745: Fixed leaking a userptr in curses panel destructor. - -- Issue #26977: Removed unnecessary, and ignored, call to sum of squares helper - in statistics.pvariance. - -- Issue #26002: Use bisect in statistics.median instead of a linear search. - Patch by Upendra Kuma. - -- Issue #25974: Make use of new Decimal.as_integer_ratio() method in statistics - module. Patch by Stefan Krah. - -- Issue #26996: Add secrets module as described in PEP 506. - -- Issue #26881: The modulefinder module now supports extended opcode arguments. - -- Issue #23815: Fixed crashes related to directly created instances of types in - _tkinter and curses.panel modules. - -- Issue #17765: weakref.ref() no longer silently ignores keyword arguments. - Patch by Georg Brandl. - -- Issue #26873: xmlrpc now raises ResponseError on unsupported type tags - instead of silently return incorrect result. - -- Issue #26915: The __contains__ methods in the collections ABCs now check - for identity before checking equality. This better matches the behavior - of the concrete classes, allows sensible handling of NaNs, and makes it - easier to reason about container invariants. - -- Issue #26711: Fixed the comparison of plistlib.Data with other types. - -- Issue #24114: Fix an uninitialized variable in `ctypes.util`. - - The bug only occurs on SunOS when the ctypes implementation searches - for the `crle` program. Patch by Xiang Zhang. Tested on SunOS by - Kees Bos. - -- Issue #26864: In urllib.request, change the proxy bypass host checking - against no_proxy to be case-insensitive, and to not match unrelated host - names that happen to have a bypassed hostname as a suffix. Patch by Xiang - Zhang. - -- Issue #24902: Print server URL on http.server startup. Initial patch by - Felix Kaiser. - -- Issue #25788: fileinput.hook_encoded() now supports an "errors" argument - for passing to open. Original patch by Joseph Hackman. - -- Issue #26634: recursive_repr() now sets __qualname__ of wrapper. Patch by - Xiang Zhang. - -- Issue #26804: urllib.request will prefer lower_case proxy environment - variables over UPPER_CASE or Mixed_Case ones. Patch contributed by Hans-Peter - Jansen. - -- Issue #26837: assertSequenceEqual() now correctly outputs non-stringified - differing items (like bytes in the -b mode). This affects assertListEqual() - and assertTupleEqual(). - -- Issue #26041: Remove "will be removed in Python 3.7" from deprecation - messages of platform.dist() and platform.linux_distribution(). - Patch by Kumaripaba Miyurusara Athukorala. - -- Issue #26822: itemgetter, attrgetter and methodcaller objects no longer - silently ignore keyword arguments. - -- Issue #26733: Disassembling a class now disassembles class and static methods. - Patch by Xiang Zhang. - -- Issue #26801: Fix error handling in :func:`shutil.get_terminal_size`, catch - :exc:`AttributeError` instead of :exc:`NameError`. Patch written by Emanuel - Barry. - -- Issue #24838: tarfile's ustar and gnu formats now correctly calculate name - and link field limits for multibyte character encodings like utf-8. - -- [Security] Issue #26657: Fix directory traversal vulnerability with - http.server on Windows. This fixes a regression that was introduced in - 3.3.4rc1 and 3.4.0rc1. Based on patch by Philipp Hagemeister. - -- Issue #26717: Stop encoding Latin-1-ized WSGI paths with UTF-8. Patch by - Anthony Sottile. - -- Issue #26782: Add STARTUPINFO to subprocess.__all__ on Windows. - -- Issue #26404: Add context manager to socketserver. Patch by Aviv Palivoda. - -- Issue #26735: Fix :func:`os.urandom` on Solaris 11.3 and newer when reading - more than 1,024 bytes: call ``getrandom()`` multiple times with a limit of - 1024 bytes per call. - -- Issue #26585: Eliminate http.server._quote_html() and use - html.escape(quote=False). Patch by Xiang Zhang. - -- Issue #26685: Raise OSError if closing a socket fails. - -- Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. - -- Issue #13952: Add .csv to mimetypes.types_map. Patch by Geoff Wilson. - -- Issue #26587: the site module now allows .pth files to specify files to be - added to sys.path (e.g. zip files). - -- Issue #25609: Introduce contextlib.AbstractContextManager and - typing.ContextManager. - -- Issue #26709: Fixed Y2038 problem in loading binary PLists. - -- Issue #23735: Handle terminal resizing with Readline 6.3+ by installing our - own SIGWINCH handler. Patch by Eric Price. - -- Issue #25951: Change SSLSocket.sendall() to return None, as explicitly - documented for plain socket objects. Patch by Aviv Palivoda. - -- Issue #26586: In http.server, respond with "413 Request header fields too - large" if there are too many header fields to parse, rather than killing - the connection and raising an unhandled exception. Patch by Xiang Zhang. - -- Issue #26676: Added missing XMLPullParser to ElementTree.__all__. - -- Issue #22854: Change BufferedReader.writable() and - BufferedWriter.readable() to always return False. - -- Issue #26492: Exhausted iterator of array.array now conforms with the behavior - of iterators of other mutable sequences: it lefts exhausted even if iterated - array is extended. - -- Issue #26641: doctest.DocFileTest and doctest.testfile() now support - packages (module splitted into multiple directories) for the package - parameter. - -- Issue #25195: Fix a regression in mock.MagicMock. _Call is a subclass of - tuple (changeset 3603bae63c13 only works for classes) so we need to - implement __ne__ ourselves. Patch by Andrew Plummer. - -- Issue #26644: Raise ValueError rather than SystemError when a negative - length is passed to SSLSocket.recv() or read(). - -- Issue #23804: Fix SSL recv(0) and read(0) methods to return zero bytes - instead of up to 1024. - -- Issue #26616: Fixed a bug in datetime.astimezone() method. - -- Issue #26637: The :mod:`importlib` module now emits an :exc:`ImportError` - rather than a :exc:`TypeError` if :func:`__import__` is tried during the - Python shutdown process but :data:`sys.path` is already cleared (set to - ``None``). - -- Issue #21925: :func:`warnings.formatwarning` now catches exceptions when - calling :func:`linecache.getline` and - :func:`tracemalloc.get_object_traceback` to be able to log - :exc:`ResourceWarning` emitted late during the Python shutdown process. - -- Issue #23848: On Windows, faulthandler.enable() now also installs an - exception handler to dump the traceback of all Python threads on any Windows - exception, not only on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). - -- Issue #26530: Add C functions :c:func:`_PyTraceMalloc_Track` and - :c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the - :mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get - the traceback of an object. - -- Issue #26588: The _tracemalloc now supports tracing memory allocations of - multiple address spaces (domains). - -- Issue #24266: Ctrl+C during Readline history search now cancels the search - mode when compiled with Readline 7. - -- Issue #26590: Implement a safe finalizer for the _socket.socket type. It now - releases the GIL to close the socket. - -- Issue #18787: spwd.getspnam() now raises a PermissionError if the user - doesn't have privileges. - -- Issue #26560: Avoid potential ValueError in BaseHandler.start_response. - Initial patch by Peter Inglesby. - -- Issue #26567: Add a new function :c:func:`PyErr_ResourceWarning` function to - pass the destroyed object. Add a *source* attribute to - :class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses - tracemalloc to get the traceback where source object was allocated. - -- [Security] Issue #26313: ssl.py _load_windows_store_certs fails if windows - cert store is empty. Patch by Baji. - -- Issue #26569: Fix :func:`pyclbr.readmodule` and :func:`pyclbr.readmodule_ex` - to support importing packages. - -- Issue #26499: Account for remaining Content-Length in - HTTPResponse.readline() and read1(). Based on patch by Silent Ghost. - Also document that HTTPResponse now supports these methods. - -- Issue #25320: Handle sockets in directories unittest discovery is scanning. - Patch from Victor van den Elzen. - -- Issue #16181: cookiejar.http2time() now returns None if year is higher than - datetime.MAXYEAR. - -- Issue #26513: Fixes platform module detection of Windows Server - -- Issue #23718: Fixed parsing time in week 0 before Jan 1. Original patch by - Tamás Bence Gedai. - -- Issue #26323: Add Mock.assert_called() and Mock.assert_called_once() - methods to unittest.mock. Patch written by Amit Saha. - -- Issue #20589: Invoking Path.owner() and Path.group() on Windows now raise - NotImplementedError instead of ImportError. - -- Issue #26177: Fixed the keys() method for Canvas and Scrollbar widgets. - -- Issue #15068: Got rid of excessive buffering in fileinput. - The bufsize parameter is now deprecated and ignored. - -- Issue #19475: Added an optional argument timespec to the datetime - isoformat() method to choose the precision of the time component. - -- Issue #2202: Fix UnboundLocalError in - AbstractDigestAuthHandler.get_algorithm_impls. Initial patch by Mathieu - Dupuy. - -- Issue #26167: Minimized overhead in copy.copy() and copy.deepcopy(). - Optimized copying and deepcopying bytearrays, NotImplemented, slices, - short lists, tuples, dicts, sets. - -- Issue #25718: Fixed pickling and copying the accumulate() iterator with - total is None. - -- Issue #26475: Fixed debugging output for regular expressions with the (?x) - flag. - -- Issue #26482: Allowed pickling recursive dequeues. - -- Issue #26335: Make mmap.write() return the number of bytes written like - other write methods. Patch by Jakub Stasiak. - -- Issue #26457: Fixed the subnets() methods in IP network classes for the case - when resulting prefix length is equal to maximal prefix length. - Based on patch by Xiang Zhang. - -- Issue #26385: Remove the file if the internal open() call in - NamedTemporaryFile() fails. Patch by Silent Ghost. - -- Issue #26402: Fix XML-RPC client to retry when the server shuts down a - persistent connection. This was a regression related to the new - http.client.RemoteDisconnected exception in 3.5.0a4. - -- Issue #25913: Leading ``<~`` is optional now in base64.a85decode() with - adobe=True. Patch by Swati Jaiswal. - -- Issue #26186: Remove an invalid type check in importlib.util.LazyLoader. - -- Issue #26367: importlib.__import__() raises ImportError like - builtins.__import__() when ``level`` is specified but without an accompanying - package specified. - -- Issue #26309: In the "socketserver" module, shut down the request (closing - the connected socket) when verify_request() returns false. Patch by Aviv - Palivoda. - -- Issue #23430: Change the socketserver module to only catch exceptions - raised from a request handler that are derived from Exception (instead of - BaseException). Therefore SystemExit and KeyboardInterrupt no longer - trigger the handle_error() method, and will now to stop a single-threaded - server. - -- [Security] Issue #25939: On Windows open the cert store readonly in - ssl.enum_certificates. - -- Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. - -- Issue #25994: Added the close() method and the support of the context manager - protocol for the os.scandir() iterator. - -- Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. - -- Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv - Palivoda. - -- Issue #26117: The os.scandir() iterator now closes file descriptor not only - when the iteration is finished, but when it was failed with error. - -- Issue #25949: __dict__ for an OrderedDict instance is now created only when - needed. - -- Issue #25911: Restored support of bytes paths in os.walk() on Windows. - -- Issue #26045: Add UTF-8 suggestion to error message when posting a - non-Latin-1 string with http.client. - -- Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). - Patch by Thomas Kluyver. - -- Issue #12923: Reset FancyURLopener's redirect counter even if there is an - exception. Based on patches by Brian Brazil and Daniel Rocco. - -- Issue #25945: Fixed a crash when unpickle the functools.partial object with - wrong state. Fixed a leak in failed functools.partial constructor. - "args" and "keywords" attributes of functools.partial have now always types - tuple and dict correspondingly. - -- Issue #26202: copy.deepcopy() now correctly copies range() objects with - non-atomic attributes. - -- Issue #23076: Path.glob() now raises a ValueError if it's called with an - invalid pattern. Patch by Thomas Nyberg. - -- Issue #19883: Fixed possible integer overflows in zipimport. - -- Issue #26227: On Windows, getnameinfo(), gethostbyaddr() and - gethostbyname_ex() functions of the socket module now decode the hostname - from the ANSI code page rather than UTF-8. - -- Issue #26099: The site module now writes an error into stderr if - sitecustomize module can be imported but executing the module raise an - ImportError. Same change for usercustomize. - -- Issue #26147: xmlrpc now works with strings not encodable with used - non-UTF-8 encoding. - -- Issue #25935: Garbage collector now breaks reference loops with OrderedDict. - -- Issue #16620: Fixed AttributeError in msilib.Directory.glob(). - -- Issue #26013: Added compatibility with broken protocol 2 pickles created - in old Python 3 versions (3.4.3 and lower). - -- Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). - -- Issue #25850: Use cross-compilation by default for 64-bit Windows. - -- Issue #25822: Add docstrings to the fields of urllib.parse results. - Patch contributed by Swati Jaiswal. - -- Issue #22642: Convert trace module option parsing mechanism to argparse. - Patch contributed by SilentGhost. - -- Issue #24705: Fix sysconfig._parse_makefile not expanding ${} vars - appearing before $() vars. - -- Issue #26069: Remove the deprecated apis in the trace module. - -- Issue #22138: Fix mock.patch behavior when patching descriptors. Restore - original values after patching. Patch contributed by Sean McCully. - -- Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode - option if it is safe to do so. - -- Issue #26012: Don't traverse into symlinks for ** pattern in - pathlib.Path.[r]glob(). - -- Issue #24120: Ignore PermissionError when traversing a tree with - pathlib.Path.[r]glob(). Patch by Ulrich Petri. - -- Issue #21815: Accept ] characters in the data portion of imap responses, - in order to handle the flags with square brackets accepted and produced - by servers such as gmail. - -- Issue #25447: fileinput now uses sys.stdin as-is if it does not have a - buffer attribute (restores backward compatibility). - -- Issue #25971: Optimized creating Fractions from floats by 2 times and from - Decimals by 3 times. - -- Issue #25802: Document as deprecated the remaining implementations of - importlib.abc.Loader.load_module(). - -- Issue #25928: Add Decimal.as_integer_ratio(). - -- Issue #25447: Copying the lru_cache() wrapper object now always works, - independently from the type of the wrapped object (by returning the original - object unchanged). - -- Issue #25768: Have the functions in compileall return booleans instead of - ints and add proper documentation and tests for the return values. - -- Issue #24103: Fixed possible use after free in ElementTree.XMLPullParser. - -- Issue #25860: os.fwalk() no longer skips remaining directories when error - occurs. Original patch by Samson Lee. - -- Issue #25914: Fixed and simplified OrderedDict.__sizeof__. - -- Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. - -- Issue #25873: Optimized iterating ElementTree. Iterating elements - Element.iter() is now 40% faster, iterating text Element.itertext() - is now up to 2.5 times faster. - -- Issue #25902: Fixed various refcount issues in ElementTree iteration. - -- Issue #22227: The TarFile iterator is reimplemented using generator. - This implementation is simpler that using class. - -- Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. - Optimized ElementTree parsing; it is now 10% faster. - -- Issue #25761: Improved detecting errors in broken pickle data. - -- Issue #25717: Restore the previous behaviour of tolerating most fstat() - errors when opening files. This was a regression in 3.5a1, and stopped - anonymous temporary files from working in special cases. - -- Issue #24903: Fix regression in number of arguments compileall accepts when - '-d' is specified. The check on the number of arguments has been dropped - completely as it never worked correctly anyway. - -- Issue #25764: In the subprocess module, preserve any exception caused by - fork() failure when preexec_fn is used. - -- Issue #25771: Tweak the exception message for importlib.util.resolve_name() - when 'package' isn't specified but necessary. - -- Issue #6478: _strptime's regexp cache now is reset after changing timezone - with time.tzset(). - -- Issue #14285: When executing a package with the "python -m package" option, - and package initialization fails, a proper traceback is now reported. The - "runpy" module now lets exceptions from package initialization pass back to - the caller, rather than raising ImportError. - -- Issue #19771: Also in runpy and the "-m" option, omit the irrelevant - message ". . . is a package and cannot be directly executed" if the package - could not even be initialized (e.g. due to a bad ``*.pyc`` file). - -- Issue #25177: Fixed problem with the mean of very small and very large - numbers. As a side effect, statistics.mean and statistics.variance should - be significantly faster. - -- Issue #25718: Fixed copying object with state with boolean value is false. - -- Issue #10131: Fixed deep copying of minidom documents. Based on patch - by Marian Ganisin. - -- Issue #7990: dir() on ElementTree.Element now lists properties: "tag", - "text", "tail" and "attrib". Original patch by Santoso Wijaya. - -- Issue #25725: Fixed a reference leak in pickle.loads() when unpickling - invalid data including tuple instructions. - -- Issue #25663: In the Readline completer, avoid listing duplicate global - names, and search the global namespace before searching builtins. - -- Issue #25688: Fixed file leak in ElementTree.iterparse() raising an error. - -- Issue #23914: Fixed SystemError raised by unpickler on broken pickle data. - -- Issue #25691: Fixed crash on deleting ElementTree.Element attributes. - -- Issue #25624: ZipFile now always writes a ZIP_STORED header for directory - entries. Patch by Dingyuan Wang. - -- Issue #25626: Change three zlib functions to accept sizes that fit in - Py_ssize_t, but internally cap those sizes to UINT_MAX. This resolves a - regression in 3.5 where GzipFile.read() failed to read chunks larger than 2 - or 4 GiB. The change affects the zlib.Decompress.decompress() max_length - parameter, the zlib.decompress() bufsize parameter, and the - zlib.Decompress.flush() length parameter. - -- Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) - when the OS gives priority to errors such as EACCES over EEXIST. - -- Issue #25593: Change semantics of EventLoop.stop() in asyncio. - -- Issue #6973: When we know a subprocess.Popen process has died, do - not allow the send_signal(), terminate(), or kill() methods to do - anything as they could potentially signal a different process. - -- Issue #23883: Added missing APIs to __all__ to match the documented APIs - for the following modules: calendar, csv, enum, fileinput, ftplib, logging, - optparse, tarfile, threading and wave. Also added a - test.support.check__all__() helper. Patches by Jacek Kołodziej, Mauro - S. M. Rodrigues and Joel Taddei. - -- Issue #25590: In the Readline completer, only call getattr() once per - attribute. Also complete names of attributes such as properties and slots - which are listed by dir() but not yet created on an instance. - -- Issue #25498: Fix a crash when garbage-collecting ctypes objects created - by wrapping a memoryview. This was a regression made in 3.5a1. Based - on patch by Eryksun. - -- Issue #25584: Added "escape" to the __all__ list in the glob module. - -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. - -- Issue #25446: Fix regression in smtplib's AUTH LOGIN support. - -- Issue #18010: Fix the pydoc web server's module search function to handle - exceptions from importing packages. - -- Issue #25554: Got rid of circular references in regular expression parsing. - -- Issue #18973: Command-line interface of the calendar module now uses argparse - instead of optparse. - -- Issue #25510: fileinput.FileInput.readline() now returns b'' instead of '' - at the end if the FileInput was opened with binary mode. - Patch by Ryosuke Ito. - -- Issue #25503: Fixed inspect.getdoc() for inherited docstrings of properties. - Original patch by John Mark Vandenberg. - -- Issue #25515: Always use os.urandom as a source of randomness in uuid.uuid4. - -- Issue #21827: Fixed textwrap.dedent() for the case when largest common - whitespace is a substring of smallest leading whitespace. - Based on patch by Robert Li. - -- Issue #25447: The lru_cache() wrapper objects now can be copied and pickled - (by returning the original object unchanged). - -- Issue #25390: typing: Don't crash on Union[str, Pattern]. - -- Issue #25441: asyncio: Raise error from drain() when socket is closed. - -- Issue #25410: Cleaned up and fixed minor bugs in C implementation of - OrderedDict. - -- Issue #25411: Improved Unicode support in SMTPHandler through better use of - the email package. Thanks to user simon04 for the patch. - -- Move the imp module from a PendingDeprecationWarning to DeprecationWarning. - -- Issue #25407: Remove mentions of the formatter module being removed in - Python 3.6. - -- Issue #25406: Fixed a bug in C implementation of OrderedDict.move_to_end() - that caused segmentation fault or hang in iterating after moving several - items to the start of ordered dict. - -- Issue #25382: pickletools.dis() now outputs implicit memo index for the - MEMOIZE opcode. - -- Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). - base64.b64encode() uses it to avoid a memory copy. - -- Issue #24164: Objects that need calling ``__new__`` with keyword arguments, - can now be pickled using pickle protocols older than protocol version 4. - -- Issue #25364: zipfile now works in threads disabled builds. - -- Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both - decode_data and enable_SMTPUTF8 are set to true. - -- Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate - extensions. Patch by Nikolay Bogoychev. - -- Issue #25316: distutils raises OSError instead of DistutilsPlatformError - when MSVC is not installed. - -- Issue #25380: Fixed protocol for the STACK_GLOBAL opcode in - pickletools.opcodes. - -- Issue #23972: Updates asyncio datagram create method allowing reuseport - and reuseaddr socket options to be set prior to binding the socket. - Mirroring the existing asyncio create_server method the reuseaddr option - for datagram sockets defaults to True if the O/S is 'posix' (except if the - platform is Cygwin). Patch by Chris Laws. - -- Issue #25304: Add asyncio.run_coroutine_threadsafe(). This lets you - submit a coroutine to a loop from another thread, returning a - concurrent.futures.Future. By Vincent Michel. - -- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the - first question mark (?) rather than the last. Patch from Xiang Zhang. - -- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the - query part of the URL as if it were a path. Patch from Xiang Zhang. - -- Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not - supported. Check if it is supported, it may not be supported on OpenBSD for - example. - -- Issue #23600: Default implementation of tzinfo.fromutc() was returning - wrong results in some cases. - -- Issue #25203: Failed readline.set_completer_delims() no longer left the - module in inconsistent state. - -- Issue #25011: rlcompleter now omits private and special attribute names unless - the prefix starts with underscores. - -- Issue #25209: rlcompleter now can add a space or a colon after completed - keyword. - -- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - -- Issue #23517: fromtimestamp() and utcfromtimestamp() methods of - datetime.datetime now round microseconds to nearest with ties going to - nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding - towards -Infinity (ROUND_FLOOR). - -- Issue #23552: Timeit now warns when there is substantial (4x) variance - between best and worst times. Patch from Serhiy Storchaka. - -- Issue #24633: site-packages/README -> README.txt. - -- Issue #24879: help() and pydoc can now list named tuple fields in the - order they were defined rather than alphabetically. The ordering is - determined by the _fields attribute if present. - -- Issue #24874: Improve speed of itertools.cycle() and make its - pickle more compact. - -- Fix crash in itertools.cycle.__setstate__() when the first argument wasn't - a list. - -- Issue #20059: urllib.parse raises ValueError on all invalid ports. - Patch by Martin Panter. - -- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid - identifiers. Patch by Matthias Bussonnier. - -- Issue #23426: run_setup was broken in distutils. - Patch from Alexander Belopolsky. - -- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. - -- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can - only be used with 'r'. Patch from Jeff Balogh and John O'Connor. - -- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #24426: Fast searching optimization in regular expressions now works - for patterns that starts with capturing groups. Fast searching optimization - now can't be disabled at compile time. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #13248: Remove deprecated inspect.getmoduleinfo function. - -- Issue #25578: Fix (another) memory leak in SSLSocket.getpeercer(). - -- Issue #25530: Disable the vulnerable SSLv3 protocol by default when creating - ssl.SSLContext. - -- Issue #25569: Fix memory leak in SSLSocket.getpeercert(). - -- Issue #25471: Sockets returned from accept() shouldn't appear to be - nonblocking. - -- Issue #25319: When threading.Event is reinitialized, the underlying condition - should use a regular lock rather than a recursive lock. - -- Skip getaddrinfo if host is already resolved. - Patch by A. Jesse Jiryu Davis. - -- Issue #26050: Add asyncio.StreamReader.readuntil() method. - Patch by Марк Коренберг. - -- Issue #25924: Avoid unnecessary serialization of getaddrinfo(3) calls on - OS X versions 10.5 or higher. Original patch by A. Jesse Jiryu Davis. - -- Issue #26406: Avoid unnecessary serialization of getaddrinfo(3) calls on - current versions of OpenBSD and NetBSD. Patch by A. Jesse Jiryu Davis. - -- Issue #26848: Fix asyncio/subprocess.communicate() to handle empty input. - Patch by Jack O'Connor. - -- Issue #27040: Add loop.get_exception_handler method - -- Issue #27041: asyncio: Add loop.create_future method - -IDLE ----- - -- Issue #20640: Add tests for idlelib.configHelpSourceEdit. - Patch by Saimadhav Heblikar. - -- In the 'IDLE-console differences' section of the IDLE doc, clarify - how running with IDLE affects sys.modules and the standard streams. - -- Issue #25507: fix incorrect change in IOBinding that prevented printing. - Augment IOBinding htest to include all major IOBinding functions. - -- Issue #25905: Revert unwanted conversion of ' to ’ RIGHT SINGLE QUOTATION - MARK in README.txt and open this and NEWS.txt with 'ascii'. - Re-encode CREDITS.txt to utf-8 and open it with 'utf-8'. - -- Issue #15348: Stop the debugger engine (normally in a user process) - before closing the debugger window (running in the IDLE process). - This prevents the RuntimeErrors that were being caught and ignored. - -- Issue #24455: Prevent IDLE from hanging when a) closing the shell while the - debugger is active (15347); b) closing the debugger with the [X] button - (15348); and c) activating the debugger when already active (24455). - The patch by Mark Roseman does this by making two changes. - 1. Suspend and resume the gui.interaction method with the tcl vwait - mechanism intended for this purpose (instead of root.mainloop & .quit). - 2. In gui.run, allow any existing interaction to terminate first. - -- Change 'The program' to 'Your program' in an IDLE 'kill program?' message - to make it clearer that the program referred to is the currently running - user program, not IDLE itself. - -- Issue #24750: Improve the appearance of the IDLE editor window status bar. - Patch by Mark Roseman. - -- Issue #25313: Change the handling of new built-in text color themes to better - address the compatibility problem introduced by the addition of IDLE Dark. - Consistently use the revised idleConf.CurrentTheme everywhere in idlelib. - -- Issue #24782: Extension configuration is now a tab in the IDLE Preferences - dialog rather than a separate dialog. The former tabs are now a sorted - list. Patch by Mark Roseman. - -- Issue #22726: Re-activate the config dialog help button with some content - about the other buttons and the new IDLE Dark theme. - -- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme. - It is more or less IDLE Classic inverted, with a cobalt blue background. - Strings, comments, keywords, ... are still green, red, orange, ... . - To use it with IDLEs released before November 2015, hit the - 'Save as New Custom Theme' button and enter a new name, - such as 'Custom Dark'. The custom theme will work with any IDLE - release, and can be modified. - -- Issue #25224: README.txt is now an idlelib index for IDLE developers and - curious users. The previous user content is now in the IDLE doc chapter. - 'IDLE' now means 'Integrated Development and Learning Environment'. - -- Issue #24820: Users can now set breakpoint colors in - Settings -> Custom Highlighting. Original patch by Mark Roseman. - -- Issue #24972: Inactive selection background now matches active selection - background, as configured by users, on all systems. Found items are now - always highlighted on Windows. Initial patch by Mark Roseman. - -- Issue #24570: Idle: make calltip and completion boxes appear on Macs - affected by a tk regression. Initial patch by Mark Roseman. - -- Issue #24988: Idle ScrolledList context menus (used in debugger) - now work on Mac Aqua. Patch by Mark Roseman. - -- Issue #24801: Make right-click for context menu work on Mac Aqua. - Patch by Mark Roseman. - -- Issue #25173: Associate tkinter messageboxes with a specific widget. - For Mac OSX, make them a 'sheet'. Patch by Mark Roseman. - -- Issue #25198: Enhance the initial html viewer now used for Idle Help. - * Properly indent fixed-pitch text (patch by Mark Roseman). - * Give code snippet a very Sphinx-like light blueish-gray background. - * Re-use initial width and height set by users for shell and editor. - * When the Table of Contents (TOC) menu is used, put the section header - at the top of the screen. - -- Issue #25225: Condense and rewrite Idle doc section on text colors. - -- Issue #21995: Explain some differences between IDLE and console Python. - -- Issue #22820: Explain need for *print* when running file from Idle editor. - -- Issue #25224: Doc: augment Idle feature list and no-subprocess section. - -- Issue #25219: Update doc for Idle command line options. - Some were missing and notes were not correct. - -- Issue #24861: Most of idlelib is private and subject to change. - Use idleib.idle.* to start Idle. See idlelib.__init__.__doc__. - -- Issue #25199: Idle: add synchronization comments for future maintainers. - -- Issue #16893: Replace help.txt with help.html for Idle doc display. - The new idlelib/help.html is rstripped Doc/build/html/library/idle.html. - It looks better than help.txt and will better document Idle as released. - The tkinter html viewer that works for this file was written by Rose Roseman. - The now unused EditorWindow.HelpDialog class and helt.txt file are deprecated. - -- Issue #24199: Deprecate unused idlelib.idlever with possible removal in 3.6. - -- Issue #24790: Remove extraneous code (which also create 2 & 3 conflicts). - -Documentation -------------- - -- Issue #26736: Used HTTPS for external links in the documentation if possible. - -- Issue #6953: Rework the Readline module documentation to group related - functions together, and add more details such as what underlying Readline - functions and variables are accessed. - -- Issue #23606: Adds note to ctypes documentation regarding cdll.msvcrt. - -- Issue #24952: Clarify the default size argument of stack_size() in - the "threading" and "_thread" modules. Patch from Mattip. - -- Issue #26014: Update 3.x packaging documentation: - * "See also" links to the new docs are now provided in the legacy pages - * links to setuptools documentation have been updated - -Tests ------ - -- Issue #21916: Added tests for the turtle module. Patch by ingrid, - Gregory Loyse and Jelle Zijlstra. - -- Issue #26295: When using "python3 -m test --testdir=TESTDIR", regrtest - doesn't add "test." prefix to test module names. - -- Issue #26523: The multiprocessing thread pool (multiprocessing.dummy.Pool) - was untested. - -- Issue #26015: Added new tests for pickling iterators of mutable sequences. - -- Issue #26325: Added test.support.check_no_resource_warning() to check that - no ResourceWarning is emitted. - -- Issue #25940: Changed test_ssl to use its internal local server more. This - avoids relying on svn.python.org, which recently changed root certificate. - -- Issue #25616: Tests for OrderedDict are extracted from test_collections - into separate file test_ordered_dict. - -- Issue #25449: Added tests for OrderedDict subclasses. - -- Issue #25188: Add -P/--pgo to test.regrtest to suppress error output when - running the test suite for the purposes of a PGO build. Initial patch by - Alecsandru Patrascu. - -- Issue #22806: Add ``python -m test --list-tests`` command to list tests. - -- Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of - file descriptors. Patch written by Richard Oudkerk. - -- Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the - list of ignored directories. - -- ``PCbuild\rt.bat`` now accepts an unlimited number of arguments to pass along - to regrtest.py. Previously there was a limit of 9. - -- Issue #26583: Skip test_timestamp_overflow in test_import if bytecode - files cannot be written. - -Build ------ - -- Issue #21277: Don't try to link _ctypes with a ffi_convenience library. - -- Issue #26884: Fix linking extension modules for cross builds. - Patch by Xavier de Gaye. - -- Issue #26932: Fixed support of RTLD_* constants defined as enum values, - not via macros (in particular on Android). Patch by Chi Hsuan Yen. - -- Issue #22359: Disable the rules for running _freeze_importlib and pgen when - cross-compiling. The output of these programs is normally saved with the - source code anyway, and is still regenerated when doing a native build. - Patch by Xavier de Gaye. - -- Issue #21668: Link audioop, _datetime, _ctypes_test modules to libm, - except on Mac OS X. Patch written by Chi Hsuan Yen. - -- Issue #25702: A --with-lto configure option has been added that will - enable link time optimizations at build time during a make profile-opt. - Some compilers and toolchains are known to not produce stable code when - using LTO, be sure to test things thoroughly before relying on it. - It can provide a few % speed up over profile-opt alone. - -- Issue #26624: Adds validation of ucrtbase[d].dll version with warning - for old versions. - -- Issue #17603: Avoid error about nonexistant fileblocks.o file by using a - lower-level check for st_blocks in struct stat. - -- Issue #26079: Fixing the build output folder for tix-8.4.3.6. Patch by - Bjoern Thiel. - -- Issue #26465: Update Windows builds to use OpenSSL 1.0.2g. - -- Issue #25348: Added ``--pgo`` and ``--pgo-job`` arguments to - ``PCbuild\build.bat`` for building with Profile-Guided Optimization. The - old ``PCbuild\build_pgo.bat`` script is removed. - -- Issue #25827: Add support for building with ICC to ``configure``, including - a new ``--with-icc`` flag. - -- Issue #25696: Fix installation of Python on UNIX with make -j9. - -- Issue #24986: It is now possible to build Python on Windows without errors - when external libraries are not available. - -- Issue #24421: Compile Modules/_math.c once, before building extensions. - Previously it could fail to compile properly if the math and cmath builds - were concurrent. - -- Issue #26465: Update OS X 10.5+ 32-bit-only installer to build - and link with OpenSSL 1.0.2g. - -- Issue #26268: Update Windows builds to use OpenSSL 1.0.2f. - -- Issue #25136: Support Apple Xcode 7's new textual SDK stub libraries. - -- Issue #24324: Do not enable unreachable code warnings when using - gcc as the option does not work correctly in older versions of gcc - and has been silently removed as of gcc-4.5. - -Windows -------- - -- Issue #27053: Updates make_zip.py to correctly generate library ZIP file. - -- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases - that don't include the contents of the include directory (that is, 1.0.2e - and later). - -- Issue #26071: bdist_wininst created binaries fail to start and find - 32bit Python - -- Issue #26073: Update the list of magic numbers in launcher - -- Issue #26065: Excludes venv from library when generating embeddable - distro. - -- Issue #25022: Removed very outdated PC/example_nt/ directory. - -Tools/Demos ------------ - -- Issue #26799: Fix python-gdb.py: don't get C types once when the Python code - is loaded, but get C types on demand. The C types can change if - python-gdb.py is loaded before the Python executable. Patch written by Thomas - Ilsche. - -- Issue #26271: Fix the Freeze tool to properly use flags passed through - configure. Patch by Daniel Shaulov. - -- Issue #26489: Add dictionary unpacking support to Tools/parser/unparse.py. - Patch by Guo Ci Teo. - -- Issue #26316: Fix variable name typo in Argument Clinic. - -- Issue #25440: Fix output of python-config --extension-suffix. - -- Issue #25154: The pyvenv script has been deprecated in favour of - `python3 -m venv`. - -C API ------ - -- Issue #26312: SystemError is now raised in all programming bugs with using - PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some - programming bugs. - -- Issue #26198: ValueError is now raised instead of TypeError on buffer - overflow in parsing "es#" and "et#" format units. SystemError is now raised - instead of TypeError on programmical error in parsing format string. - - -What's New in Python 3.5.3? -=========================== - -Release date: 2017-01-17 - -There were no code changes between 3.5.3rc1 and 3.5.3 final. - - -What's New in Python 3.5.3 release candidate 1? -=============================================== - -Release date: 2017-01-02 - -Core and Builtins ------------------ - -- Issue #29073: bytearray formatting no longer truncates on first null byte. - -- Issue #28932: Do not include if it does not exist. - -- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() - must not convert combined table into split table. - -- Issue #25677: Correct the positioning of the syntax error caret for - indented blocks. Based on patch by Michael Layzell. - -- Issue #29000: Fixed bytes formatting of octals with zero padding in alternate - form. - -- Issue #28512: Fixed setting the offset attribute of SyntaxError by - PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). - -- Issue #28991: functools.lru_cache() was susceptible to an obscure reentrancy - bug caused by a monkey-patched len() function. - -- Issue #28648: Fixed crash in Py_DecodeLocale() in debug build on Mac OS X - when decode astral characters. Patch by Xiang Zhang. - -- Issue #19398: Extra slash no longer added to sys.path components in case of - empty compile-time PYTHONPATH components. - -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). - Patch by Xiang Zhang. - -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: The constructor of range_iterator now checks that step is not 0. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. - -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. - -- Issue #28350: String constants with null character no longer interned. - -- Issue #26617: Fix crash when GC runs during weakref callbacks. - -- Issue #27942: String constants now interned recursively in tuples and frozensets. - -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. - -- Issue #28203: Fix incorrect type in error message from - ``complex(1.0, {2:3})``. Patch by Soumya Sharma. - -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. - -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. - -- Issue #25221: Fix corrupted result from PyLong_FromLong(0) when - Python is compiled with NSMALLPOSINTS = 0. - -- Issue #25758: Prevents zipimport from unnecessarily encoding a filename - (patch by Eryk Sun) - -- Issue #28189: dictitems_contains no longer swallows compare errors. - (Patch by Xiang Zhang) - -- Issue #27812: Properly clear out a generator's frame's backreference to the - generator to prevent crashes in frame.clear(). - -- Issue #27811: Fix a crash when a coroutine that has not been awaited is - finalized with warnings-as-errors enabled. - -- Issue #27587: Fix another issue found by PVS-Studio: Null pointer check - after use of 'def' in _PyState_AddModule(). - Initial patch by Christian Heimes. - -- Issue #26020: set literal evaluation order did not match documented behaviour. - -- Issue #27782: Multi-phase extension module import now correctly allows the - ``m_methods`` field to be used to add module level functions to instances - of non-module types returned from ``Py_create_mod``. Patch by Xiang Zhang. - -- Issue #27936: The round() function accepted a second None argument - for some types but not for others. Fixed the inconsistency by - accepting None for all numeric types. - -- Issue #27487: Warn if a submodule argument to "python -m" or - runpy.run_module() is found in sys.modules after parent packages are - imported, but before the submodule is executed. - -- Issue #27558: Fix a SystemError in the implementation of "raise" statement. - In a brand new thread, raise a RuntimeError since there is no active - exception to reraise. Patch written by Xiang Zhang. - -- Issue #27419: Standard __import__() no longer look up "__import__" in globals - or builtins for importing submodules or "from import". Fixed handling an - error of non-string package name. - -- Issue #27083: Respect the PYTHONCASEOK environment variable under Windows. - -- Issue #27514: Make having too many statically nested blocks a SyntaxError - instead of SystemError. - -- Issue #27473: Fixed possible integer overflow in bytes and bytearray - concatenations. Patch by Xiang Zhang. - -- Issue #27507: Add integer overflow check in bytearray.extend(). Patch by - Xiang Zhang. - -- Issue #27581: Don't rely on wrapping for overflow check in - PySequence_Tuple(). Patch by Xiang Zhang. - -- Issue #27443: __length_hint__() of bytearray iterators no longer return a - negative integer for a resized bytearray. - -- Issue #27942: Fix memory leak in codeobject.c - -Library -------- - -- Issue #15812: inspect.getframeinfo() now correctly shows the first line of - a context. Patch by Sam Breese. - -- Issue #29094: Offsets in a ZIP file created with extern file object and modes - "w" and "x" now are relative to the start of the file. - -- Issue #13051: Fixed recursion errors in large or resized - curses.textpad.Textbox. Based on patch by Tycho Andersen. - -- Issue #29119: Fix weakrefs in the pure python version of - collections.OrderedDict move_to_end() method. - Contributed by Andra Bogildea. - -- Issue #9770: curses.ascii predicates now work correctly with negative - integers. - -- Issue #28427: old keys should not remove new values from - WeakValueDictionary when collecting from another thread. - -- Issue 28923: Remove editor artifacts from Tix.py. - -- Issue #28871: Fixed a crash when deallocate deep ElementTree. - -- Issue #19542: Fix bugs in WeakValueDictionary.setdefault() and - WeakValueDictionary.pop() when a GC collection happens in another - thread. - -- Issue #20191: Fixed a crash in resource.prlimit() when pass a sequence that - doesn't own its elements as limits. - -- Issue #28779: multiprocessing.set_forkserver_preload() would crash the - forkserver process if a preloaded module instantiated some - multiprocessing objects such as locks. - -- Issue #28847: dbm.dumb now supports reading read-only files and no longer - writes the index file when it is not changed. - -- Issue #25659: In ctypes, prevent a crash calling the from_buffer() and - from_buffer_copy() methods on abstract classes like Array. - -- Issue #28732: Fix crash in os.spawnv() with no elements in args - -- Issue #28485: Always raise ValueError for negative - compileall.compile_dir(workers=...) parameter, even when multithreading is - unavailable. - -- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when - the garbage collector is invoked in other thread. Based on patch by - Sebastian Cufre. - -- Issue #27517: LZMA compressor and decompressor no longer raise exceptions if - given empty data twice. Patch by Benjamin Fogle. - -- Issue #28549: Fixed segfault in curses's addch() with ncurses6. - -- Issue #28449: tarfile.open() with mode "r" or "r:" now tries to open a tar - file with compression before trying to open it without compression. Otherwise - it had 50% chance failed with ignore_zeros=True. - -- Issue #23262: The webbrowser module now supports Firefox 36+ and derived - browsers. Based on patch by Oleg Broytman. - -- Issue #27939: Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused - by representing the scale as float value internally in Tk. tkinter.IntVar - now works if float value is set to underlying Tk variable. - -- Issue #28255: calendar.TextCalendar().prmonth() no longer prints a space - at the start of new line after printing a month's calendar. Patch by - Xiang Zhang. - -- Issue #20491: The textwrap.TextWrapper class now honors non-breaking spaces. - Based on patch by Kaarle Ritvanen. - -- Issue #28353: os.fwalk() no longer fails on broken links. - -- Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin - a workaround to Tix library bug. - -- Issue #28488: shutil.make_archive() no longer add entry "./" to ZIP archive. - -- Issue #24452: Make webbrowser support Chrome on Mac OS X. - -- Issue #20766: Fix references leaked by pdb in the handling of SIGINT - handlers. - -- Issue #26293: Fixed writing ZIP files that starts not from the start of the - file. Offsets in ZIP file now are relative to the start of the archive in - conforming to the specification. - -- Issue #28321: Fixed writing non-BMP characters with binary format in plistlib. - -- Issue #28322: Fixed possible crashes when unpickle itertools objects from - incorrect pickle data. Based on patch by John Leitch. - -- Fix possible integer overflows and crashes in the mmap module with unusual - usage patterns. - -- Issue #1703178: Fix the ability to pass the --link-objects option to the - distutils build_ext command. - -- Issue #28253: Fixed calendar functions for extreme months: 0001-01 - and 9999-12. - - Methods itermonthdays() and itermonthdays2() are reimplemented so - that they don't call itermonthdates() which can cause datetime.date - under/overflow. - -- Issue #28275: Fixed possible use after free in the decompress() - methods of the LZMADecompressor and BZ2Decompressor classes. - Original patch by John Leitch. - -- Issue #27897: Fixed possible crash in sqlite3.Connection.create_collation() - if pass invalid string-like object as a name. Patch by Xiang Zhang. - -- Issue #18893: Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. - Patch by Madison May. - -- Issue #27611: Fixed support of default root window in the tkinter.tix module. - -- Issue #27348: In the traceback module, restore the formatting of exception - messages like "Exception: None". This fixes a regression introduced in - 3.5a2. - -- Issue #25651: Allow falsy values to be used for msg parameter of subTest(). - -- Issue #27932: Prevent memory leak in win32_ver(). - -- Fix UnboundLocalError in socket._sendfile_use_sendfile. - -- Issue #28075: Check for ERROR_ACCESS_DENIED in Windows implementation of - os.stat(). Patch by Eryk Sun. - -- Issue #25270: Prevent codecs.escape_encode() from raising SystemError when - an empty bytestring is passed. - -- Issue #28181: Get antigravity over HTTPS. Patch by Kaartic Sivaraam. - -- Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. - Patch by Gergely Imreh and Markus Holtermann. - -- Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). - -- Issue #19003:m email.generator now replaces only \r and/or \n line - endings, per the RFC, instead of all unicode line endings. - -- Issue #28019: itertools.count() no longer rounds non-integer step in range - between 1.0 and 2.0 to 1. - -- Issue #25969: Update the lib2to3 grammar to handle the unpacking - generalizations added in 3.5. - -- Issue #14977: mailcap now respects the order of the lines in the mailcap - files ("first match"), as required by RFC 1542. Patch by Michael Lazar. - -- Issue #24594: Validates persist parameter when opening MSI database - -- Issue #17582: xml.etree.ElementTree nows preserves whitespaces in attributes - (Patch by Duane Griffin. Reviewed and approved by Stefan Behnel.) - -- Issue #28047: Fixed calculation of line length used for the base64 CTE - in the new email policies. - -- Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). - Patch by Claude Paroz. - -- Issue #22450: urllib now includes an "Accept: */*" header among the - default headers. This makes the results of REST API requests more - consistent and predictable especially when proxy servers are involved. - -- lib2to3.pgen3.driver.load_grammar() now creates a stable cache file - between runs given the same Grammar.txt input regardless of the hash - randomization setting. - -- Issue #27570: Avoid zero-length memcpy() etc calls with null source - pointers in the "ctypes" and "array" modules. - -- Issue #22233: Break email header lines *only* on the RFC specified CR and LF - characters, not on arbitrary unicode line breaks. This also fixes a bug in - HTTP header parsing. - -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. - -- Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name - fields in X.509 certs. - -- Issue #27850: Remove 3DES from ssl module's default cipher list to counter - measure sweet32 attack (CVE-2016-2183). - -- Issue #27766: Add ChaCha20 Poly1305 to ssl module's default ciper list. - (Required OpenSSL 1.1.0 or LibreSSL). - -- Issue #26470: Port ssl and hashlib module to OpenSSL 1.1.0. - -- Remove support for passing a file descriptor to os.access. It never worked but - previously didn't raise. - -- Issue #12885: Fix error when distutils encounters symlink. - -- Issue #27881: Fixed possible bugs when setting sqlite3.Connection.isolation_level. - Based on patch by Xiang Zhang. - -- Issue #27861: Fixed a crash in sqlite3.Connection.cursor() when a factory - creates not a cursor. Patch by Xiang Zhang. - -- Issue #19884: Avoid spurious output on OS X with Gnu Readline. - -- Issue #27706: Restore deterministic behavior of random.Random().seed() - for string seeds using seeding version 1. Allows sequences of calls - to random() to exactly match those obtained in Python 2. - Patch by Nofar Schnider. - -- Issue #10513: Fix a regression in Connection.commit(). Statements should - not be reset after a commit. - -- A new version of typing.py from https://github.com/python/typing: - - Collection (only for 3.6) (Issue #27598) - - Add FrozenSet to __all__ (upstream #261) - - fix crash in _get_type_vars() (upstream #259) - - Remove the dict constraint in ForwardRef._eval_type (upstream #252) - -- Issue #27539: Fix unnormalised ``Fraction.__pow__`` result in the case - of negative exponent and negative base. - -- Issue #21718: cursor.description is now available for queries using CTEs. - -- Issue #2466: posixpath.ismount now correctly recognizes mount points which - the user does not have permission to access. - -- Issue #27773: Correct some memory management errors server_hostname in - _ssl.wrap_socket(). - -- Issue #26750: unittest.mock.create_autospec() now works properly for - subclasses of property() and other data descriptors. - -- In the curses module, raise an error if window.getstr() or window.instr() is - passed a negative value. - -- Issue #27783: Fix possible usage of uninitialized memory in - operator.methodcaller. - -- Issue #27774: Fix possible Py_DECREF on unowned object in _sre. - -- Issue #27760: Fix possible integer overflow in binascii.b2a_qp. - -- Issue #27758: Fix possible integer overflow in the _csv module for large - record lengths. - -- Issue #27568: Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the - HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates - that the script is in CGI mode. - -- Issue #27656: Do not assume sched.h defines any SCHED_* constants. - -- Issue #27130: In the "zlib" module, fix handling of large buffers - (typically 4 GiB) when compressing and decompressing. Previously, inputs - were limited to 4 GiB, and compression and decompression operations did not - properly handle results of 4 GiB. - -- Issue #27533: Release GIL in nt._isdir - -- Issue #17711: Fixed unpickling by the persistent ID with protocol 0. - Original patch by Alexandre Vassalotti. - -- Issue #27522: Avoid an unintentional reference cycle in email.feedparser. - -- Issue #26844: Fix error message for imp.find_module() to refer to 'path' - instead of 'name'. Patch by Lev Maximov. - -- Issue #23804: Fix SSL zero-length recv() calls to not block and not raise - an error about unclean EOF. - -- Issue #27466: Change time format returned by http.cookie.time2netscape, - confirming the netscape cookie format and making it consistent with - documentation. - -- Issue #26664: Fix activate.fish by removing mis-use of ``$``. - -- Issue #22115: Fixed tracing Tkinter variables: trace_vdelete() with wrong - mode no longer break tracing, trace_vinfo() now always returns a list of - pairs of strings, tracing in the "u" mode now works. - -- Fix a scoping issue in importlib.util.LazyLoader which triggered an - UnboundLocalError when lazy-loading a module that was already put into - sys.modules. - -- Issue #27079: Fixed curses.ascii functions isblank(), iscntrl() and ispunct(). - -- Issue #26754: Some functions (compile() etc) accepted a filename argument - encoded as an iterable of integers. Now only strings and byte-like objects - are accepted. - -- Issue #27048: Prevents distutils failing on Windows when environment - variables contain non-ASCII characters - -- Issue #27330: Fixed possible leaks in the ctypes module. - -- Issue #27238: Got rid of bare excepts in the turtle module. Original patch - by Jelle Zijlstra. - -- Issue #27122: When an exception is raised within the context being managed - by a contextlib.ExitStack() and one of the exit stack generators - catches and raises it in a chain, do not re-raise the original exception - when exiting, let the new chained one through. This avoids the PEP 479 - bug described in issue25782. - -- [Security] Issue #27278: Fix os.urandom() implementation using getrandom() on - Linux. Truncate size to INT_MAX and loop until we collected enough random - bytes, instead of casting a directly Py_ssize_t to int. - -- Issue #26386: Fixed ttk.TreeView selection operations with item id's - containing spaces. - -- [Security] Issue #22636: Avoid shell injection problems with - ctypes.util.find_library(). - -- Issue #16182: Fix various functions in the "readline" module to use the - locale encoding, and fix get_begidx() and get_endidx() to return code point - indexes. - -- Issue #27392: Add loop.connect_accepted_socket(). - Patch by Jim Fulton. - -- Issue #27930: Improved behaviour of logging.handlers.QueueListener. - Thanks to Paulo Andrade and Petr Viktorin for the analysis and patch. - -- Issue #21201: Improves readability of multiprocessing error message. Thanks - to Wojciech Walczak for patch. - -- Issue #27456: asyncio: Set TCP_NODELAY by default. - -- Issue #27906: Fix socket accept exhaustion during high TCP traffic. - Patch by Kevin Conway. - -- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. - Patch by Seth Michael Larson. - -- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. - Patch by iceboy. - -- Issue #26909: Fix slow pipes IO in asyncio. - Patch by INADA Naoki. - -- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. - -- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. - Patch by Mark Williams. - -- Issue #28368: Refuse monitoring processes if the child watcher has - no loop attached. - Patch by Vincent Michel. - -- Issue #28369: Raise RuntimeError when transport's FD is used with - add_reader, add_writer, etc. - -- Issue #28370: Speedup asyncio.StreamReader.readexactly. - Patch by Коренберг Марк. - -- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. - -- Issue #28372: Fix asyncio to support formatting of non-python coroutines. - -- Issue #28399: Remove UNIX socket from FS before binding. - Patch by Коренберг Марк. - -- Issue #27972: Prohibit Tasks to await on themselves. - -- Issue #26923: Fix asyncio.Gather to refuse being cancelled once all - children are done. - Patch by Johannes Ebke. - -- Issue #26796: Don't configure the number of workers for default - threadpool executor. - Initial patch by Hans Lawrenz. - -- Issue #28600: Optimize loop.call_soon(). - -- Issue #28613: Fix get_event_loop() return the current loop if - called from coroutines/callbacks. - -- Issue #28639: Fix inspect.isawaitable to always return bool - Patch by Justin Mayfield. - -- Issue #28652: Make loop methods reject socket kinds they do not support. - -- Issue #28653: Fix a refleak in functools.lru_cache. - -- Issue #28703: Fix asyncio.iscoroutinefunction to handle Mock objects. - -- Issue #24142: Reading a corrupt config file left the parser in an - invalid state. Original patch by Florian Höch. - -- Issue #28990: Fix SSL hanging if connection is closed before handshake - completed. - (Patch by HoHo-Ho) - -IDLE ----- - -- Issue #15308: Add 'interrupt execution' (^C) to Shell menu. - Patch by Roger Serwy, updated by Bayard Randel. - -- Issue #27922: Stop IDLE tests from 'flashing' gui widgets on the screen. - -- Add version to title of IDLE help window. - -- Issue #25564: In section on IDLE -- console differences, mention that - using exec means that __builtins__ is defined for each statement. - -- Issue #27714: text_textview and test_autocomplete now pass when re-run - in the same process. This occurs when test_idle fails when run with the - -w option but without -jn. Fix warning from test_config. - -- Issue #25507: IDLE no longer runs buggy code because of its tkinter imports. - Users must include the same imports required to run directly in Python. - -- Issue #27452: add line counter and crc to IDLE configHandler test dump. - -- Issue #27365: Allow non-ascii chars in IDLE NEWS.txt, for contributor names. - -- Issue #27245: IDLE: Cleanly delete custom themes and key bindings. - Previously, when IDLE was started from a console or by import, a cascade - of warnings was emitted. Patch by Serhiy Storchaka. - -C API ------ - -- Issue #28808: PyUnicode_CompareWithASCIIString() now never raises exceptions. - -- Issue #26754: PyUnicode_FSDecoder() accepted a filename argument encoded as - an iterable of integers. Now only strings and bytes-like objects are accepted. - -Documentation -------------- - -- Issue #28513: Documented command-line interface of zipfile. - -Tests ------ - -- Issue #28950: Disallow -j0 to be combined with -T/-l/-M in regrtest - command line arguments. - -- Issue #28666: Now test.support.rmtree is able to remove unwritable or - unreadable directories. - -- Issue #23839: Various caches now are cleared before running every test file. - -- Issue #28409: regrtest: fix the parser of command line arguments. - -- Issue #27787: Call gc.collect() before checking each test for "dangling - threads", since the dangling threads are weak references. - -- Issue #27369: In test_pyexpat, avoid testing an error message detail that - changed in Expat 2.2.0. - -Tools/Demos ------------ - -- Issue #27952: Get Tools/scripts/fixcid.py working with Python 3 and the - current "re" module, avoid invalid Python backslash escapes, and fix a bug - parsing escaped C quote signs. - -- Issue #27332: Fixed the type of the first argument of module-level functions - generated by Argument Clinic. Patch by Petr Viktorin. - -- Issue #27418: Fixed Tools/importbench/importbench.py. - -Windows -------- - -- Issue #28251: Improvements to help manuals on Windows. - -- Issue #28110: launcher.msi has different product codes between 32-bit and - 64-bit - -- Issue #25144: Ensures TargetDir is set before continuing with custom - install. - -- Issue #27469: Adds a shell extension to the launcher so that drag and drop - works correctly. - -- Issue #27309: Enabled proper Windows styles in python[w].exe manifest. - -Build ------ - -- Issue #29080: Removes hard dependency on hg.exe from PCBuild/build.bat - -- Issue #23903: Added missed names to PC/python3.def. - -- Issue #10656: Fix out-of-tree building on AIX. Patch by Tristan Carel and - Michael Haubenwallner. - -- Issue #26359: Rename --with-optimiations to --enable-optimizations. - -- Issue #28444: Fix missing extensions modules when cross compiling. - -- Issue #28248: Update Windows build and OS X installers to use OpenSSL 1.0.2j. - -- Issue #28258: Fixed build with Estonian locale (python-config and distclean - targets in Makefile). Patch by Arfrever Frehtes Taifersar Arahesis. - -- Issue #26661: setup.py now detects system libffi with multiarch wrapper. - -- Issue #28066: Fix the logic that searches build directories for generated - include files when building outside the source tree. - -- Issue #15819: Remove redundant include search directory option for building - outside the source tree. - -- Issue #27566: Fix clean target in freeze makefile (patch by Lisa Roach) - -- Issue #27705: Update message in validate_ucrtbase.py - -- Issue #27983: Cause lack of llvm-profdata tool when using clang as - required for PGO linking to be a configure time error rather than - make time when --with-optimizations is enabled. Also improve our - ability to find the llvm-profdata tool on MacOS and some Linuxes. - -- Issue #26307: The profile-opt build now applies PGO to the built-in modules. - -- Issue #26359: Add the --with-optimizations configure flag. - -- Issue #27713: Suppress spurious build warnings when updating importlib's - bootstrap files. Patch by Xiang Zhang - -- Issue #25825: Correct the references to Modules/python.exp and ld_so_aix, - which are required on AIX. This updates references to an installation path - that was changed in 3.2a4, and undoes changed references to the build tree - that were made in 3.5.0a1. - -- Issue #27453: CPP invocation in configure must use CPPFLAGS. Patch by - Chi Hsuan Yen. - -- Issue #27641: The configure script now inserts comments into the makefile - to prevent the pgen and _freeze_importlib executables from being cross- - compiled. - -- Issue #26662: Set PYTHON_FOR_GEN in configure as the Python program to be - used for file generation during the build. - -- Issue #10910: Avoid C++ compilation errors on FreeBSD and OS X. - Also update FreedBSD version checks for the original ctype UTF-8 workaround. - -- Issue #28676: Prevent missing 'getentropy' declaration warning on macOS. - Patch by Gareth Rees. - - -What's New in Python 3.5.2? -=========================== - -Release date: 2016-06-26 - -Core and Builtins ------------------ - -- Issue #26930: Update Windows builds to use OpenSSL 1.0.2h. - -Tests ------ - -- Issue #26867: Ubuntu's openssl OP_NO_SSLv3 is forced on by default; fix test. - -IDLE ----- - -- Issue #27365: Allow non-ascii in idlelib/NEWS.txt - minimal part for 3.5.2. - - -What's New in Python 3.5.2 release candidate 1? -=============================================== - -Release date: 2016-06-12 - -Core and Builtins ------------------ - -- Issue #27066: Fixed SystemError if a custom opener (for open()) returns a - negative number without setting an exception. - -- Issue #20041: Fixed TypeError when frame.f_trace is set to None. - Patch by Xavier de Gaye. - -- Issue #26168: Fixed possible refleaks in failing Py_BuildValue() with the "N" - format unit. - -- Issue #26991: Fix possible refleak when creating a function with annotations. - -- Issue #27039: Fixed bytearray.remove() for values greater than 127. Patch by - Joe Jevnik. - -- Issue #23640: int.from_bytes() no longer bypasses constructors for subclasses. - -- Issue #26811: gc.get_objects() no longer contains a broken tuple with NULL - pointer. - -- Issue #20120: Use RawConfigParser for .pypirc parsing, - removing support for interpolation unintentionally added - with move to Python 3. Behavior no longer does any - interpolation in .pypirc files, matching behavior in Python - 2.7 and Setuptools 19.0. - -- Issue #26659: Make the builtin slice type support cycle collection. - -- Issue #26718: super.__init__ no longer leaks memory if called multiple times. - NOTE: A direct call of super.__init__ is not endorsed! - -- Issue #25339: PYTHONIOENCODING now has priority over locale in setting the - error handler for stdin and stdout. - -- Issue #26494: Fixed crash on iterating exhausting iterators. - Affected classes are generic sequence iterators, iterators of str, bytes, - bytearray, list, tuple, set, frozenset, dict, OrderedDict, corresponding - views and os.scandir() iterator. - -- Issue #26581: If coding cookie is specified multiple times on a line in - Python source code file, only the first one is taken to account. - -- Issue #26464: Fix str.translate() when string is ASCII and first replacements - removes character, but next replacement uses a non-ASCII character or a - string longer than 1 character. Regression introduced in Python 3.5.0. - -- Issue #22836: Ensure exception reports from PyErr_Display() and - PyErr_WriteUnraisable() are sensible even when formatting them produces - secondary errors. This affects the reports produced by - sys.__excepthook__() and when __del__() raises an exception. - -- Issue #26302: Correct behavior to reject comma as a legal character for - cookie names. - -- Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and - Daniel Urban. - -- Issue #27138: Fix the doc comment for FileFinder.find_spec(). - -- Issue #26154: Add a new private _PyThreadState_UncheckedGet() function to get - the current Python thread state, but don't issue a fatal error if it is NULL. - This new function must be used instead of accessing directly the - _PyThreadState_Current variable. The variable is no more exposed since - Python 3.5.1 to hide the exact implementation of atomic C types, to avoid - compiler issues. - -- Issue #26194: Deque.insert() gave odd results for bounded deques that had - reached their maximum size. Now an IndexError will be raised when attempting - to insert into a full deque. - -- Issue #25843: When compiling code, don't merge constants if they are equal - but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` - is now correctly compiled to two different functions: ``f1()`` returns ``1`` - (``int``) and ``f2()`` returns ``1.0`` (``int``), even if ``1`` and ``1.0`` - are equal. - -- Issue #22995: [UPDATE] Comment out the one of the pickleability tests in - _PyObject_GetState() due to regressions observed in Cython-based projects. - -- Issue #25961: Disallowed null characters in the type name. - -- Issue #25973: Fix segfault when an invalid nonlocal statement binds a name - starting with two underscores. - -- Issue #22995: Instances of extension types with a state that aren't - subclasses of list or dict and haven't implemented any pickle-related - methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__, - or __getstate__), can no longer be pickled. Including memoryview. - -- Issue #20440: Massive replacing unsafe attribute setting code with special - macro Py_SETREF. - -- Issue #25766: Special method __bytes__() now works in str subclasses. - -- Issue #25421: __sizeof__ methods of builtin types now use dynamic basic size. - This allows sys.getsize() to work correctly with their subclasses with - __slots__ defined. - -- Issue #25709: Fixed problem with in-place string concatenation and utf-8 - cache. - -- Issue #27147: Mention PEP 420 in the importlib docs. - -- Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside - __getattr__. - -- Issue #24731: Fixed crash on converting objects with special methods - __bytes__, __trunc__, and __float__ returning instances of subclasses of - bytes, int, and float to subclasses of bytes, int, and float correspondingly. - -- Issue #26478: Fix semantic bugs when using binary operators with dictionary - views and tuples. - -- Issue #26171: Fix possible integer overflow and heap corruption in - zipimporter.get_data(). - -- Issue #25660: Fix TAB key behaviour in REPL with readline. - -- Issue #25887: Raise a RuntimeError when a coroutine object is awaited - more than once. - -- Issue #27243: Update the __aiter__ protocol: instead of returning - an awaitable that resolves to an asynchronous iterator, the asynchronous - iterator should be returned directly. Doing the former will trigger a - PendingDeprecationWarning. - - -Library -------- - -- [Security] Issue #26556: Update expat to 2.1.1, fixes CVE-2015-1283. - -- [Security] Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. - Reported by Team Oststrom - -- Issue #21386: Implement missing IPv4Address.is_global property. It was - documented since 07a5610bae9d. Initial patch by Roger Luethi. - -- Issue #20900: distutils register command now decodes HTTP responses - correctly. Initial patch by ingrid. - -- A new version of typing.py provides several new classes and - features: @overload outside stubs, Reversible, DefaultDict, Text, - ContextManager, Type[], NewType(), TYPE_CHECKING, and numerous bug - fixes (note that some of the new features are not yet implemented in - mypy or other static analyzers). Also classes for PEP 492 - (Awaitable, AsyncIterable, AsyncIterator) have been added (in fact - they made it into 3.5.1 but were never mentioned). - -- Issue #25738: Stop http.server.BaseHTTPRequestHandler.send_error() from - sending a message body for 205 Reset Content. Also, don't send Content - header fields in responses that don't have a body. Patch by Susumu - Koshiba. - -- Issue #21313: Fix the "platform" module to tolerate when sys.version - contains truncated build information. - -- [Security] Issue #26839: On Linux, :func:`os.urandom` now calls - ``getrandom()`` with ``GRND_NONBLOCK`` to fall back on reading - ``/dev/urandom`` if the urandom entropy pool is not initialized yet. Patch - written by Colm Buckley. - -- Issue #27164: In the zlib module, allow decompressing raw Deflate streams - with a predefined zdict. Based on patch by Xiang Zhang. - -- Issue #24291: Fix wsgiref.simple_server.WSGIRequestHandler to completely - write data to the client. Previously it could do partial writes and - truncate data. Also, wsgiref.handler.ServerHandler can now handle stdout - doing partial writes, but this is deprecated. - -- Issue #26809: Add ``__all__`` to :mod:`string`. Patch by Emanuel Barry. - -- Issue #26373: subprocess.Popen.communicate now correctly ignores - BrokenPipeError when the child process dies before .communicate() - is called in more/all circumstances. - -- Issue #21776: distutils.upload now correctly handles HTTPError. - Initial patch by Claudiu Popa. - -- Issue #27114: Fix SSLContext._load_windows_store_certs fails with - PermissionError - -- Issue #18383: Avoid creating duplicate filters when using filterwarnings - and simplefilter. Based on patch by Alex Shkop. - -- Issue #27057: Fix os.set_inheritable() on Android, ioctl() is blocked by - SELinux and fails with EACCESS. The function now falls back to fcntl(). - Patch written by Michał Bednarski. - -- Issue #27014: Fix infinite recursion using typing.py. Thanks to Kalle Tuure! - -- Issue #14132: Fix urllib.request redirect handling when the target only has - a query string. Original fix by Ján Janech. - -- Issue #17214: The "urllib.request" module now percent-encodes non-ASCII - bytes found in redirect target URLs. Some servers send Location header - fields with non-ASCII bytes, but "http.client" requires the request target - to be ASCII-encodable, otherwise a UnicodeEncodeError is raised. Based on - patch by Christian Heimes. - -- Issue #26892: Honor debuglevel flag in urllib.request.HTTPHandler. Patch - contributed by Chi Hsuan Yen. - -- Issue #22274: In the subprocess module, allow stderr to be redirected to - stdout even when stdout is not redirected. Patch by Akira Li. - -- Issue #26807: mock_open 'files' no longer error on readline at end of file. - Patch from Yolanda Robla. - -- Issue #25745: Fixed leaking a userptr in curses panel destructor. - -- Issue #26977: Removed unnecessary, and ignored, call to sum of squares helper - in statistics.pvariance. - -- Issue #26881: The modulefinder module now supports extended opcode arguments. - -- Issue #23815: Fixed crashes related to directly created instances of types in - _tkinter and curses.panel modules. - -- Issue #17765: weakref.ref() no longer silently ignores keyword arguments. - Patch by Georg Brandl. - -- Issue #26873: xmlrpc now raises ResponseError on unsupported type tags - instead of silently return incorrect result. - -- Issue #26711: Fixed the comparison of plistlib.Data with other types. - -- Issue #24114: Fix an uninitialized variable in `ctypes.util`. - - The bug only occurs on SunOS when the ctypes implementation searches - for the `crle` program. Patch by Xiang Zhang. Tested on SunOS by - Kees Bos. - -- Issue #26864: In urllib.request, change the proxy bypass host checking - against no_proxy to be case-insensitive, and to not match unrelated host - names that happen to have a bypassed hostname as a suffix. Patch by Xiang - Zhang. - -- Issue #26634: recursive_repr() now sets __qualname__ of wrapper. Patch by - Xiang Zhang. - -- Issue #26804: urllib.request will prefer lower_case proxy environment - variables over UPPER_CASE or Mixed_Case ones. Patch contributed by Hans-Peter - Jansen. - -- Issue #26837: assertSequenceEqual() now correctly outputs non-stringified - differing items (like bytes in the -b mode). This affects assertListEqual() - and assertTupleEqual(). - -- Issue #26041: Remove "will be removed in Python 3.7" from deprecation - messages of platform.dist() and platform.linux_distribution(). - Patch by Kumaripaba Miyurusara Athukorala. - -- Issue #26822: itemgetter, attrgetter and methodcaller objects no longer - silently ignore keyword arguments. - -- Issue #26733: Disassembling a class now disassembles class and static methods. - Patch by Xiang Zhang. - -- Issue #26801: Fix error handling in :func:`shutil.get_terminal_size`, catch - :exc:`AttributeError` instead of :exc:`NameError`. Patch written by Emanuel - Barry. - -- Issue #24838: tarfile's ustar and gnu formats now correctly calculate name - and link field limits for multibyte character encodings like utf-8. - -- [Security] Issue #26657: Fix directory traversal vulnerability with - http.server on Windows. This fixes a regression that was introduced in - 3.3.4rc1 and 3.4.0rc1. Based on patch by Philipp Hagemeister. - -- Issue #26717: Stop encoding Latin-1-ized WSGI paths with UTF-8. Patch by - Anthony Sottile. - -- Issue #26735: Fix :func:`os.urandom` on Solaris 11.3 and newer when reading - more than 1,024 bytes: call ``getrandom()`` multiple times with a limit of - 1024 bytes per call. - -- Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. - -- Issue #13952: Add .csv to mimetypes.types_map. Patch by Geoff Wilson. - -- Issue #26709: Fixed Y2038 problem in loading binary PLists. - -- Issue #23735: Handle terminal resizing with Readline 6.3+ by installing our - own SIGWINCH handler. Patch by Eric Price. - -- Issue #26586: In http.server, respond with "413 Request header fields too - large" if there are too many header fields to parse, rather than killing - the connection and raising an unhandled exception. Patch by Xiang Zhang. - -- Issue #22854: Change BufferedReader.writable() and - BufferedWriter.readable() to always return False. - -- Issue #25195: Fix a regression in mock.MagicMock. _Call is a subclass of - tuple (changeset 3603bae63c13 only works for classes) so we need to - implement __ne__ ourselves. Patch by Andrew Plummer. - -- Issue #26644: Raise ValueError rather than SystemError when a negative - length is passed to SSLSocket.recv() or read(). - -- Issue #23804: Fix SSL recv(0) and read(0) methods to return zero bytes - instead of up to 1024. - -- Issue #26616: Fixed a bug in datetime.astimezone() method. - -- Issue #21925: :func:`warnings.formatwarning` now catches exceptions on - ``linecache.getline(...)`` to be able to log :exc:`ResourceWarning` emitted - late during the Python shutdown process. - -- Issue #24266: Ctrl+C during Readline history search now cancels the search - mode when compiled with Readline 7. - -- Issue #26560: Avoid potential ValueError in BaseHandler.start_response. - Initial patch by Peter Inglesby. - -- [Security] Issue #26313: ssl.py _load_windows_store_certs fails if windows - cert store is empty. Patch by Baji. - -- Issue #26569: Fix :func:`pyclbr.readmodule` and :func:`pyclbr.readmodule_ex` - to support importing packages. - -- Issue #26499: Account for remaining Content-Length in - HTTPResponse.readline() and read1(). Based on patch by Silent Ghost. - Also document that HTTPResponse now supports these methods. - -- Issue #25320: Handle sockets in directories unittest discovery is scanning. - Patch from Victor van den Elzen. - -- Issue #16181: cookiejar.http2time() now returns None if year is higher than - datetime.MAXYEAR. - -- Issue #26513: Fixes platform module detection of Windows Server - -- Issue #23718: Fixed parsing time in week 0 before Jan 1. Original patch by - Tamás Bence Gedai. - -- Issue #20589: Invoking Path.owner() and Path.group() on Windows now raise - NotImplementedError instead of ImportError. - -- Issue #26177: Fixed the keys() method for Canvas and Scrollbar widgets. - -- Issue #15068: Got rid of excessive buffering in the fileinput module. - The bufsize parameter is no longer used. - -- Issue #2202: Fix UnboundLocalError in - AbstractDigestAuthHandler.get_algorithm_impls. Initial patch by Mathieu - Dupuy. - -- Issue #25718: Fixed pickling and copying the accumulate() iterator with - total is None. - -- Issue #26475: Fixed debugging output for regular expressions with the (?x) - flag. - -- Issue #26457: Fixed the subnets() methods in IP network classes for the case - when resulting prefix length is equal to maximal prefix length. - Based on patch by Xiang Zhang. - -- Issue #26385: Remove the file if the internal open() call in - NamedTemporaryFile() fails. Patch by Silent Ghost. - -- Issue #26402: Fix XML-RPC client to retry when the server shuts down a - persistent connection. This was a regression related to the new - http.client.RemoteDisconnected exception in 3.5.0a4. - -- Issue #25913: Leading ``<~`` is optional now in base64.a85decode() with - adobe=True. Patch by Swati Jaiswal. - -- Issue #26186: Remove an invalid type check in importlib.util.LazyLoader. - -- Issue #26367: importlib.__import__() raises SystemError like - builtins.__import__() when ``level`` is specified but without an accompanying - package specified. - -- Issue #26309: In the "socketserver" module, shut down the request (closing - the connected socket) when verify_request() returns false. Patch by Aviv - Palivoda. - -- [Security] Issue #25939: On Windows open the cert store readonly in - ssl.enum_certificates. - -- Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. - -- Issue #26117: The os.scandir() iterator now closes file descriptor not only - when the iteration is finished, but when it was failed with error. - -- Issue #25911: Restored support of bytes paths in os.walk() on Windows. - -- Issue #26045: Add UTF-8 suggestion to error message when posting a - non-Latin-1 string with http.client. - -- Issue #12923: Reset FancyURLopener's redirect counter even if there is an - exception. Based on patches by Brian Brazil and Daniel Rocco. - -- Issue #25945: Fixed a crash when unpickle the functools.partial object with - wrong state. Fixed a leak in failed functools.partial constructor. - "args" and "keywords" attributes of functools.partial have now always types - tuple and dict correspondingly. - -- Issue #26202: copy.deepcopy() now correctly copies range() objects with - non-atomic attributes. - -- Issue #23076: Path.glob() now raises a ValueError if it's called with an - invalid pattern. Patch by Thomas Nyberg. - -- Issue #19883: Fixed possible integer overflows in zipimport. - -- Issue #26227: On Windows, getnameinfo(), gethostbyaddr() and - gethostbyname_ex() functions of the socket module now decode the hostname - from the ANSI code page rather than UTF-8. - -- Issue #26147: xmlrpc now works with strings not encodable with used - non-UTF-8 encoding. - -- Issue #25935: Garbage collector now breaks reference loops with OrderedDict. - -- Issue #16620: Fixed AttributeError in msilib.Directory.glob(). - -- Issue #26013: Added compatibility with broken protocol 2 pickles created - in old Python 3 versions (3.4.3 and lower). - -- Issue #25850: Use cross-compilation by default for 64-bit Windows. - -- Issue #17633: Improve zipimport's support for namespace packages. - -- Issue #24705: Fix sysconfig._parse_makefile not expanding ${} vars - appearing before $() vars. - -- Issue #22138: Fix mock.patch behavior when patching descriptors. Restore - original values after patching. Patch contributed by Sean McCully. - -- Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode - option if it is safe to do so. - -- Issue #26012: Don't traverse into symlinks for ** pattern in - pathlib.Path.[r]glob(). - -- Issue #24120: Ignore PermissionError when traversing a tree with - pathlib.Path.[r]glob(). Patch by Ulrich Petri. - -- Issue #25447: fileinput now uses sys.stdin as-is if it does not have a - buffer attribute (restores backward compatibility). - -- Issue #25447: Copying the lru_cache() wrapper object now always works, - independedly from the type of the wrapped object (by returning the original - object unchanged). - -- Issue #24103: Fixed possible use after free in ElementTree.XMLPullParser. - -- Issue #25860: os.fwalk() no longer skips remaining directories when error - occurs. Original patch by Samson Lee. - -- Issue #25914: Fixed and simplified OrderedDict.__sizeof__. - -- Issue #25902: Fixed various refcount issues in ElementTree iteration. - -- Issue #25717: Restore the previous behaviour of tolerating most fstat() - errors when opening files. This was a regression in 3.5a1, and stopped - anonymous temporary files from working in special cases. - -- Issue #24903: Fix regression in number of arguments compileall accepts when - '-d' is specified. The check on the number of arguments has been dropped - completely as it never worked correctly anyway. - -- Issue #25764: In the subprocess module, preserve any exception caused by - fork() failure when preexec_fn is used. - -- Issue #6478: _strptime's regexp cache now is reset after changing timezone - with time.tzset(). - -- Issue #14285: When executing a package with the "python -m package" option, - and package initialization fails, a proper traceback is now reported. The - "runpy" module now lets exceptions from package initialization pass back to - the caller, rather than raising ImportError. - -- Issue #19771: Also in runpy and the "-m" option, omit the irrelevant - message ". . . is a package and cannot be directly executed" if the package - could not even be initialized (e.g. due to a bad ``*.pyc`` file). - -- Issue #25177: Fixed problem with the mean of very small and very large - numbers. As a side effect, statistics.mean and statistics.variance should - be significantly faster. - -- Issue #25718: Fixed copying object with state with boolean value is false. - -- Issue #10131: Fixed deep copying of minidom documents. Based on patch - by Marian Ganisin. - -- Issue #25725: Fixed a reference leak in pickle.loads() when unpickling - invalid data including tuple instructions. - -- Issue #25663: In the Readline completer, avoid listing duplicate global - names, and search the global namespace before searching builtins. - -- Issue #25688: Fixed file leak in ElementTree.iterparse() raising an error. - -- Issue #23914: Fixed SystemError raised by unpickler on broken pickle data. - -- Issue #25691: Fixed crash on deleting ElementTree.Element attributes. - -- Issue #25624: ZipFile now always writes a ZIP_STORED header for directory - entries. Patch by Dingyuan Wang. - -- Skip getaddrinfo if host is already resolved. - Patch by A. Jesse Jiryu Davis. - -- Issue #26050: Add asyncio.StreamReader.readuntil() method. - Patch by Марк Коренберг. - -- Issue #25924: Avoid unnecessary serialization of getaddrinfo(3) calls on - OS X versions 10.5 or higher. Original patch by A. Jesse Jiryu Davis. - -- Issue #26406: Avoid unnecessary serialization of getaddrinfo(3) calls on - current versions of OpenBSD and NetBSD. Patch by A. Jesse Jiryu Davis. - -- Issue #26848: Fix asyncio/subprocess.communicate() to handle empty input. - Patch by Jack O'Connor. - -- Issue #27040: Add loop.get_exception_handler method - -- Issue #27041: asyncio: Add loop.create_future method - -- Issue #27223: asyncio: Fix _read_ready and _write_ready to respect - _conn_lost. - Patch by Łukasz Langa. - -- Issue #22970: asyncio: Fix inconsistency cancelling Condition.wait. - Patch by David Coles. - -IDLE ----- - -- Issue #5124: Paste with text selected now replaces the selection on X11. - This matches how paste works on Windows, Mac, most modern Linux apps, - and ttk widgets. Original patch by Serhiy Storchaka. - -- Issue #24759: Make clear in idlelib.idle_test.__init__ that the directory - is a private implementation of test.test_idle and tool for maintainers. - -- Issue #27196: Stop 'ThemeChanged' warnings when running IDLE tests. - These persisted after other warnings were suppressed in #20567. - Apply Serhiy Storchaka's update_idletasks solution to four test files. - Record this additional advice in idle_test/README.txt - -- Issue #20567: Revise idle_test/README.txt with advice about avoiding - tk warning messages from tests. Apply advice to several IDLE tests. - -- Issue #27117: Make colorizer htest and turtledemo work with dark themes. - Move code for configuring text widget colors to a new function. - -- Issue #26673: When tk reports font size as 0, change to size 10. - Such fonts on Linux prevented the configuration dialog from opening. - -- Issue #21939: Add test for IDLE's percolator. - Original patch by Saimadhav Heblikar. - -- Issue #21676: Add test for IDLE's replace dialog. - Original patch by Saimadhav Heblikar. - -- Issue #18410: Add test for IDLE's search dialog. - Original patch by Westley Martínez. - -- Issue #21703: Add test for IDLE's undo delegator. - Original patch by Saimadhav Heblikar . - -- Issue #27044: Add ConfigDialog.remove_var_callbacks to stop memory leaks. - -- Issue #23977: Add more asserts to test_delegator. - -- Issue #20640: Add tests for idlelib.configHelpSourceEdit. - Patch by Saimadhav Heblikar. - -- In the 'IDLE-console differences' section of the IDLE doc, clarify - how running with IDLE affects sys.modules and the standard streams. - -- Issue #25507: fix incorrect change in IOBinding that prevented printing. - Augment IOBinding htest to include all major IOBinding functions. - -- Issue #25905: Revert unwanted conversion of ' to ’ RIGHT SINGLE QUOTATION - MARK in README.txt and open this and NEWS.txt with 'ascii'. - Re-encode CREDITS.txt to utf-8 and open it with 'utf-8'. - -Documentation -------------- - -- Issue #19489: Moved the search box from the sidebar to the header and footer - of each page. Patch by Ammar Askar. - -- Issue #24136: Document the new PEP 448 unpacking syntax of 3.5. - -- Issue #26736: Used HTTPS for external links in the documentation if possible. - -- Issue #6953: Rework the Readline module documentation to group related - functions together, and add more details such as what underlying Readline - functions and variables are accessed. - -- Issue #23606: Adds note to ctypes documentation regarding cdll.msvcrt. - -- Issue #25500: Fix documentation to not claim that __import__ is searched for - in the global scope. - -- Issue #26014: Update 3.x packaging documentation: - * "See also" links to the new docs are now provided in the legacy pages - * links to setuptools documentation have been updated - -Tests ------ - -- Issue #21916: Added tests for the turtle module. Patch by ingrid, - Gregory Loyse and Jelle Zijlstra. - -- Issue #26523: The multiprocessing thread pool (multiprocessing.dummy.Pool) - was untested. - -- Issue #26015: Added new tests for pickling iterators of mutable sequences. - -- Issue #26325: Added test.support.check_no_resource_warning() to check that - no ResourceWarning is emitted. - -- Issue #25940: Changed test_ssl to use self-signed.pythontest.net. This - avoids relying on svn.python.org, which recently changed root certificate. - -- Issue #25616: Tests for OrderedDict are extracted from test_collections - into separate file test_ordered_dict. - -- Issue #26583: Skip test_timestamp_overflow in test_import if bytecode - files cannot be written. - -Build ------ - -- Issue #26884: Fix linking extension modules for cross builds. - Patch by Xavier de Gaye. - -- Issue #22359: Disable the rules for running _freeze_importlib and pgen when - cross-compiling. The output of these programs is normally saved with the - source code anyway, and is still regenerated when doing a native build. - Patch by Xavier de Gaye. - -- Issue #27229: Fix the cross-compiling pgen rule for in-tree builds. Patch - by Xavier de Gaye. - -- Issue #21668: Link audioop, _datetime, _ctypes_test modules to libm, - except on Mac OS X. Patch written by Xavier de Gaye. - -- Issue #25702: A --with-lto configure option has been added that will - enable link time optimizations at build time during a make profile-opt. - Some compilers and toolchains are known to not produce stable code when - using LTO, be sure to test things thoroughly before relying on it. - It can provide a few % speed up over profile-opt alone. - -- Issue #26624: Adds validation of ucrtbase[d].dll version with warning - for old versions. - -- Issue #17603: Avoid error about nonexistant fileblocks.o file by using a - lower-level check for st_blocks in struct stat. - -- Issue #26079: Fixing the build output folder for tix-8.4.3.6. Patch by - Bjoern Thiel. - -- Issue #26465: Update Windows builds to use OpenSSL 1.0.2g. - -- Issue #24421: Compile Modules/_math.c once, before building extensions. - Previously it could fail to compile properly if the math and cmath builds - were concurrent. - -- Issue #25348: Added ``--pgo`` and ``--pgo-job`` arguments to - ``PCbuild\build.bat`` for building with Profile-Guided Optimization. The - old ``PCbuild\build_pgo.bat`` script is now deprecated, and simply calls - ``PCbuild\build.bat --pgo %*``. - -- Issue #25827: Add support for building with ICC to ``configure``, including - a new ``--with-icc`` flag. - -- Issue #25696: Fix installation of Python on UNIX with make -j9. - -- Issue #26930: Update OS X 10.5+ 32-bit-only installer to build - and link with OpenSSL 1.0.2h. - -- Issue #26268: Update Windows builds to use OpenSSL 1.0.2f. - -- Issue #25136: Support Apple Xcode 7's new textual SDK stub libraries. - -- Issue #24324: Do not enable unreachable code warnings when using - gcc as the option does not work correctly in older versions of gcc - and has been silently removed as of gcc-4.5. - -Windows -------- - -- Issue #27053: Updates make_zip.py to correctly generate library ZIP file. - -- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases - that don't include the contents of the include directory (that is, 1.0.2e - and later). - -- Issue #26071: bdist_wininst created binaries fail to start and find - 32bit Python - -- Issue #26073: Update the list of magic numbers in launcher - -- Issue #26065: Excludes venv from library when generating embeddable - distro. - -Tools/Demos ------------ - -- Issue #26799: Fix python-gdb.py: don't get C types once when the Python code - is loaded, but get C types on demand. The C types can change if - python-gdb.py is loaded before the Python executable. Patch written by Thomas - Ilsche. - -- Issue #26271: Fix the Freeze tool to properly use flags passed through - configure. Patch by Daniel Shaulov. - -- Issue #26489: Add dictionary unpacking support to Tools/parser/unparse.py. - Patch by Guo Ci Teo. - -- Issue #26316: Fix variable name typo in Argument Clinic. - -Misc ----- - -- Issue #17500, and https://github.com/python/pythondotorg/issues/945: Remove - unused and outdated icons. - - -What's New in Python 3.5.1 final? -================================= - -Release date: 2015-12-06 - -Core and Builtins ------------------ - -- Issue #25709: Fixed problem with in-place string concatenation and - utf-8 cache. - -Windows -------- - -- Issue #25715: Python 3.5.1 installer shows wrong upgrade path and incorrect - logic for launcher detection. - - -What's New in Python 3.5.1 release candidate 1? -=============================================== - -Release date: 2015-11-22 - -Core and Builtins ------------------ - -- Issue #25630: Fix a possible segfault during argument parsing in functions - that accept filesystem paths. - -- Issue #23564: Fixed a partially broken sanity check in the _posixsubprocess - internals regarding how fds_to_pass were passed to the child. The bug had - no actual impact as subprocess.py already avoided it. - -- Issue #25388: Fixed tokenizer crash when processing undecodable source code - with a null byte. - -- Issue #25462: The hash of the key now is calculated only once in most - operations in C implementation of OrderedDict. - -- Issue #22995: Default implementation of __reduce__ and __reduce_ex__ now - rejects builtin types with not defined __new__. - -- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node - when compiling AST from Python objects. - -- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec() - and eval() are passed bytes-like objects. These objects are not - necessarily terminated by a null byte, but the functions assumed they were. - -- Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that - was mutated by direct calls of dict methods. - -- Issue #25449: Iterating OrderedDict with keys with unstable hash now raises - KeyError in C implementations as well as in Python implementation. - -- Issue #25395: Fixed crash when highly nested OrderedDict structures were - garbage collected. - -- Issue #25274: sys.setrecursionlimit() now raises a RecursionError if the new - recursion limit is too low depending at the current recursion depth. Modify - also the "lower-water mark" formula to make it monotonic. This mark is used - to decide when the overflowed flag of the thread state is reset. - -- Issue #24402: Fix input() to prompt to the redirected stdout when - sys.stdout.fileno() fails. - -- Issue #24806: Prevent builtin types that are not allowed to be subclassed from - being subclassed through multiple inheritance. - -- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data. - -- Issue #25280: Import trace messages emitted in verbose (-v) mode are no - longer formatted twice. - -- Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the - getrandom() function instead of the getentropy() function. The getentropy() - function is blocking to generate very good quality entropy, os.urandom() - doesn't need such high-quality entropy. - -- Issue #25182: The stdprinter (used as sys.stderr before the io module is - imported at startup) now uses the backslashreplace error handler. - -- Issue #25131: Make the line number and column offset of set/dict literals and - comprehensions correspond to the opening brace. - -- Issue #25150: Hide the private _Py_atomic_xxx symbols from the public - Python.h header to fix a compilation error with OpenMP. PyThreadState_GET() - becomes an alias to PyThreadState_Get() to avoid ABI incompatibilies. - -Library -------- - -- Issue #25626: Change three zlib functions to accept sizes that fit in - Py_ssize_t, but internally cap those sizes to UINT_MAX. This resolves a - regression in 3.5 where GzipFile.read() failed to read chunks larger than 2 - or 4 GiB. The change affects the zlib.Decompress.decompress() max_length - parameter, the zlib.decompress() bufsize parameter, and the - zlib.Decompress.flush() length parameter. - -- Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) - when the OS gives priority to errors such as EACCES over EEXIST. - -- Issue #25593: Change semantics of EventLoop.stop() in asyncio. - -- Issue #6973: When we know a subprocess.Popen process has died, do - not allow the send_signal(), terminate(), or kill() methods to do - anything as they could potentially signal a different process. - -- Issue #25590: In the Readline completer, only call getattr() once per - attribute. - -- Issue #25498: Fix a crash when garbage-collecting ctypes objects created - by wrapping a memoryview. This was a regression made in 3.5a1. Based - on patch by Eryksun. - -- Issue #25584: Added "escape" to the __all__ list in the glob module. - -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. - -- Issue #25446: Fix regression in smtplib's AUTH LOGIN support. - -- Issue #18010: Fix the pydoc web server's module search function to handle - exceptions from importing packages. - -- Issue #25554: Got rid of circular references in regular expression parsing. - -- Issue #25510: fileinput.FileInput.readline() now returns b'' instead of '' - at the end if the FileInput was opened with binary mode. - Patch by Ryosuke Ito. - -- Issue #25503: Fixed inspect.getdoc() for inherited docstrings of properties. - Original patch by John Mark Vandenberg. - -- Issue #25515: Always use os.urandom as a source of randomness in uuid.uuid4. - -- Issue #21827: Fixed textwrap.dedent() for the case when largest common - whitespace is a substring of smallest leading whitespace. - Based on patch by Robert Li. - -- Issue #25447: The lru_cache() wrapper objects now can be copied and pickled - (by returning the original object unchanged). - -- Issue #25390: typing: Don't crash on Union[str, Pattern]. - -- Issue #25441: asyncio: Raise error from drain() when socket is closed. - -- Issue #25410: Cleaned up and fixed minor bugs in C implementation of - OrderedDict. - -- Issue #25411: Improved Unicode support in SMTPHandler through better use of - the email package. Thanks to user simon04 for the patch. - -- Issue #25407: Remove mentions of the formatter module being removed in - Python 3.6. - -- Issue #25406: Fixed a bug in C implementation of OrderedDict.move_to_end() - that caused segmentation fault or hang in iterating after moving several - items to the start of ordered dict. - -- Issue #25364: zipfile now works in threads disabled builds. - -- Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both - decode_data and enable_SMTPUTF8 are set to true. - -- Issue #25316: distutils raises OSError instead of DistutilsPlatformError - when MSVC is not installed. - -- Issue #25380: Fixed protocol for the STACK_GLOBAL opcode in - pickletools.opcodes. - -- Issue #23972: Updates asyncio datagram create method allowing reuseport - and reuseaddr socket options to be set prior to binding the socket. - Mirroring the existing asyncio create_server method the reuseaddr option - for datagram sockets defaults to True if the O/S is 'posix' (except if the - platform is Cygwin). Patch by Chris Laws. - -- Issue #25304: Add asyncio.run_coroutine_threadsafe(). This lets you - submit a coroutine to a loop from another thread, returning a - concurrent.futures.Future. By Vincent Michel. - -- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the - first question mark (?) rather than the last. Patch from Xiang Zhang. - -- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the - query part of the URL as if it were a path. Patch from Xiang Zhang. - -- Issue #24483: C implementation of functools.lru_cache() now calculates key's - hash only once. - -- Issue #22958: Constructor and update method of weakref.WeakValueDictionary - now accept the self and the dict keyword arguments. - -- Issue #22609: Constructor of collections.UserDict now accepts the self keyword - argument. - -- Issue #25111: Fixed comparison of traceback.FrameSummary. - -- Issue #25262: Added support for BINBYTES8 opcode in Python implementation of - unpickler. Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8 - opcodes no longer silently ignored on 32-bit platforms in C implementation. - -- Issue #25034: Fix string.Formatter problem with auto-numbering and - nested format_specs. Patch by Anthon van der Neut. - -- Issue #25233: Rewrite the guts of asyncio.Queue and - asyncio.Semaphore to be more understandable and correct. - -- Issue #25203: Failed readline.set_completer_delims() no longer left the - module in inconsistent state. - -- Issue #23600: Default implementation of tzinfo.fromutc() was returning - wrong results in some cases. - -- Issue #23329: Allow the ssl module to be built with older versions of - LibreSSL. - -- Prevent overflow in _Unpickler_Read. - -- Issue #25047: The XML encoding declaration written by Element Tree now - respects the letter case given by the user. This restores the ability to - write encoding names in uppercase like "UTF-8", which worked in Python 2. - -- Issue #25135: Make deque_clear() safer by emptying the deque before clearing. - This helps avoid possible reentrancy issues. - -- Issue #19143: platform module now reads Windows version from kernel32.dll to - avoid compatibility shims. - -- Issue #25092: Fix datetime.strftime() failure when errno was already set to - EINVAL. - -- Issue #23517: Fix rounding in fromtimestamp() and utcfromtimestamp() methods - of datetime.datetime: microseconds are now rounded to nearest with ties - going to nearest even integer (ROUND_HALF_EVEN), instead of being rounding - towards minus infinity (ROUND_FLOOR). It's important that these methods use - the same rounding mode than datetime.timedelta to keep the property: - (datetime(1970,1,1) + timedelta(seconds=t)) == datetime.utcfromtimestamp(t). - It also the rounding mode used by round(float) for example. - -- Issue #25155: Fix datetime.datetime.now() and datetime.datetime.utcnow() on - Windows to support date after year 2038. It was a regression introduced in - Python 3.5.0. - -- Issue #25108: Omitted internal frames in traceback functions print_stack(), - format_stack(), and extract_stack() called without arguments. - -- Issue #25118: Fix a regression of Python 3.5.0 in os.waitpid() on Windows. - -- Issue #24684: socket.socket.getaddrinfo() now calls - PyUnicode_AsEncodedString() instead of calling the encode() method of the - host, to handle correctly custom string with an encode() method which doesn't - return a byte string. The encoder of the IDNA codec is now called directly - instead of calling the encode() method of the string. - -- Issue #25060: Correctly compute stack usage of the BUILD_MAP opcode. - -- Issue #24857: Comparing call_args to a long sequence now correctly returns a - boolean result instead of raising an exception. Patch by A Kaptur. - -- Issue #23144: Make sure that HTMLParser.feed() returns all the data, even - when convert_charrefs is True. - -- Issue #24982: shutil.make_archive() with the "zip" format now adds entries - for directories (including empty directories) in ZIP file. - -- Issue #25019: Fixed a crash caused by setting non-string key of expat parser. - Based on patch by John Leitch. - -- Issue #16180: Exit pdb if file has syntax error, instead of trapping user - in an infinite loop. Patch by Xavier de Gaye. - -- Issue #24891: Fix a race condition at Python startup if the file descriptor - of stdin (0), stdout (1) or stderr (2) is closed while Python is creating - sys.stdin, sys.stdout and sys.stderr objects. These attributes are now set - to None if the creation of the object failed, instead of raising an OSError - exception. Initial patch written by Marco Paolini. - -- Issue #24992: Fix error handling and a race condition (related to garbage - collection) in collections.OrderedDict constructor. - -- Issue #24881: Fixed setting binary mode in Python implementation of FileIO - on Windows and Cygwin. Patch from Akira Li. - -- Issue #25578: Fix (another) memory leak in SSLSocket.getpeercer(). - -- Issue #25530: Disable the vulnerable SSLv3 protocol by default when creating - ssl.SSLContext. - -- Issue #25569: Fix memory leak in SSLSocket.getpeercert(). - -- Issue #25471: Sockets returned from accept() shouldn't appear to be - nonblocking. - -- Issue #25319: When threading.Event is reinitialized, the underlying condition - should use a regular lock rather than a recursive lock. - -- Issue #21112: Fix regression in unittest.expectedFailure on subclasses. - Patch from Berker Peksag. - -- Issue #24764: cgi.FieldStorage.read_multi() now ignores the Content-Length - header in part headers. Patch written by Peter Landry and reviewed by Pierre - Quentel. - -- Issue #24913: Fix overrun error in deque.index(). - Found by John Leitch and Bryce Darling. - -- Issue #24774: Fix docstring in http.server.test. Patch from Chiu-Hsiang Hsu. - -- Issue #21159: Improve message in configparser.InterpolationMissingOptionError. - Patch from Łukasz Langa. - -- Issue #20362: Honour TestCase.longMessage correctly in assertRegex. - Patch from Ilia Kurenkov. - -- Issue #23572: Fixed functools.singledispatch on classes with falsy - metaclasses. Patch by Ethan Furman. - -- asyncio: ensure_future() now accepts awaitable objects. - -IDLE ----- - -- Issue #15348: Stop the debugger engine (normally in a user process) - before closing the debugger window (running in the IDLE process). - This prevents the RuntimeErrors that were being caught and ignored. - -- Issue #24455: Prevent IDLE from hanging when a) closing the shell while the - debugger is active (15347); b) closing the debugger with the [X] button - (15348); and c) activating the debugger when already active (24455). - The patch by Mark Roseman does this by making two changes. - 1. Suspend and resume the gui.interaction method with the tcl vwait - mechanism intended for this purpose (instead of root.mainloop & .quit). - 2. In gui.run, allow any existing interaction to terminate first. - -- Change 'The program' to 'Your program' in an IDLE 'kill program?' message - to make it clearer that the program referred to is the currently running - user program, not IDLE itself. - -- Issue #24750: Improve the appearance of the IDLE editor window status bar. - Patch by Mark Roseman. - -- Issue #25313: Change the handling of new built-in text color themes to better - address the compatibility problem introduced by the addition of IDLE Dark. - Consistently use the revised idleConf.CurrentTheme everywhere in idlelib. - -- Issue #24782: Extension configuration is now a tab in the IDLE Preferences - dialog rather than a separate dialog. The former tabs are now a sorted - list. Patch by Mark Roseman. - -- Issue #22726: Re-activate the config dialog help button with some content - about the other buttons and the new IDLE Dark theme. - -- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme. - It is more or less IDLE Classic inverted, with a cobalt blue background. - Strings, comments, keywords, ... are still green, red, orange, ... . - To use it with IDLEs released before November 2015, hit the - 'Save as New Custom Theme' button and enter a new name, - such as 'Custom Dark'. The custom theme will work with any IDLE - release, and can be modified. - -- Issue #25224: README.txt is now an idlelib index for IDLE developers and - curious users. The previous user content is now in the IDLE doc chapter. - 'IDLE' now means 'Integrated Development and Learning Environment'. - -- Issue #24820: Users can now set breakpoint colors in - Settings -> Custom Highlighting. Original patch by Mark Roseman. - -- Issue #24972: Inactive selection background now matches active selection - background, as configured by users, on all systems. Found items are now - always highlighted on Windows. Initial patch by Mark Roseman. - -- Issue #24570: Idle: make calltip and completion boxes appear on Macs - affected by a tk regression. Initial patch by Mark Roseman. - -- Issue #24988: Idle ScrolledList context menus (used in debugger) - now work on Mac Aqua. Patch by Mark Roseman. - -- Issue #24801: Make right-click for context menu work on Mac Aqua. - Patch by Mark Roseman. - -- Issue #25173: Associate tkinter messageboxes with a specific widget. - For Mac OSX, make them a 'sheet'. Patch by Mark Roseman. - -- Issue #25198: Enhance the initial html viewer now used for Idle Help. - * Properly indent fixed-pitch text (patch by Mark Roseman). - * Give code snippet a very Sphinx-like light blueish-gray background. - * Re-use initial width and height set by users for shell and editor. - * When the Table of Contents (TOC) menu is used, put the section header - at the top of the screen. - -- Issue #25225: Condense and rewrite Idle doc section on text colors. - -- Issue #21995: Explain some differences between IDLE and console Python. - -- Issue #22820: Explain need for *print* when running file from Idle editor. - -- Issue #25224: Doc: augment Idle feature list and no-subprocess section. - -- Issue #25219: Update doc for Idle command line options. - Some were missing and notes were not correct. - -- Issue #24861: Most of idlelib is private and subject to change. - Use idleib.idle.* to start Idle. See idlelib.__init__.__doc__. - -- Issue #25199: Idle: add synchronization comments for future maintainers. - -- Issue #16893: Replace help.txt with help.html for Idle doc display. - The new idlelib/help.html is rstripped Doc/build/html/library/idle.html. - It looks better than help.txt and will better document Idle as released. - The tkinter html viewer that works for this file was written by Mark Roseman. - The now unused EditorWindow.HelpDialog class and helt.txt file are deprecated. - -- Issue #24199: Deprecate unused idlelib.idlever with possible removal in 3.6. - -- Issue #24790: Remove extraneous code (which also create 2 & 3 conflicts). - -Documentation -------------- - -- Issue #22558: Add remaining doc links to source code for Python-coded modules. - Patch by Yoni Lavi. - -- Issue #12067: Rewrite Comparisons section in the Expressions chapter of the - language reference. Some of the details of comparing mixed types were - incorrect or ambiguous. NotImplemented is only relevant at a lower level - than the Expressions chapter. Added details of comparing range() objects, - and default behaviour and consistency suggestions for user-defined classes. - Patch from Andy Maier. - -- Issue #24952: Clarify the default size argument of stack_size() in - the "threading" and "_thread" modules. Patch from Mattip. - -- Issue #23725: Overhaul tempfile docs. Note deprecated status of mktemp. - Patch from Zbigniew Jędrzejewski-Szmek. - -- Issue #24808: Update the types of some PyTypeObject fields. Patch by - Joseph Weston. - -- Issue #22812: Fix unittest discovery examples. - Patch from Pam McA'Nulty. - -Tests ------ - -- Issue #25449: Added tests for OrderedDict subclasses. - -- Issue #25099: Make test_compileall not fail when an entry on sys.path cannot - be written to (commonly seen in administrative installs on Windows). - -- Issue #23919: Prevents assert dialogs appearing in the test suite. - -- ``PCbuild\rt.bat`` now accepts an unlimited number of arguments to pass along - to regrtest.py. Previously there was a limit of 9. - -Build ------ - -- Issue #24915: Add LLVM support for PGO builds and use the test suite to - generate the profile data. Initial patch by Alecsandru Patrascu of Intel. - -- Issue #24910: Windows MSIs now have unique display names. - -- Issue #24986: It is now possible to build Python on Windows without errors - when external libraries are not available. - -Windows -------- - -- Issue #25450: Updates shortcuts to start Python in installation directory. - -- Issue #25164: Changes default all-users install directory to match per-user - directory. - -- Issue #25143: Improves installer error messages for unsupported platforms. - -- Issue #25163: Display correct directory in installer when using non-default - settings. - -- Issue #25361: Disables use of SSE2 instructions in Windows 32-bit build - -- Issue #25089: Adds logging to installer for case where launcher is not - selected on upgrade. - -- Issue #25165: Windows uninstallation should not remove launcher if other - versions remain - -- Issue #25112: py.exe launcher is missing icons - -- Issue #25102: Windows installer does not precompile for -O or -OO. - -- Issue #25081: Makes Back button in installer go back to upgrade page when - upgrading. - -- Issue #25091: Increases font size of the installer. - -- Issue #25126: Clarifies that the non-web installer will download some - components. - -- Issue #25213: Restores requestedExecutionLevel to manifest to disable - UAC virtualization. - -- Issue #25022: Removed very outdated PC/example_nt/ directory. - -Tools/Demos ------------ - -- Issue #25440: Fix output of python-config --extension-suffix. - - -What's New in Python 3.5.0 final? -================================= - -Release date: 2015-09-13 - -Build ------ - -- Issue #25071: Windows installer should not require TargetDir - parameter when installing quietly. - - -What's New in Python 3.5.0 release candidate 4? -=============================================== - -Release date: 2015-09-09 - -Library -------- - -- Issue #25029: Fixes MemoryError in test_strptime. - -Build ------ - -- Issue #25027: Reverts partial-static build options and adds - vcruntime140.dll to Windows installation. - - -What's New in Python 3.5.0 release candidate 3? -=============================================== - -Release date: 2015-09-07 - -Core and Builtins ------------------ - -- Issue #24305: Prevent import subsystem stack frames from being counted - by the warnings.warn(stacklevel=) parameter. - -- Issue #24912: Prevent __class__ assignment to immutable built-in objects. - -- Issue #24975: Fix AST compilation for PEP 448 syntax. - -Library -------- - -- Issue #24917: time_strftime() buffer over-read. - -- Issue #24748: To resolve a compatibility problem found with py2exe and - pywin32, imp.load_dynamic() once again ignores previously loaded modules - to support Python modules replacing themselves with extension modules. - Patch by Petr Viktorin. - -- Issue #24635: Fixed a bug in typing.py where isinstance([], typing.Iterable) - would return True once, then False on subsequent calls. - -- Issue #24989: Fixed buffer overread in BytesIO.readline() if a position is - set beyond size. Based on patch by John Leitch. - -- Issue #24913: Fix overrun error in deque.index(). - Found by John Leitch and Bryce Darling. - - -What's New in Python 3.5.0 release candidate 2? -=============================================== - -Release date: 2015-08-25 - -Core and Builtins ------------------ - -- Issue #24769: Interpreter now starts properly when dynamic loading - is disabled. Patch by Petr Viktorin. - -- Issue #21167: NAN operations are now handled correctly when python is - compiled with ICC even if -fp-model strict is not specified. - -- Issue #24492: A "package" lacking a __name__ attribute when trying to perform - a ``from .. import ...`` statement will trigger an ImportError instead of an - AttributeError. - -Library -------- - -- Issue #24847: Removes vcruntime140.dll dependency from Tcl/Tk. - -- Issue #24839: platform._syscmd_ver raises DeprecationWarning - -- Issue #24867: Fix Task.get_stack() for 'async def' coroutines - - -What's New in Python 3.5.0 release candidate 1? -=============================================== - -Release date: 2015-08-09 - -Core and Builtins ------------------ - -- Issue #24667: Resize odict in all cases that the underlying dict resizes. - -Library -------- - -- Issue #24824: Signatures of codecs.encode() and codecs.decode() now are - compatible with pydoc. - -- Issue #24634: Importing uuid should not try to load libc on Windows - -- Issue #24798: _msvccompiler.py doesn't properly support manifests - -- Issue #4395: Better testing and documentation of binary operators. - Patch by Martin Panter. - -- Issue #23973: Update typing.py from GitHub repo. - -- Issue #23004: mock_open() now reads binary data correctly when the type of - read_data is bytes. Initial patch by Aaron Hill. - -- Issue #23888: Handle fractional time in cookie expiry. Patch by ssh. - -- Issue #23652: Make it possible to compile the select module against the - libc headers from the Linux Standard Base, which do not include some - EPOLL macros. Patch by Matt Frank. - -- Issue #22932: Fix timezones in email.utils.formatdate. - Patch from Dmitry Shachnev. - -- Issue #23779: imaplib raises TypeError if authenticator tries to abort. - Patch from Craig Holmquist. - -- Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch - written by Matthieu Gautier. - -- Issue #23254: Document how to close the TCPServer listening socket. - Patch from Martin Panter. - -- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. - -- Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. - -- Issue #24791: Fix grammar regression for call syntax: 'g(\*a or b)'. - -IDLE ----- - -- Issue #23672: Allow Idle to edit and run files with astral chars in name. - Patch by Mohd Sanad Zaki Rizvi. - -- Issue #24745: Idle editor default font. Switch from Courier to - platform-sensitive TkFixedFont. This should not affect current customized - font selections. If there is a problem, edit $HOME/.idlerc/config-main.cfg - and remove 'fontxxx' entries from [Editor Window]. Patch by Mark Roseman. - -- Issue #21192: Idle editor. When a file is run, put its name in the restart bar. - Do not print false prompts. Original patch by Adnan Umer. - -- Issue #13884: Idle menus. Remove tearoff lines. Patch by Roger Serwy. - -Documentation -------------- - -- Issue #24129: Clarify the reference documentation for name resolution. - This includes removing the assumption that readers will be familiar with the - name resolution scheme Python used prior to the introduction of lexical - scoping for function namespaces. Patch by Ivan Levkivskyi. - -- Issue #20769: Improve reload() docs. Patch by Dorian Pula. - -- Issue #23589: Remove duplicate sentence from the FAQ. Patch by Yongzhi Pan. - -- Issue #24729: Correct IO tutorial to match implementation regarding - encoding parameter to open function. - -Tests ------ - -- Issue #24751: When running regrtest with the ``-w`` command line option, - a test run is no longer marked as a failure if all tests succeed when - re-run. - - -What's New in Python 3.5.0 beta 4? -================================== - -Release date: 2015-07-26 - -Core and Builtins ------------------ - -- Issue #23573: Restored optimization of bytes.rfind() and bytearray.rfind() - for single-byte argument on Linux. - -- Issue #24569: Make PEP 448 dictionary evaluation more consistent. - -- Issue #24583: Fix crash when set is mutated while being updated. - -- Issue #24407: Fix crash when dict is mutated while being updated. - -- Issue #24619: New approach for tokenizing async/await. As a consequence, - it is now possible to have one-line 'async def foo(): await ..' functions. - -- Issue #24687: Plug refleak on SyntaxError in function parameters - annotations. - -- Issue #15944: memoryview: Allow arbitrary formats when casting to bytes. - Patch by Martin Panter. - -Library -------- - -- Issue #23441: rcompleter now prints a tab character instead of displaying - possible completions for an empty word. Initial patch by Martin Sekera. - -- Issue #24683: Fixed crashes in _json functions called with arguments of - inappropriate type. - -- Issue #21697: shutil.copytree() now correctly handles symbolic links that - point to directories. Patch by Eduardo Seabra and Thomas Kluyver. - -- Issue #14373: Fixed segmentation fault when gc.collect() is called during - constructing lru_cache (C implementation). - -- Issue #24695: Fix a regression in traceback.print_exception(). If - exc_traceback is None we shouldn't print a traceback header like described - in the documentation. - -- Issue #24620: Random.setstate() now validates the value of state last element. - -- Issue #22485: Fixed an issue that caused `inspect.getsource` to return - incorrect results on nested functions. - -- Issue #22153: Improve unittest docs. Patch from Martin Panter and evilzero. - -- Issue #24580: Symbolic group references to open group in re patterns now are - explicitly forbidden as well as numeric group references. - -- Issue #24206: Fixed __eq__ and __ne__ methods of inspect classes. - -- Issue #24631: Fixed regression in the timeit module with multiline setup. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #24608: chunk.Chunk.read() now always returns bytes, not str. - -- Issue #18684: Fixed reading out of the buffer in the re module. - -- Issue #24259: tarfile now raises a ReadError if an archive is truncated - inside a data segment. - -- Issue #15014: SMTP.auth() and SMTP.login() now support RFC 4954's optional - initial-response argument to the SMTP AUTH command. - -- Issue #24669: Fix inspect.getsource() for 'async def' functions. - Patch by Kai Groner. - -- Issue #24688: ast.get_docstring() for 'async def' functions. - -Build ------ - -- Issue #24603: Update Windows builds and OS X 10.5 installer to use OpenSSL - 1.0.2d. - - -What's New in Python 3.5.0 beta 3? -================================== - -Release date: 2015-07-05 - -Core and Builtins ------------------ - -- Issue #24467: Fixed possible buffer over-read in bytearray. The bytearray - object now always allocates place for trailing null byte and it's buffer now - is always null-terminated. - -- Upgrade to Unicode 8.0.0. - -- Issue #24345: Add Py_tp_finalize slot for the stable ABI. - -- Issue #24400: Introduce a distinct type for PEP 492 coroutines; add - types.CoroutineType, inspect.getcoroutinestate, inspect.getcoroutinelocals; - coroutines no longer use CO_GENERATOR flag; sys.set_coroutine_wrapper - works only for 'async def' coroutines; inspect.iscoroutine no longer - uses collections.abc.Coroutine, it's intended to test for pure 'async def' - coroutines only; add new opcode: GET_YIELD_FROM_ITER; fix generators wrapper - used in types.coroutine to be instance of collections.abc.Generator; - collections.abc.Awaitable and collections.abc.Coroutine can no longer - be used to detect generator-based coroutines--use inspect.isawaitable - instead. - -- Issue #24450: Add gi_yieldfrom to generators and cr_await to coroutines. - Contributed by Benno Leslie and Yury Selivanov. - -- Issue #19235: Add new RecursionError exception. Patch by Georg Brandl. - -Library -------- - -- Issue #21750: mock_open.read_data can now be read from each instance, as it - could in Python 3.3. - -- Issue #24552: Fix use after free in an error case of the _pickle module. - -- Issue #24514: tarfile now tolerates number fields consisting of only - whitespace. - -- Issue #19176: Fixed doctype() related bugs in C implementation of ElementTree. - A deprecation warning no longer issued by XMLParser subclass with default - doctype() method. Direct call of doctype() now issues a warning. Parser's - doctype() now is not called if target's doctype() is called. Based on patch - by Martin Panter. - -- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize - for tab-indented blocks. - -- Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() - functions of the audioop module. - -- Issue #24336: The contextmanager decorator now works with functions with - keyword arguments called "func" and "self". Patch by Martin Panter. - -- Issue #24522: Fix possible integer overflow in json accelerator module. - -- Issue #24489: ensure a previously set C errno doesn't disturb cmath.polar(). - -- Issue #24408: Fixed AttributeError in measure() and metrics() methods of - tkinter.Font. - -- Issue #14373: C implementation of functools.lru_cache() now can be used with - methods. - -- Issue #24347: Set KeyError if PyDict_GetItemWithError returns NULL. - -- Issue #24348: Drop superfluous incref/decref. - -- Issue #24359: Check for changed OrderedDict size during iteration. - -- Issue #24368: Support keyword arguments in OrderedDict methods. - -- Issue #24362: Simplify the C OrderedDict fast nodes resize logic. - -- Issue #24377: Fix a ref leak in OrderedDict.__repr__. - -- Issue #24369: Defend against key-changes during iteration. - -Tests ------ - -- Issue #24373: _testmultiphase and xxlimited now use tp_traverse and - tp_finalize to avoid reference leaks encountered when combining tp_dealloc - with PyType_FromSpec (see issue #16690 for details) - -Documentation -------------- - -- Issue #24458: Update documentation to cover multi-phase initialization for - extension modules (PEP 489). Patch by Petr Viktorin. - -- Issue #24351: Clarify what is meant by "identifier" in the context of - string.Template instances. - -Build ------ - -- Issue #24432: Update Windows builds and OS X 10.5 installer to use OpenSSL - 1.0.2c. - - -What's New in Python 3.5.0 beta 2? -================================== - -Release date: 2015-05-31 - -Core and Builtins ------------------ - -- Issue #24284: The startswith and endswith methods of the str class no longer - return True when finding the empty string and the indexes are completely out - of range. - -- Issue #24115: Update uses of PyObject_IsTrue(), PyObject_Not(), - PyObject_IsInstance(), PyObject_RichCompareBool() and _PyDict_Contains() - to check for and handle errors correctly. - -- Issue #24328: Fix importing one character extension modules. - -- Issue #11205: In dictionary displays, evaluate the key before the value. - -- Issue #24285: Fixed regression that prevented importing extension modules - from inside packages. Patch by Petr Viktorin. - -Library -------- - -- Issue #23247: Fix a crash in the StreamWriter.reset() of CJK codecs. - -- Issue #24270: Add math.isclose() and cmath.isclose() functions as per PEP 485. - Contributed by Chris Barker and Tal Einat. - -- Issue #5633: Fixed timeit when the statement is a string and the setup is not. - -- Issue #24326: Fixed audioop.ratecv() with non-default weightB argument. - Original patch by David Moore. - -- Issue #16991: Add a C implementation of OrderedDict. - -- Issue #23934: Fix inspect.signature to fail correctly for builtin types - lacking signature information. Initial patch by James Powell. - - -What's New in Python 3.5.0 beta 1? -================================== - -Release date: 2015-05-24 - -Core and Builtins ------------------ - -- Issue #24276: Fixed optimization of property descriptor getter. - -- Issue #24268: PEP 489: Multi-phase extension module initialization. - Patch by Petr Viktorin. - -- Issue #23955: Add pyvenv.cfg option to suppress registry/environment - lookup for generating sys.path on Windows. - -- Issue #24257: Fixed system error in the comparison of faked - types.SimpleNamespace. - -- Issue #22939: Fixed integer overflow in iterator object. Patch by - Clement Rouault. - -- Issue #23985: Fix a possible buffer overrun when deleting a slice from - the front of a bytearray and then appending some other bytes data. - -- Issue #24102: Fixed exception type checking in standard error handlers. - -- Issue #15027: The UTF-32 encoder is now 3x to 7x faster. - -- Issue #23290: Optimize set_merge() for cases where the target is empty. - (Contributed by Serhiy Storchaka.) - -- Issue #2292: PEP 448: Additional Unpacking Generalizations. - -- Issue #24096: Make warnings.warn_explicit more robust against mutation of the - warnings.filters list. - -- Issue #23996: Avoid a crash when a delegated generator raises an - unnormalized StopIteration exception. Patch by Stefan Behnel. - -- Issue #23910: Optimize property() getter calls. Patch by Joe Jevnik. - -- Issue #23911: Move path-based importlib bootstrap code to a separate - frozen module. - -- Issue #24192: Fix namespace package imports. - -- Issue #24022: Fix tokenizer crash when processing undecodable source code. - -- Issue #9951: Added a hex() method to bytes, bytearray, and memoryview. - -- Issue #22906: PEP 479: Change StopIteration handling inside generators. - -- Issue #24017: PEP 492: Coroutines with async and await syntax. - -Library -------- - -- Issue #14373: Added C implementation of functools.lru_cache(). Based on - patches by Matt Joiner and Alexey Kachayev. - -- Issue #24230: The tempfile module now accepts bytes for prefix, suffix and dir - parameters and returns bytes in such situations (matching the os module APIs). - -- Issue #22189: collections.UserString now supports __getnewargs__(), - __rmod__(), casefold(), format_map(), isprintable(), and maketrans(). - Patch by Joe Jevnik. - -- Issue #24244: Prevents termination when an invalid format string is - encountered on Windows in strftime. - -- Issue #23973: PEP 484: Add the typing module. - -- Issue #23086: The collections.abc.Sequence() abstract base class added - *start* and *stop* parameters to the index() mixin. - Patch by Devin Jeanpierre. - -- Issue #20035: Replaced the ``tkinter._fix`` module used for setting up the - Tcl/Tk environment on Windows with a private function in the ``_tkinter`` - module that makes no permanent changes to the environment. - -- Issue #24257: Fixed segmentation fault in sqlite3.Row constructor with faked - cursor type. - -- Issue #15836: assertRaises(), assertRaisesRegex(), assertWarns() and - assertWarnsRegex() assertments now check the type of the first argument - to prevent possible user error. Based on patch by Daniel Wagner-Hall. - -- Issue #9858: Add missing method stubs to _io.RawIOBase. Patch by Laura - Rupprecht. - -- Issue #22955: attrgetter, itemgetter and methodcaller objects in the operator - module now support pickling. Added readable and evaluable repr for these - objects. Based on patch by Josh Rosenberg. - -- Issue #22107: tempfile.gettempdir() and tempfile.mkdtemp() now try again - when a directory with the chosen name already exists on Windows as well as - on Unix. tempfile.mkstemp() now fails early if parent directory is not - valid (not exists or is a file) on Windows. - -- Issue #23780: Improved error message in os.path.join() with single argument. - -- Issue #6598: Increased time precision and random number range in - email.utils.make_msgid() to strengthen the uniqueness of the message ID. - -- Issue #24091: Fixed various crashes in corner cases in C implementation of - ElementTree. - -- Issue #21931: msilib.FCICreate() now raises TypeError in the case of a bad - argument instead of a ValueError with a bogus FCI error number. - Patch by Jeffrey Armstrong. - -- Issue #13866: *quote_via* argument added to urllib.parse.urlencode. - -- Issue #20098: New mangle_from policy option for email, default True - for compat32, but False for all other policies. - -- Issue #24211: The email library now supports RFC 6532: it can generate - headers using utf-8 instead of encoded words. - -- Issue #16314: Added support for the LZMA compression in distutils. - -- Issue #21804: poplib now supports RFC 6856 (UTF8). - -- Issue #18682: Optimized pprint functions for builtin scalar types. - -- Issue #22027: smtplib now supports RFC 6531 (SMTPUTF8). - -- Issue #23488: Random generator objects now consume 2x less memory on 64-bit. - -- Issue #1322: platform.dist() and platform.linux_distribution() functions are - now deprecated. Initial patch by Vajrasky Kok. - -- Issue #22486: Added the math.gcd() function. The fractions.gcd() function - now is deprecated. Based on patch by Mark Dickinson. - -- Issue #24064: Property() docstrings are now writeable. - (Patch by Berker Peksag.) - -- Issue #22681: Added support for the koi8_t encoding. - -- Issue #22682: Added support for the kz1048 encoding. - -- Issue #23796: peek and read1 methods of BufferedReader now raise ValueError - if they called on a closed object. Patch by John Hergenroeder. - -- Issue #21795: smtpd now supports the 8BITMIME extension whenever - the new *decode_data* constructor argument is set to False. - -- Issue #24155: optimize heapq.heapify() for better cache performance - when heapifying large lists. - -- Issue #21800: imaplib now supports RFC 5161 (enable), RFC 6855 - (utf8/internationalized email) and automatically encodes non-ASCII - usernames and passwords to UTF8. - -- Issue #20274: When calling a _sqlite.Connection, it now complains if passed - any keyword arguments. Previously it silently ignored them. - -- Issue #20274: Remove ignored and erroneous "kwargs" parameters from three - METH_VARARGS methods on _sqlite.Connection. - -- Issue #24134: assertRaises(), assertRaisesRegex(), assertWarns() and - assertWarnsRegex() checks now emits a deprecation warning when callable is - None or keyword arguments except msg is passed in the context manager mode. - -- Issue #24018: Add a collections.abc.Generator abstract base class. - Contributed by Stefan Behnel. - -- Issue #23880: Tkinter's getint() and getdouble() now support Tcl_Obj. - Tkinter's getdouble() now supports any numbers (in particular int). - -- Issue #22619: Added negative limit support in the traceback module. - Based on patch by Dmitry Kazakov. - -- Issue #24094: Fix possible crash in json.encode with poorly behaved dict - subclasses. - -- Issue #9246: On POSIX, os.getcwd() now supports paths longer than 1025 bytes. - Patch written by William Orr. - -- Issue #17445: add difflib.diff_bytes() to support comparison of - byte strings (fixes a regression from Python 2). - -- Issue #23917: Fall back to sequential compilation when ProcessPoolExecutor - doesn't exist. Patch by Claudiu Popa. - -- Issue #23008: Fixed resolving attributes with boolean value is False in pydoc. - -- Fix asyncio issue 235: LifoQueue and PriorityQueue's put didn't - increment unfinished tasks (this bug was introduced when - JoinableQueue was merged with Queue). - -- Issue #23908: os functions now reject paths with embedded null character - on Windows instead of silently truncating them. - -- Issue #23728: binascii.crc_hqx() could return an integer outside of the range - 0-0xffff for empty data. - -- Issue #23887: urllib.error.HTTPError now has a proper repr() representation. - Patch by Berker Peksag. - -- asyncio: New event loop APIs: set_task_factory() and get_task_factory(). - -- asyncio: async() function is deprecated in favour of ensure_future(). - -- Issue #24178: asyncio.Lock, Condition, Semaphore, and BoundedSemaphore - support new 'async with' syntax. Contributed by Yury Selivanov. - -- Issue #24179: Support 'async for' for asyncio.StreamReader. - Contributed by Yury Selivanov. - -- Issue #24184: Add AsyncIterator and AsyncIterable ABCs to - collections.abc. Contributed by Yury Selivanov. - -- Issue #22547: Implement informative __repr__ for inspect.BoundArguments. - Contributed by Yury Selivanov. - -- Issue #24190: Implement inspect.BoundArgument.apply_defaults() method. - Contributed by Yury Selivanov. - -- Issue #20691: Add 'follow_wrapped' argument to - inspect.Signature.from_callable() and inspect.signature(). - Contributed by Yury Selivanov. - -- Issue #24248: Deprecate inspect.Signature.from_function() and - inspect.Signature.from_builtin(). - -- Issue #23898: Fix inspect.classify_class_attrs() to support attributes - with overloaded __eq__ and __bool__. Patch by Mike Bayer. - -- Issue #24298: Fix inspect.signature() to correctly unwrap wrappers - around bound methods. - -IDLE ----- - -- Issue #23184: remove unused names and imports in idlelib. - Initial patch by Al Sweigart. - -Tests ------ - -- Issue #21520: test_zipfile no longer fails if the word 'bad' appears - anywhere in the name of the current directory. - -- Issue #9517: Move script_helper into the support package. - Patch by Christie Wilson. - -Documentation -------------- - -- Issue #22155: Add File Handlers subsection with createfilehandler to tkinter - doc. Remove obsolete example from FAQ. Patch by Martin Panter. - -- Issue #24029: Document the name binding behavior for submodule imports. - -- Issue #24077: Fix typo in man page for -I command option: -s, not -S - -Tools/Demos ------------ - -- Issue #24000: Improved Argument Clinic's mapping of converters to legacy - "format units". Updated the documentation to match. - -- Issue #24001: Argument Clinic converters now use accept={type} - instead of types={'type'} to specify the types the converter accepts. - -- Issue #23330: h2py now supports arbitrary filenames in #include. - -- Issue #24031: make patchcheck now supports git checkouts, too. - - -What's New in Python 3.5.0 alpha 4? -=================================== - -Release date: 2015-04-19 - -Core and Builtins ------------------ - -- Issue #22980: Under Linux, GNU/KFreeBSD and the Hurd, C extensions now include - the architecture triplet in the extension name, to make it easy to test builds - for different ABIs in the same working tree. Under OS X, the extension name - now includes PEP 3149-style information. - -- Issue #22631: Added Linux-specific socket constant CAN_RAW_FD_FRAMES. - Patch courtesy of Joe Jevnik. - -- Issue #23731: Implement PEP 488: removal of .pyo files. - -- Issue #23726: Don't enable GC for user subclasses of non-GC types that - don't add any new fields. Patch by Eugene Toder. - -- Issue #23309: Avoid a deadlock at shutdown if a daemon thread is aborted - while it is holding a lock to a buffered I/O object, and the main thread - tries to use the same I/O object (typically stdout or stderr). A fatal - error is emitted instead. - -- Issue #22977: Fixed formatting Windows error messages on Wine. - Patch by Martin Panter. - -- Issue #23466: %c, %o, %x, and %X in bytes formatting now raise TypeError on - non-integer input. - -- Issue #24044: Fix possible null pointer dereference in list.sort in out of - memory conditions. - -- Issue #21354: PyCFunction_New function is exposed by python DLL again. - -Library -------- - -- Issue #23840: tokenize.open() now closes the temporary binary file on error - to fix a resource warning. - -- Issue #16914: new debuglevel 2 in smtplib adds timestamps to debug output. - -- Issue #7159: urllib.request now supports sending auth credentials - automatically after the first 401. This enhancement is a superset of the - enhancement from issue #19494 and supersedes that change. - -- Issue #23703: Fix a regression in urljoin() introduced in 901e4e52b20a. - Patch by Demian Brecht. - -- Issue #4254: Adds _curses.update_lines_cols(). Patch by Arnon Yaari - -- Issue #19933: Provide default argument for ndigits in round. Patch by - Vajrasky Kok. - -- Issue #23193: Add a numeric_owner parameter to - tarfile.TarFile.extract and tarfile.TarFile.extractall. Patch by - Michael Vogt and Eric Smith. - -- Issue #23342: Add a subprocess.run() function than returns a CalledProcess - instance for a more consistent API than the existing call* functions. - -- Issue #21217: inspect.getsourcelines() now tries to compute the start and end - lines from the code object, fixing an issue when a lambda function is used as - decorator argument. Patch by Thomas Ballinger and Allison Kaptur. - -- Issue #24521: Fix possible integer overflows in the pickle module. - -- Issue #22931: Allow '[' and ']' in cookie values. - -- The keywords attribute of functools.partial is now always a dictionary. - -- Issue #23811: Add missing newline to the PyCompileError error message. - Patch by Alex Shkop. - -- Issue #21116: Avoid blowing memory when allocating a multiprocessing shared - array that's larger than 50% of the available RAM. Patch by Médéric Boquien. - -- Issue #22982: Improve BOM handling when seeking to multiple positions of - a writable text file. - -- Issue #23464: Removed deprecated asyncio JoinableQueue. - -- Issue #23529: Limit the size of decompressed data when reading from - GzipFile, BZ2File or LZMAFile. This defeats denial of service attacks - using compressed bombs (i.e. compressed payloads which decompress to a huge - size). Patch by Martin Panter and Nikolaus Rath. - -- Issue #21859: Added Python implementation of io.FileIO. - -- Issue #23865: close() methods in multiple modules now are idempotent and more - robust at shutdown. If they need to release multiple resources, all are - released even if errors occur. - -- Issue #23400: Raise same exception on both Python 2 and 3 if sem_open is not - available. Patch by Davin Potts. - -- Issue #10838: The subprocess now module includes SubprocessError and - TimeoutError in its list of exported names for the users wild enough - to use ``from subprocess import *``. - -- Issue #23411: Added DefragResult, ParseResult, SplitResult, DefragResultBytes, - ParseResultBytes, and SplitResultBytes to urllib.parse.__all__. - Patch by Martin Panter. - -- Issue #23881: urllib.request.ftpwrapper constructor now closes the socket if - the FTP connection failed to fix a ResourceWarning. - -- Issue #23853: :meth:`socket.socket.sendall` does no more reset the socket - timeout each time data is sent successfully. The socket timeout is now the - maximum total duration to send all data. - -- Issue #22721: An order of multiline pprint output of set or dict containing - orderable and non-orderable elements no longer depends on iteration order of - set or dict. - -- Issue #15133: _tkinter.tkapp.getboolean() now supports Tcl_Obj and always - returns bool. tkinter.BooleanVar now validates input values (accepted bool, - int, str, and Tcl_Obj). tkinter.BooleanVar.get() now always returns bool. - -- Issue #10590: xml.sax.parseString() now supports string argument. - -- Issue #23338: Fixed formatting ctypes error messages on Cygwin. - Patch by Makoto Kato. - -- Issue #15582: inspect.getdoc() now follows inheritance chains. - -- Issue #2175: SAX parsers now support a character stream of InputSource object. - -- Issue #16840: Tkinter now supports 64-bit integers added in Tcl 8.4 and - arbitrary precision integers added in Tcl 8.5. - -- Issue #23834: Fix socket.sendto(), use the C Py_ssize_t type to store the - result of sendto() instead of the C int type. - -- Issue #23618: :meth:`socket.socket.connect` now waits until the connection - completes instead of raising :exc:`InterruptedError` if the connection is - interrupted by signals, signal handlers don't raise an exception and the - socket is blocking or has a timeout. :meth:`socket.socket.connect` still - raise :exc:`InterruptedError` for non-blocking sockets. - -- Issue #21526: Tkinter now supports new boolean type in Tcl 8.5. - -- Issue #23836: Fix the faulthandler module to handle reentrant calls to - its signal handlers. - -- Issue #23838: linecache now clears the cache and returns an empty result on - MemoryError. - -- Issue #10395: Added os.path.commonpath(). Implemented in posixpath and ntpath. - Based on patch by Rafik Draoui. - -- Issue #23611: Serializing more "lookupable" objects (such as unbound methods - or nested classes) now are supported with pickle protocols < 4. - -- Issue #13583: sqlite3.Row now supports slice indexing. - -- Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no - longer applied to modules already mapped with full name mapping. - -- Issue #23485: select.select() is now retried automatically with the - recomputed timeout when interrupted by a signal, except if the signal handler - raises an exception. This change is part of the PEP 475. - -- Issue #23752: When built from an existing file descriptor, io.FileIO() now - only calls fstat() once. Before fstat() was called twice, which was not - necessary. - -- Issue #23704: collections.deque() objects now support __add__, __mul__, and - __imul__(). - -- Issue #23171: csv.Writer.writerow() now supports arbitrary iterables. - -- Issue #23745: The new email header parser now handles duplicate MIME - parameter names without error, similar to how get_param behaves. - -- Issue #22117: Fix os.utime(), it now rounds the timestamp towards minus - infinity (-inf) instead of rounding towards zero. - -- Issue #23310: Fix MagicMock's initializer to work with __methods__, just - like configure_mock(). Patch by Kasia Jachim. - -Build ------ - -- Issue #23817: FreeBSD now uses "1.0" in the SOVERSION as other operating - systems, instead of just "1". - -- Issue #23501: Argument Clinic now generates code into separate files by default. - -Tests ------ - -- Issue #23799: Added test.support.start_threads() for running and - cleaning up multiple threads. - -- Issue #22390: test.regrtest now emits a warning if temporary files or - directories are left after running a test. - -Tools/Demos ------------ - -- Issue #18128: pygettext now uses standard +NNNN format in the - POT-Creation-Date header. - -- Issue #23935: Argument Clinic's understanding of format units - accepting bytes, bytearrays, and buffers is now consistent with - both the documentation and the implementation. - -- Issue #23944: Argument Clinic now wraps long impl prototypes at column 78. - -- Issue #20586: Argument Clinic now ensures that functions without docstrings - have signatures. - -- Issue #23492: Argument Clinic now generates argument parsing code with - PyArg_Parse instead of PyArg_ParseTuple if possible. - -- Issue #23500: Argument Clinic is now smarter about generating the "#ifndef" - (empty) definition of the methoddef macro: it's only generated once, even - if Argument Clinic processes the same symbol multiple times, and it's emitted - at the end of all processing rather than immediately after the first use. - -C API ------ - -- Issue #23998: PyImport_ReInitLock() now checks for lock allocation error - - -What's New in Python 3.5.0 alpha 3? -=================================== - -Release date: 2015-03-28 - -Core and Builtins ------------------ - -- Issue #23573: Increased performance of string search operations (str.find, - str.index, str.count, the in operator, str.split, str.partition) with - arguments of different kinds (UCS1, UCS2, UCS4). - -- Issue #23753: Python doesn't support anymore platforms without stat() or - fstat(), these functions are always required. - -- Issue #23681: The -b option now affects comparisons of bytes with int. - -- Issue #23632: Memoryviews now allow tuple indexing (including for - multi-dimensional memoryviews). - -- Issue #23192: Fixed generator lambdas. Patch by Bruno Cauet. - -- Issue #23629: Fix the default __sizeof__ implementation for variable-sized - objects. - -Library -------- - -- Issue #14260: The groupindex attribute of regular expression pattern object - now is non-modifiable mapping. - -- Issue #23792: Ignore KeyboardInterrupt when the pydoc pager is active. - This mimics the behavior of the standard unix pagers, and prevents - pipepager from shutting down while the pager itself is still running. - -- Issue #23775: pprint() of OrderedDict now outputs the same representation - as repr(). - -- Issue #23765: Removed IsBadStringPtr calls in ctypes - -- Issue #22364: Improved some re error messages using regex for hints. - -- Issue #23742: ntpath.expandvars() no longer loses unbalanced single quotes. - -- Issue #21717: The zipfile.ZipFile.open function now supports 'x' (exclusive - creation) mode. - -- Issue #21802: The reader in BufferedRWPair now is closed even when closing - writer failed in BufferedRWPair.close(). - -- Issue #23622: Unknown escapes in regular expressions that consist of ``'\'`` - and ASCII letter now raise a deprecation warning and will be forbidden in - Python 3.6. - -- Issue #23671: string.Template now allows specifying the "self" parameter as - a keyword argument. string.Formatter now allows specifying the "self" and - the "format_string" parameters as keyword arguments. - -- Issue #23502: The pprint module now supports mapping proxies. - -- Issue #17530: pprint now wraps long bytes objects and bytearrays. - -- Issue #22687: Fixed some corner cases in breaking words in tetxtwrap. - Got rid of quadratic complexity in breaking long words. - -- Issue #4727: The copy module now uses pickle protocol 4 (PEP 3154) and - supports copying of instances of classes whose __new__ method takes - keyword-only arguments. - -- Issue #23491: Added a zipapp module to support creating executable zip - file archives of Python code. Registered ".pyz" and ".pyzw" extensions - on Windows for these archives (PEP 441). - -- Issue #23657: Avoid explicit checks for str in zipapp, adding support - for pathlib.Path objects as arguments. - -- Issue #23688: Added support of arbitrary bytes-like objects and avoided - unnecessary copying of memoryview in gzip.GzipFile.write(). - Original patch by Wolfgang Maier. - -- Issue #23252: Added support for writing ZIP files to unseekable streams. - -- Issue #23647: Increase impalib's MAXLINE to accommodate modern mailbox sizes. - -- Issue #23539: If body is None, http.client.HTTPConnection.request now sets - Content-Length to 0 for PUT, POST, and PATCH headers to avoid 411 errors from - some web servers. - -- Issue #22351: The nntplib.NNTP constructor no longer leaves the connection - and socket open until the garbage collector cleans them up. Patch by - Martin Panter. - -- Issue #23704: collections.deque() objects now support methods for index(), - insert(), and copy(). This allows deques to be registered as a - MutableSequence and it improves their substitutability for lists. - -- Issue #23715: :func:`signal.sigwaitinfo` and :func:`signal.sigtimedwait` are - now retried when interrupted by a signal not in the *sigset* parameter, if - the signal handler does not raise an exception. signal.sigtimedwait() - recomputes the timeout with a monotonic clock when it is retried. - -- Issue #23001: Few functions in modules mmap, ossaudiodev, socket, ssl, and - codecs, that accepted only read-only bytes-like object now accept writable - bytes-like object too. - -- Issue #23646: If time.sleep() is interrupted by a signal, the sleep is now - retried with the recomputed delay, except if the signal handler raises an - exception (PEP 475). - -- Issue #23136: _strptime now uniformly handles all days in week 0, including - Dec 30 of previous year. Based on patch by Jim Carroll. - -- Issue #23700: Iterator of NamedTemporaryFile now keeps a reference to - NamedTemporaryFile instance. Patch by Bohuslav Kabrda. - -- Issue #22903: The fake test case created by unittest.loader when it fails - importing a test module is now picklable. - -- Issue #22181: On Linux, os.urandom() now uses the new getrandom() syscall if - available, syscall introduced in the Linux kernel 3.17. It is more reliable - and more secure, because it avoids the need of a file descriptor and waits - until the kernel has enough entropy. - -- Issue #2211: Updated the implementation of the http.cookies.Morsel class. - Setting attributes key, value and coded_value directly now is deprecated. - update() and setdefault() now transform and check keys. Comparing for - equality now takes into account attributes key, value and coded_value. - copy() now returns a Morsel, not a dict. repr() now contains all attributes. - Optimized checking keys and quoting values. Added new tests. - Original patch by Demian Brecht. - -- Issue #18983: Allow selection of output units in timeit. - Patch by Julian Gindi. - -- Issue #23631: Fix traceback.format_list when a traceback has been mutated. - -- Issue #23568: Add rdivmod support to MagicMock() objects. - Patch by Håkan Lövdahl. - -- Issue #2052: Add charset parameter to HtmlDiff.make_file(). - -- Issue #23668: Support os.truncate and os.ftruncate on Windows. - -- Issue #23138: Fixed parsing cookies with absent keys or values in cookiejar. - Patch by Demian Brecht. - -- Issue #23051: multiprocessing.Pool methods imap() and imap_unordered() now - handle exceptions raised by an iterator. Patch by Alon Diamant and Davin - Potts. - -- Issue #23581: Add matmul support to MagicMock. Patch by Håkan Lövdahl. - -- Issue #23566: enable(), register(), dump_traceback() and - dump_traceback_later() functions of faulthandler now accept file - descriptors. Patch by Wei Wu. - -- Issue #22928: Disabled HTTP header injections in http.client. - Original patch by Demian Brecht. - -- Issue #23615: Modules bz2, tarfile and tokenize now can be reloaded with - imp.reload(). Patch by Thomas Kluyver. - -- Issue #23605: os.walk() now calls os.scandir() instead of os.listdir(). - The usage of os.scandir() reduces the number of calls to os.stat(). - Initial patch written by Ben Hoyt. - -Build ------ - -- Issue #23585: make patchcheck will ensure the interpreter is built. - -Tests ------ - -- Issue #23583: Added tests for standard IO streams in IDLE. - -- Issue #22289: Prevent test_urllib2net failures due to ftp connection timeout. - -Tools/Demos ------------ - -- Issue #22826: The result of open() in Tools/freeze/bkfile.py is now better - compatible with regular files (in particular it now supports the context - management protocol). - - -What's New in Python 3.5 alpha 2? -================================= - -Release date: 2015-03-09 - -Core and Builtins ------------------ - -- Issue #23571: PyObject_Call() and PyCFunction_Call() now raise a SystemError - if a function returns a result and raises an exception. The SystemError is - chained to the previous exception. - -Library -------- - -- Issue #22524: New os.scandir() function, part of the PEP 471: "os.scandir() - function -- a better and faster directory iterator". Patch written by Ben - Hoyt. - -- Issue #23103: Reduced the memory consumption of IPv4Address and IPv6Address. - -- Issue #21793: BaseHTTPRequestHandler again logs response code as numeric, - not as stringified enum. Patch by Demian Brecht. - -- Issue #23476: In the ssl module, enable OpenSSL's X509_V_FLAG_TRUSTED_FIRST - flag on certificate stores when it is available. - -- Issue #23576: Avoid stalling in SSL reads when EOF has been reached in the - SSL layer but the underlying connection hasn't been closed. - -- Issue #23504: Added an __all__ to the types module. - -- Issue #23563: Optimized utility functions in urllib.parse. - -- Issue #7830: Flatten nested functools.partial. - -- Issue #20204: Added the __module__ attribute to _tkinter classes. - -- Issue #19980: Improved help() for non-recognized strings. help('') now - shows the help on str. help('help') now shows the help on help(). - Original patch by Mark Lawrence. - -- Issue #23521: Corrected pure python implementation of timedelta division. - - * Eliminated OverflowError from timedelta * float for some floats; - * Corrected rounding in timedlta true division. - -- Issue #21619: Popen objects no longer leave a zombie after exit in the with - statement if the pipe was broken. Patch by Martin Panter. - -- Issue #22936: Make it possible to show local variables in tracebacks for - both the traceback module and unittest. - -- Issue #15955: Add an option to limit the output size in bz2.decompress(). - Patch by Nikolaus Rath. - -- Issue #6639: Module-level turtle functions no longer raise TclError after - closing the window. - -- Issues #814253, #9179: Group references and conditional group references now - work in lookbehind assertions in regular expressions. - -- Issue #23215: Multibyte codecs with custom error handlers that ignores errors - consumed too much memory and raised SystemError or MemoryError. - Original patch by Aleksi Torhamo. - -- Issue #5700: io.FileIO() called flush() after closing the file. - flush() was not called in close() if closefd=False. - -- Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding - differs from file system encoding (e.g. on Mac OS). - -- Issue #23481: Remove RC4 from the SSL module's default cipher list. - -- Issue #21548: Fix pydoc.synopsis() and pydoc.apropos() on modules with empty - docstrings. - -- Issue #22885: Fixed arbitrary code execution vulnerability in the dbm.dumb - module. Original patch by Claudiu Popa. - -- Issue #23239: ssl.match_hostname() now supports matching of IP addresses. - -- Issue #23146: Fix mishandling of absolute Windows paths with forward - slashes in pathlib. - -- Issue #23096: Pickle representation of floats with protocol 0 now is the same - for both Python and C implementations. - -- Issue #19105: pprint now more efficiently uses free space at the right. - -- Issue #14910: Add allow_abbrev parameter to argparse.ArgumentParser. Patch by - Jonathan Paugh, Steven Bethard, paul j3 and Daniel Eriksson. - -- Issue #21717: tarfile.open() now supports 'x' (exclusive creation) mode. - -- Issue #23344: marshal.dumps() is now 20-25% faster on average. - -- Issue #20416: marshal.dumps() with protocols 3 and 4 is now 40-50% faster on - average. - -- Issue #23421: Fixed compression in tarfile CLI. Patch by wdv4758h. - -- Issue #23367: Fix possible overflows in the unicodedata module. - -- Issue #23361: Fix possible overflow in Windows subprocess creation code. - -- logging.handlers.QueueListener now takes a respect_handler_level keyword - argument which, if set to True, will pass messages to handlers taking handler - levels into account. - -- Issue #19705: turtledemo now has a visual sorting algorithm demo. Original - patch from Jason Yeo. - -- Issue #23801: Fix issue where cgi.FieldStorage did not always ignore the - entire preamble to a multipart body. - -Build ------ - -- Issue #23445: pydebug builds now use "gcc -Og" where possible, to make - the resulting executable faster. - -- Issue #23686: Update OS X 10.5 installer build to use OpenSSL 1.0.2a. - -C API ------ - -- Issue #20204: Deprecation warning is now raised for builtin types without the - __module__ attribute. - -Windows -------- - -- Issue #23465: Implement PEP 486 - Make the Python Launcher aware of virtual - environments. Patch by Paul Moore. - -- Issue #23437: Make user scripts directory versioned on Windows. Patch by Paul - Moore. - - -What's New in Python 3.5 alpha 1? -================================= - -Release date: 2015-02-08 - -Core and Builtins ------------------ - -- Issue #23285: PEP 475 - EINTR handling. - -- Issue #22735: Fix many edge cases (including crashes) involving custom mro() - implementations. - -- Issue #22896: Avoid using PyObject_AsCharBuffer(), PyObject_AsReadBuffer() - and PyObject_AsWriteBuffer(). - -- Issue #21295: Revert some changes (issue #16795) to AST line numbers and - column offsets that constituted a regression. - -- Issue #22986: Allow changing an object's __class__ between a dynamic type and - static type in some cases. - -- Issue #15859: PyUnicode_EncodeFSDefault(), PyUnicode_EncodeMBCS() and - PyUnicode_EncodeCodePage() now raise an exception if the object is not a - Unicode object. For PyUnicode_EncodeFSDefault(), it was already the case on - platforms other than Windows. Patch written by Campbell Barton. - -- Issue #21408: The default __ne__() now returns NotImplemented if __eq__() - returned NotImplemented. Original patch by Martin Panter. - -- Issue #23321: Fixed a crash in str.decode() when error handler returned - replacment string longer than mailformed input data. - -- Issue #22286: The "backslashreplace" error handlers now works with - decoding and translating. - -- Issue #23253: Delay-load ShellExecute[AW] in os.startfile for reduced - startup overhead on Windows. - -- Issue #22038: pyatomic.h now uses stdatomic.h or GCC built-in functions for - atomic memory access if available. Patch written by Vitor de Lima and Gustavo - Temple. - -- Issue #20284: %-interpolation (aka printf) formatting added for bytes and - bytearray. - -- Issue #23048: Fix jumping out of an infinite while loop in the pdb. - -- Issue #20335: bytes constructor now raises TypeError when encoding or errors - is specified with non-string argument. Based on patch by Renaud Blanch. - -- Issue #22834: If the current working directory ends up being set to a - non-existent directory then import will no longer raise FileNotFoundError. - -- Issue #22869: Move the interpreter startup & shutdown code to a new - dedicated pylifecycle.c module - -- Issue #22847: Improve method cache efficiency. - -- Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff - bytes on a 32-bit platform. - -- Issue #22653: Fix an assertion failure in debug mode when doing a reentrant - dict insertion in debug mode. - -- Issue #22643: Fix integer overflow in Unicode case operations (upper, lower, - title, swapcase, casefold). - -- Issue #17636: Circular imports involving relative imports are now - supported. - -- Issue #22604: Fix assertion error in debug mode when dividing a complex - number by (nan+0j). - -- Issue #21052: Do not raise ImportWarning when sys.path_hooks or sys.meta_path - are set to None. - -- Issue #16518: Use 'bytes-like object required' in error messages that - previously used the far more cryptic "'x' does not support the buffer - protocol. - -- Issue #22470: Fixed integer overflow issues in "backslashreplace", - "xmlcharrefreplace", and "surrogatepass" error handlers. - -- Issue #22540: speed up `PyObject_IsInstance` and `PyObject_IsSubclass` in the - common case that the second argument has metaclass `type`. - -- Issue #18711: Add a new `PyErr_FormatV` function, similar to `PyErr_Format` - but accepting a `va_list` argument. - -- Issue #22520: Fix overflow checking when generating the repr of a unicode - object. - -- Issue #22519: Fix overflow checking in PyBytes_Repr. - -- Issue #22518: Fix integer overflow issues in latin-1 encoding. - -- Issue #16324: _charset parameter of MIMEText now also accepts - email.charset.Charset instances. Initial patch by Claude Paroz. - -- Issue #1764286: Fix inspect.getsource() to support decorated functions. - Patch by Claudiu Popa. - -- Issue #18554: os.__all__ includes posix functions. - -- Issue #21391: Use os.path.abspath in the shutil module. - -- Issue #11471: avoid generating a JUMP_FORWARD instruction at the end of - an if-block if there is no else-clause. Original patch by Eugene Toder. - -- Issue #22215: Now ValueError is raised instead of TypeError when str or bytes - argument contains not permitted null character or byte. - -- Issue #22258: Fix the internal function set_inheritable() on Illumos. - This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails - with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() - now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). - -- Issue #21389: Displaying the __qualname__ of the underlying function in the - repr of a bound method. - -- Issue #22206: Using pthread, PyThread_create_key() now sets errno to ENOMEM - and returns -1 (error) on integer overflow. - -- Issue #20184: Argument Clinic based signature introspection added for - 30 of the builtin functions. - -- Issue #22116: C functions and methods (of the 'builtin_function_or_method' - type) can now be weakref'ed. Patch by Wei Wu. - -- Issue #22077: Improve index error messages for bytearrays, bytes, lists, - and tuples by adding 'or slices'. Added ', not ' for bytearrays. - Original patch by Claudiu Popa. - -- Issue #20179: Apply Argument Clinic to bytes and bytearray. - Patch by Tal Einat. - -- Issue #22082: Clear interned strings in slotdefs. - -- Upgrade Unicode database to Unicode 7.0.0. - -- Issue #21897: Fix a crash with the f_locals attribute with closure - variables when frame.clear() has been called. - -- Issue #21205: Add a new ``__qualname__`` attribute to generator, the - qualified name, and use it in the representation of a generator - (``repr(gen)``). The default name of the generator (``__name__`` attribute) - is now get from the function instead of the code. Use ``gen.gi_code.co_name`` - to get the name of the code. - -- Issue #21669: With the aid of heuristics in SyntaxError.__init__, the - parser now attempts to generate more meaningful (or at least more search - engine friendly) error messages when "exec" and "print" are used as - statements. - -- Issue #21642: In the conditional if-else expression, allow an integer written - with no space between itself and the ``else`` keyword (e.g. ``True if 42else - False``) to be valid syntax. - -- Issue #21523: Fix over-pessimistic computation of the stack effect of - some opcodes in the compiler. This also fixes a quadratic compilation - time issue noticeable when compiling code with a large number of "and" - and "or" operators. - -- Issue #21418: Fix a crash in the builtin function super() when called without - argument and without current frame (ex: embedded Python). - -- Issue #21425: Fix flushing of standard streams in the interactive - interpreter. - -- Issue #21435: In rare cases, when running finalizers on objects in cyclic - trash a bad pointer dereference could occur due to a subtle flaw in - internal iteration logic. - -- Issue #21377: PyBytes_Concat() now tries to concatenate in-place when the - first argument has a reference count of 1. Patch by Nikolaus Rath. - -- Issue #20355: -W command line options now have higher priority than the - PYTHONWARNINGS environment variable. Patch by Arfrever. - -- Issue #21274: Define PATH_MAX for GNU/Hurd in Python/pythonrun.c. - -- Issue #20904: Support setting FPU precision on m68k. - -- Issue #21209: Fix sending tuples to custom generator objects with the yield - from syntax. - -- Issue #21193: pow(a, b, c) now raises ValueError rather than TypeError when b - is negative. Patch by Josh Rosenberg. - -- PEP 465 and Issue #21176: Add the '@' operator for matrix multiplication. - -- Issue #21134: Fix segfault when str is called on an uninitialized - UnicodeEncodeError, UnicodeDecodeError, or UnicodeTranslateError object. - -- Issue #19537: Fix PyUnicode_DATA() alignment under m68k. Patch by - Andreas Schwab. - -- Issue #20929: Add a type cast to avoid shifting a negative number. - -- Issue #20731: Properly position in source code files even if they - are opened in text mode. Patch by Serhiy Storchaka. - -- Issue #20637: Key-sharing now also works for instance dictionaries of - subclasses. Patch by Peter Ingebretson. - -- Issue #8297: Attributes missing from modules now include the module name - in the error text. Original patch by ysj.ray. - -- Issue #19995: %c, %o, %x, and %X now raise TypeError on non-integer input. - -- Issue #19655: The ASDL parser - used by the build process to generate code for - managing the Python AST in C - was rewritten. The new parser is self contained - and does not require to carry long the spark.py parser-generator library; - spark.py was removed from the source base. - -- Issue #12546: Allow ``\x00`` to be used as a fill character when using str, int, - float, and complex __format__ methods. - -- Issue #20480: Add ipaddress.reverse_pointer. Patch by Leon Weber. - -- Issue #13598: Modify string.Formatter to support auto-numbering of - replacement fields. It now matches the behavior of str.format() in - this regard. Patches by Phil Elson and Ramchandra Apte. - -- Issue #8931: Make alternate formatting ('#') for type 'c' raise an - exception. In versions prior to 3.5, '#' with 'c' had no effect. Now - specifying it is an error. Patch by Torsten Landschoff. - -- Issue #23165: Perform overflow checks before allocating memory in the - _Py_char2wchar function. - -Library -------- - -- Issue #23399: pyvenv creates relative symlinks where possible. - -- Issue #20289: cgi.FieldStorage() now supports the context management - protocol. - -- Issue #13128: Print response headers for CONNECT requests when debuglevel - > 0. Patch by Demian Brecht. - -- Issue #15381: Optimized io.BytesIO to make less allocations and copyings. - -- Issue #22818: Splitting on a pattern that could match an empty string now - raises a warning. Patterns that can only match empty strings are now - rejected. - -- Issue #23099: Closing io.BytesIO with exported buffer is rejected now to - prevent corrupting exported buffer. - -- Issue #23326: Removed __ne__ implementations. Since fixing default __ne__ - implementation in issue #21408 they are redundant. - -- Issue #23363: Fix possible overflow in itertools.permutations. - -- Issue #23364: Fix possible overflow in itertools.product. - -- Issue #23366: Fixed possible integer overflow in itertools.combinations. - -- Issue #23369: Fixed possible integer overflow in - _json.encode_basestring_ascii. - -- Issue #23353: Fix the exception handling of generators in - PyEval_EvalFrameEx(). At entry, save or swap the exception state even if - PyEval_EvalFrameEx() is called with throwflag=0. At exit, the exception state - is now always restored or swapped, not only if why is WHY_YIELD or - WHY_RETURN. Patch co-written with Antoine Pitrou. - -- Issue #14099: Restored support of writing ZIP files to tellable but - non-seekable streams. - -- Issue #14099: Writing to ZipFile and reading multiple ZipExtFiles is - threadsafe now. - -- Issue #19361: JSON decoder now raises JSONDecodeError instead of ValueError. - -- Issue #18518: timeit now rejects statements which can't be compiled outside - a function or a loop (e.g. "return" or "break"). - -- Issue #23094: Fixed readline with frames in Python implementation of pickle. - -- Issue #23268: Fixed bugs in the comparison of ipaddress classes. - -- Issue #21408: Removed incorrect implementations of __ne__() which didn't - returned NotImplemented if __eq__() returned NotImplemented. The default - __ne__() now works correctly. - -- Issue #19996: :class:`email.feedparser.FeedParser` now handles (malformed) - headers with no key rather than assuming the body has started. - -- Issue #20188: Support Application-Layer Protocol Negotiation (ALPN) in the ssl - module. - -- Issue #23133: Pickling of ipaddress objects now produces more compact and - portable representation. - -- Issue #23248: Update ssl error codes from latest OpenSSL git master. - -- Issue #23266: Much faster implementation of ipaddress.collapse_addresses() - when there are many non-consecutive addresses. - -- Issue #23098: 64-bit dev_t is now supported in the os module. - -- Issue #21817: When an exception is raised in a task submitted to a - ProcessPoolExecutor, the remote traceback is now displayed in the - parent process. Patch by Claudiu Popa. - -- Issue #15955: Add an option to limit output size when decompressing LZMA - data. Patch by Nikolaus Rath and Martin Panter. - -- Issue #23250: In the http.cookies module, capitalize "HttpOnly" and "Secure" - as they are written in the standard. - -- Issue #23063: In the disutils' check command, fix parsing of reST with code or - code-block directives. - -- Issue #23209, #23225: selectors.BaseSelector.get_key() now raises a - RuntimeError if the selector is closed. And selectors.BaseSelector.close() - now clears its internal reference to the selector mapping to break a - reference cycle. Initial patch written by Martin Richard. - -- Issue #17911: Provide a way to seed the linecache for a PEP-302 module - without actually loading the code. - -- Issue #17911: Provide a new object API for traceback, including the ability - to not lookup lines at all until the traceback is actually rendered, without - any trace of the original objects being kept alive. - -- Issue #19777: Provide a home() classmethod on Path objects. Contributed - by Victor Salgado and Mayank Tripathi. - -- Issue #23206: Make ``json.dumps(..., ensure_ascii=False)`` as fast as the - default case of ``ensure_ascii=True``. Patch by Naoki Inada. - -- Issue #23185: Add math.inf and math.nan constants. - -- Issue #23186: Add ssl.SSLObject.shared_ciphers() and - ssl.SSLSocket.shared_ciphers() to fetch the client's list ciphers sent at - handshake. - -- Issue #23143: Remove compatibility with OpenSSLs older than 0.9.8. - -- Issue #23132: Improve performance and introspection support of comparison - methods created by functool.total_ordering. - -- Issue #19776: Add an expanduser() method on Path objects. - -- Issue #23112: Fix SimpleHTTPServer to correctly carry the query string and - fragment when it redirects to add a trailing slash. - -- Issue #21793: Added http.HTTPStatus enums (i.e. HTTPStatus.OK, - HTTPStatus.NOT_FOUND). Patch by Demian Brecht. - -- Issue #23093: In the io, module allow more operations to work on detached - streams. - -- Issue #23111: In the ftplib, make ssl.PROTOCOL_SSLv23 the default protocol - version. - -- Issue #22585: On OpenBSD 5.6 and newer, os.urandom() now calls getentropy(), - instead of reading /dev/urandom, to get pseudo-random bytes. - -- Issue #19104: pprint now produces evaluable output for wrapped strings. - -- Issue #23071: Added missing names to codecs.__all__. Patch by Martin Panter. - -- Issue #22783: Pickling now uses the NEWOBJ opcode instead of the NEWOBJ_EX - opcode if possible. - -- Issue #15513: Added a __sizeof__ implementation for pickle classes. - -- Issue #19858: pickletools.optimize() now aware of the MEMOIZE opcode, can - produce more compact result and no longer produces invalid output if input - data contains MEMOIZE opcodes together with PUT or BINPUT opcodes. - -- Issue #22095: Fixed HTTPConnection.set_tunnel with default port. The port - value in the host header was set to "None". Patch by Demian Brecht. - -- Issue #23016: A warning no longer produces an AttributeError when the program - is run with pythonw.exe. - -- Issue #21775: shutil.copytree(): fix crash when copying to VFAT. An exception - handler assumed that OSError objects always have a 'winerror' attribute. - That is not the case, so the exception handler itself raised AttributeError - when run on Linux (and, presumably, any other non-Windows OS). - Patch by Greg Ward. - -- Issue #1218234: Fix inspect.getsource() to load updated source of - reloaded module. Initial patch by Berker Peksag. - -- Issue #21740: Support wrapped callables in doctest. Patch by Claudiu Popa. - -- Issue #23009: Make sure selectors.EpollSelecrtor.select() works when no - FD is registered. - -- Issue #22959: In the constructor of http.client.HTTPSConnection, prefer the - context's check_hostname attribute over the *check_hostname* parameter. - -- Issue #22696: Add function :func:`sys.is_finalizing` to know about - interpreter shutdown. - -- Issue #16043: Add a default limit for the amount of data xmlrpclib.gzip_decode - will return. This resolves CVE-2013-1753. - -- Issue #14099: ZipFile.open() no longer reopen the underlying file. Objects - returned by ZipFile.open() can now operate independently of the ZipFile even - if the ZipFile was created by passing in a file-like object as the first - argument to the constructor. - -- Issue #22966: Fix __pycache__ pyc file name clobber when pyc_compile is - asked to compile a source file containing multiple dots in the source file - name. - -- Issue #21971: Update turtledemo doc and add module to the index. - -- Issue #21032: Fixed socket leak if HTTPConnection.getresponse() fails. - Original patch by Martin Panter. - -- Issue #22407: Deprecated the use of re.LOCALE flag with str patterns or - re.ASCII. It was newer worked. - -- Issue #22902: The "ip" command is now used on Linux to determine MAC address - in uuid.getnode(). Pach by Bruno Cauet. - -- Issue #22960: Add a context argument to xmlrpclib.ServerProxy constructor. - -- Issue #22389: Add contextlib.redirect_stderr(). - -- Issue #21356: Make ssl.RAND_egd() optional to support LibreSSL. The - availability of the function is checked during the compilation. Patch written - by Bernard Spil. - -- Issue #22915: SAX parser now supports files opened with file descriptor or - bytes path. - -- Issue #22609: Constructors and update methods of mapping classes in the - collections module now accept the self keyword argument. - -- Issue #22940: Add readline.append_history_file. - -- Issue #19676: Added the "namereplace" error handler. - -- Issue #22788: Add *context* parameter to logging.handlers.HTTPHandler. - -- Issue #22921: Allow SSLContext to take the *hostname* parameter even if - OpenSSL doesn't support SNI. - -- Issue #22894: TestCase.subTest() would cause the test suite to be stopped - when in failfast mode, even in the absence of failures. - -- Issue #22796: HTTP cookie parsing is now stricter, in order to protect - against potential injection attacks. - -- Issue #22370: Windows detection in pathlib is now more robust. - -- Issue #22841: Reject coroutines in asyncio add_signal_handler(). - Patch by Ludovic.Gasc. - -- Issue #19494: Added urllib.request.HTTPBasicPriorAuthHandler. Patch by - Matej Cepl. - -- Issue #22578: Added attributes to the re.error class. - -- Issue #22849: Fix possible double free in the io.TextIOWrapper constructor. - -- Issue #12728: Different Unicode characters having the same uppercase but - different lowercase are now matched in case-insensitive regular expressions. - -- Issue #22821: Fixed fcntl() with integer argument on 64-bit big-endian - platforms. - -- Issue #21650: Add an `--sort-keys` option to json.tool CLI. - -- Issue #22824: Updated reprlib output format for sets to use set literals. - Patch contributed by Berker Peksag. - -- Issue #22824: Updated reprlib output format for arrays to display empty - arrays without an unnecessary empty list. Suggested by Serhiy Storchaka. - -- Issue #22406: Fixed the uu_codec codec incorrectly ported to 3.x. - Based on patch by Martin Panter. - -- Issue #17293: uuid.getnode() now determines MAC address on AIX using netstat. - Based on patch by Aivars Kalvāns. - -- Issue #22769: Fixed ttk.Treeview.tag_has() when called without arguments. - -- Issue #22417: Verify certificates by default in httplib (PEP 476). - -- Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 - and above. Patch by Tim Graham. - -- Issue #22776: Brought excluded code into the scope of a try block in - SysLogHandler.emit(). - -- Issue #22665: Add missing get_terminal_size and SameFileError to - shutil.__all__. - -- Issue #6623: Remove deprecated Netrc class in the ftplib module. Patch by - Matt Chaput. - -- Issue #17381: Fixed handling of case-insensitive ranges in regular - expressions. - -- Issue #22410: Module level functions in the re module now cache compiled - locale-dependent regular expressions taking into account the locale. - -- Issue #22759: Query methods on pathlib.Path() (exists(), is_dir(), etc.) - now return False when the underlying stat call raises NotADirectoryError. - -- Issue #8876: distutils now falls back to copying files when hard linking - doesn't work. This allows use with special filesystems such as VirtualBox - shared folders. - -- Issue #22217: Implemented reprs of classes in the zipfile module. - -- Issue #22457: Honour load_tests in the start_dir of discovery. - -- Issue #18216: gettext now raises an error when a .mo file has an - unsupported major version number. Patch by Aaron Hill. - -- Issue #13918: Provide a locale.delocalize() function which can remove - locale-specific number formatting from a string representing a number, - without then converting it to a specific type. Patch by Cédric Krier. - -- Issue #22676: Make the pickling of global objects which don't have a - __module__ attribute less slow. - -- Issue #18853: Fixed ResourceWarning in shlex.__nain__. - -- Issue #9351: Defaults set with set_defaults on an argparse subparser - are no longer ignored when also set on the parent parser. - -- Issue #7559: unittest test loading ImportErrors are reported as import errors - with their import exception rather than as attribute errors after the import - has already failed. - -- Issue #19746: Make it possible to examine the errors from unittest - discovery without executing the test suite. The new `errors` attribute - on TestLoader exposes these non-fatal errors encountered during discovery. - -- Issue #21991: Make email.headerregistry's header 'params' attributes - be read-only (MappingProxyType). Previously the dictionary was modifiable - but a new one was created on each access of the attribute. - -- Issue #22638: SSLv3 is now disabled throughout the standard library. - It can still be enabled by instantiating a SSLContext manually. - -- Issue #22641: In asyncio, the default SSL context for client connections - is now created using ssl.create_default_context(), for stronger security. - -- Issue #17401: Include closefd in io.FileIO repr. - -- Issue #21338: Add silent mode for compileall. quiet parameters of - compile_{dir, file, path} functions now have a multilevel value. Also, - -q option of the CLI now have a multilevel value. Patch by Thomas Kluyver. - -- Issue #20152: Convert the array and cmath modules to Argument Clinic. - -- Issue #18643: Add socket.socketpair() on Windows. - -- Issue #22435: Fix a file descriptor leak when socketserver bind fails. - -- Issue #13096: Fixed segfault in CTypes POINTER handling of large - values. - -- Issue #11694: Raise ConversionError in xdrlib as documented. Patch - by Filip Gruszczyński and Claudiu Popa. - -- Issue #19380: Optimized parsing of regular expressions. - -- Issue #1519638: Now unmatched groups are replaced with empty strings in re.sub() - and re.subn(). - -- Issue #18615: sndhdr.what/whathdr now return a namedtuple. - -- Issue #22462: Fix pyexpat's creation of a dummy frame to make it - appear in exception tracebacks. - -- Issue #21965: Add support for in-memory SSL to the ssl module. Patch - by Geert Jansen. - -- Issue #21173: Fix len() on a WeakKeyDictionary when .clear() was called - with an iterator alive. - -- Issue #11866: Eliminated race condition in the computation of names - for new threads. - -- Issue #21905: Avoid RuntimeError in pickle.whichmodule() when sys.modules - is mutated while iterating. Patch by Olivier Grisel. - -- Issue #11271: concurrent.futures.Executor.map() now takes a *chunksize* - argument to allow batching of tasks in child processes and improve - performance of ProcessPoolExecutor. Patch by Dan O'Reilly. - -- Issue #21883: os.path.join() and os.path.relpath() now raise a TypeError with - more helpful error message for unsupported or mismatched types of arguments. - -- Issue #22219: The zipfile module CLI now adds entries for directories - (including empty directories) in ZIP file. - -- Issue #22449: In the ssl.SSLContext.load_default_certs, consult the - environmental variables SSL_CERT_DIR and SSL_CERT_FILE on Windows. - -- Issue #22508: The email.__version__ variable has been removed; the email - code is no longer shipped separately from the stdlib, and __version__ - hasn't been updated in several releases. - -- Issue #20076: Added non derived UTF-8 aliases to locale aliases table. - -- Issue #20079: Added locales supported in glibc 2.18 to locale alias table. - -- Issue #20218: Added convenience methods read_text/write_text and read_bytes/ - write_bytes to pathlib.Path objects. - -- Issue #22396: On 32-bit AIX platform, don't expose os.posix_fadvise() nor - os.posix_fallocate() because their prototypes in system headers are wrong. - -- Issue #22517: When an io.BufferedRWPair object is deallocated, clear its - weakrefs. - -- Issue #22437: Number of capturing groups in regular expression is no longer - limited by 100. - -- Issue #17442: InteractiveInterpreter now displays the full chained traceback - in its showtraceback method, to match the built in interactive interpreter. - -- Issue #23392: Added tests for marshal C API that works with FILE*. - - -- Issue #10510: distutils register and upload methods now use HTML standards - compliant CRLF line endings. - -- Issue #9850: Fixed macpath.join() for empty first component. Patch by - Oleg Oshmyan. - -- Issue #5309: distutils' build and build_ext commands now accept a ``-j`` - option to enable parallel building of extension modules. - -- Issue #22448: Improve canceled timer handles cleanup to prevent - unbound memory usage. Patch by Joshua Moore-Oliva. - -- Issue #22427: TemporaryDirectory no longer attempts to clean up twice when - used in the with statement in generator. - -- Issue #22362: Forbidden ambiguous octal escapes out of range 0-0o377 in - regular expressions. - -- Issue #20912: Now directories added to ZIP file have correct Unix and MS-DOS - directory attributes. - -- Issue #21866: ZipFile.close() no longer writes ZIP64 central directory - records if allowZip64 is false. - -- Issue #22278: Fix urljoin problem with relative urls, a regression observed - after changes to issue22118 were submitted. - -- Issue #22415: Fixed debugging output of the GROUPREF_EXISTS opcode in the re - module. Removed trailing spaces in debugging output. - -- Issue #22423: Unhandled exception in thread no longer causes unhandled - AttributeError when sys.stderr is None. - -- Issue #21332: Ensure that ``bufsize=1`` in subprocess.Popen() selects - line buffering, rather than block buffering. Patch by Akira Li. - -- Issue #21091: Fix API bug: email.message.EmailMessage.is_attachment is now - a method. - -- Issue #21079: Fix email.message.EmailMessage.is_attachment to return the - correct result when the header has parameters as well as a value. - -- Issue #22247: Add NNTPError to nntplib.__all__. - -- Issue #22366: urllib.request.urlopen will accept a context object - (SSLContext) as an argument which will then be used for HTTPS connection. - Patch by Alex Gaynor. - -- Issue #4180: The warnings registries are now reset when the filters - are modified. - -- Issue #22419: Limit the length of incoming HTTP request in wsgiref server to - 65536 bytes and send a 414 error code for higher lengths. Patch contributed - by Devin Cook. - -- Lax cookie parsing in http.cookies could be a security issue when combined - with non-standard cookie handling in some Web browsers. Reported by - Sergey Bobrov. - -- Issue #20537: logging methods now accept an exception instance as well as a - Boolean value or exception tuple. Thanks to Yury Selivanov for the patch. - -- Issue #22384: An exception in Tkinter callback no longer crashes the program - when it is run with pythonw.exe. - -- Issue #22168: Prevent turtle AttributeError with non-default Canvas on OS X. - -- Issue #21147: sqlite3 now raises an exception if the request contains a null - character instead of truncating it. Based on patch by Victor Stinner. - -- Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. - -- Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with - empty string or tuple argument. - -- Issue #21951: Tkinter now most likely raises MemoryError instead of crash - if the memory allocation fails. - -- Issue #22338: Fix a crash in the json module on memory allocation failure. - -- Issue #12410: imaplib.IMAP4 now supports the context management protocol. - Original patch by Tarek Ziadé. - -- Issue #21270: We now override tuple methods in mock.call objects so that - they can be used as normal call attributes. - -- Issue #16662: load_tests() is now unconditionally run when it is present in - a package's __init__.py. TestLoader.loadTestsFromModule() still accepts - use_load_tests, but it is deprecated and ignored. A new keyword-only - attribute `pattern` is added and documented. Patch given by Robert Collins, - tweaked by Barry Warsaw. - -- Issue #22226: First letter no longer is stripped from the "status" key in - the result of Treeview.heading(). - -- Issue #19524: Fixed resource leak in the HTTP connection when an invalid - response is received. Patch by Martin Panter. - -- Issue #20421: Add a .version() method to SSL sockets exposing the actual - protocol version in use. - -- Issue #19546: configparser exceptions no longer expose implementation details. - Chained KeyErrors are removed, which leads to cleaner tracebacks. Patch by - Claudiu Popa. - -- Issue #22051: turtledemo no longer reloads examples to re-run them. - Initialization of variables and gui setup should be done in main(), - which is called each time a demo is run, but not on import. - -- Issue #21933: Turtledemo users can change the code font size with a menu - selection or control(command) '-' or '+' or control-mousewheel. - Original patch by Lita Cho. - -- Issue #21597: The separator between the turtledemo text pane and the drawing - canvas can now be grabbed and dragged with a mouse. The code text pane can - be widened to easily view or copy the full width of the text. The canvas - can be widened on small screens. Original patches by Jan Kanis and Lita Cho. - -- Issue #18132: Turtledemo buttons no longer disappear when the window is - shrunk. Original patches by Jan Kanis and Lita Cho. - -- Issue #22043: time.monotonic() is now always available. - ``threading.Lock.acquire()``, ``threading.RLock.acquire()`` and socket - operations now use a monotonic clock, instead of the system clock, when a - timeout is used. - -- Issue #21527: Add a default number of workers to ThreadPoolExecutor equal - to 5 times the number of CPUs. Patch by Claudiu Popa. - -- Issue #22216: smtplib now resets its state more completely after a quit. The - most obvious consequence of the previous behavior was a STARTTLS failure - during a connect/starttls/quit/connect/starttls sequence. - -- Issue #22098: ctypes' BigEndianStructure and LittleEndianStructure now - define an empty __slots__ so that subclasses don't always get an instance - dict. Patch by Claudiu Popa. - -- Issue #22185: Fix an occasional RuntimeError in threading.Condition.wait() - caused by mutation of the waiters queue without holding the lock. Patch - by Doug Zongker. - -- Issue #22287: On UNIX, _PyTime_gettimeofday() now uses - clock_gettime(CLOCK_REALTIME) if available. As a side effect, Python now - depends on the librt library on Solaris and on Linux (only with glibc older - than 2.17). - -- Issue #22182: Use e.args to unpack exceptions correctly in - distutils.file_util.move_file. Patch by Claudiu Popa. - -- The webbrowser module now uses subprocess's start_new_session=True rather - than a potentially risky preexec_fn=os.setsid call. - -- Issue #22042: signal.set_wakeup_fd(fd) now raises an exception if the file - descriptor is in blocking mode. - -- Issue #16808: inspect.stack() now returns a named tuple instead of a tuple. - Patch by Daniel Shahaf. - -- Issue #22236: Fixed Tkinter images copying operations in NoDefaultRoot mode. - -- Issue #2527: Add a *globals* argument to timeit functions, in order to - override the globals namespace in which the timed code is executed. - Patch by Ben Roberts. - -- Issue #22118: Switch urllib.parse to use RFC 3986 semantics for the - resolution of relative URLs, rather than RFCs 1808 and 2396. - Patch by Demian Brecht. - -- Issue #21549: Added the "members" parameter to TarFile.list(). - -- Issue #19628: Allow compileall recursion depth to be specified with a -r - option. - -- Issue #15696: Add a __sizeof__ implementation for mmap objects on Windows. - -- Issue #22068: Avoided reference loops with Variables and Fonts in Tkinter. - -- Issue #22165: SimpleHTTPRequestHandler now supports undecodable file names. - -- Issue #15381: Optimized line reading in io.BytesIO. - -- Issue #8797: Raise HTTPError on failed Basic Authentication immediately. - Initial patch by Sam Bull. - -- Issue #20729: Restored the use of lazy iterkeys()/itervalues()/iteritems() - in the mailbox module. - -- Issue #21448: Changed FeedParser feed() to avoid O(N**2) behavior when - parsing long line. Original patch by Raymond Hettinger. - -- Issue #22184: The functools LRU Cache decorator factory now gives an earlier - and clearer error message when the user forgets the required parameters. - -- Issue #17923: glob() patterns ending with a slash no longer match non-dirs on - AIX. Based on patch by Delhallt. - -- Issue #21725: Added support for RFC 6531 (SMTPUTF8) in smtpd. - -- Issue #22176: Update the ctypes module's libffi to v3.1. This release - adds support for the Linux AArch64 and POWERPC ELF ABIv2 little endian - architectures. - -- Issue #5411: Added support for the "xztar" format in the shutil module. - -- Issue #21121: Don't force 3rd party C extensions to be built with - -Werror=declaration-after-statement. - -- Issue #21975: Fixed crash when using uninitialized sqlite3.Row (in particular - when unpickling pickled sqlite3.Row). sqlite3.Row is now initialized in the - __new__() method. - -- Issue #20170: Convert posixmodule to use Argument Clinic. - -- Issue #21539: Add an *exists_ok* argument to `Pathlib.mkdir()` to mimic - `mkdir -p` and `os.makedirs()` functionality. When true, ignore - FileExistsErrors. Patch by Berker Peksag. - -- Issue #22127: Bypass IDNA for pure-ASCII host names in the socket module - (in particular for numeric IPs). - -- Issue #21047: set the default value for the *convert_charrefs* argument - of HTMLParser to True. Patch by Berker Peksag. - -- Add an __all__ to html.entities. - -- Issue #15114: the strict mode and argument of HTMLParser, HTMLParser.error, - and the HTMLParserError exception have been removed. - -- Issue #22085: Dropped support of Tk 8.3 in Tkinter. - -- Issue #21580: Now Tkinter correctly handles bytes arguments passed to Tk. - In particular this allows initializing images from binary data. - -- Issue #22003: When initialized from a bytes object, io.BytesIO() now - defers making a copy until it is mutated, improving performance and - memory use on some use cases. Patch by David Wilson. - -- Issue #22018: On Windows, signal.set_wakeup_fd() now also supports sockets. - A side effect is that Python depends to the WinSock library. - -- Issue #22054: Add os.get_blocking() and os.set_blocking() functions to get - and set the blocking mode of a file descriptor (False if the O_NONBLOCK flag - is set, True otherwise). These functions are not available on Windows. - -- Issue #17172: Make turtledemo start as active on OS X even when run with - subprocess. Patch by Lita Cho. - -- Issue #21704: Fix build error for _multiprocessing when semaphores - are not available. Patch by Arfrever Frehtes Taifersar Arahesis. - -- Issue #20173: Convert sha1, sha256, sha512 and md5 to ArgumentClinic. - Patch by Vajrasky Kok. - -- Fix repr(_socket.socket) on Windows 64-bit: don't fail with OverflowError - on closed socket. repr(socket.socket) already works fine. - -- Issue #22033: Reprs of most Python implemened classes now contain actual - class name instead of hardcoded one. - -- Issue #21947: The dis module can now disassemble generator-iterator - objects based on their gi_code attribute. Patch by Clement Rouault. - -- Issue #16133: The asynchat.async_chat.handle_read() method now ignores - BlockingIOError exceptions. - -- Issue #22044: Fixed premature DECREF in call_tzinfo_method. - Patch by Tom Flanagan. - -- Issue #19884: readline: Disable the meta modifier key if stdout is not - a terminal to not write the ANSI sequence ``"\033[1034h"`` into stdout. This - sequence is used on some terminal (ex: TERM=xterm-256color") to enable - support of 8 bit characters. - -- Issue #4350: Removed a number of out-of-dated and non-working for a long time - Tkinter methods. - -- Issue #6167: Scrollbar.activate() now returns the name of active element if - the argument is not specified. Scrollbar.set() now always accepts only 2 - arguments. - -- Issue #15275: Clean up and speed up the ntpath module. - -- Issue #21888: plistlib's load() and loads() now work if the fmt parameter is - specified. - -- Issue #22032: __qualname__ instead of __name__ is now always used to format - fully qualified class names of Python implemented classes. - -- Issue #22031: Reprs now always use hexadecimal format with the "0x" prefix - when contain an id in form " at 0x...". - -- Issue #22018: signal.set_wakeup_fd() now raises an OSError instead of a - ValueError on ``fstat()`` failure. - -- Issue #21044: tarfile.open() now handles fileobj with an integer 'name' - attribute. Based on patch by Antoine Pietri. - -- Issue #21966: Respect -q command-line option when code module is ran. - -- Issue #19076: Don't pass the redundant 'file' argument to self.error(). - -- Issue #16382: Improve exception message of warnings.warn() for bad - category. Initial patch by Phil Elson. - -- Issue #21932: os.read() now uses a :c:func:`Py_ssize_t` type instead of - :c:type:`int` for the size to support reading more than 2 GB at once. On - Windows, the size is truncted to INT_MAX. As any call to os.read(), the OS - may read less bytes than the number of requested bytes. - -- Issue #21942: Fixed source file viewing in pydoc's server mode on Windows. - -- Issue #11259: asynchat.async_chat().set_terminator() now raises a ValueError - if the number of received bytes is negative. - -- Issue #12523: asynchat.async_chat.push() now raises a TypeError if it doesn't - get a bytes string - -- Issue #21707: Add missing kwonlyargcount argument to - ModuleFinder.replace_paths_in_code(). - -- Issue #20639: calling Path.with_suffix('') allows removing the suffix - again. Patch by July Tikhonov. - -- Issue #21714: Disallow the construction of invalid paths using - Path.with_name(). Original patch by Antony Lee. - -- Issue #15014: Added 'auth' method to smtplib to make implementing auth - mechanisms simpler, and used it internally in the login method. - -- Issue #21151: Fixed a segfault in the winreg module when ``None`` is passed - as a ``REG_BINARY`` value to SetValueEx. Patch by John Ehresman. - -- Issue #21090: io.FileIO.readall() does not ignore I/O errors anymore. Before, - it ignored I/O errors if at least the first C call read() succeed. - -- Issue #5800: headers parameter of wsgiref.headers.Headers is now optional. - Initial patch by Pablo Torres Navarrete and SilentGhost. - -- Issue #21781: ssl.RAND_add() now supports strings longer than 2 GB. - -- Issue #21679: Prevent extraneous fstat() calls during open(). Patch by - Bohuslav Kabrda. - -- Issue #21863: cProfile now displays the module name of C extension functions, - in addition to their own name. - -- Issue #11453: asyncore: emit a ResourceWarning when an unclosed file_wrapper - object is destroyed. The destructor now closes the file if needed. The - close() method can now be called twice: the second call does nothing. - -- Issue #21858: Better handling of Python exceptions in the sqlite3 module. - -- Issue #21476: Make sure the email.parser.BytesParser TextIOWrapper is - discarded after parsing, so the input file isn't unexpectedly closed. - -- Issue #20295: imghdr now recognizes OpenEXR format images. - -- Issue #21729: Used the "with" statement in the dbm.dumb module to ensure - files closing. Patch by Claudiu Popa. - -- Issue #21491: socketserver: Fix a race condition in child processes reaping. - -- Issue #21719: Added the ``st_file_attributes`` field to os.stat_result on - Windows. - -- Issue #21832: Require named tuple inputs to be exact strings. - -- Issue #21722: The distutils "upload" command now exits with a non-zero - return code when uploading fails. Patch by Martin Dengler. - -- Issue #21723: asyncio.Queue: support any type of number (ex: float) for the - maximum size. Patch written by Vajrasky Kok. - -- Issue #21711: support for "site-python" directories has now been removed - from the site module (it was deprecated in 3.4). - -- Issue #17552: new socket.sendfile() method allowing a file to be sent over a - socket by using high-performance os.sendfile() on UNIX. - Patch by Giampaolo Rodola'. - -- Issue #18039: dbm.dump.open() now always creates a new database when the - flag has the value 'n'. Patch by Claudiu Popa. - -- Issue #21326: Add a new is_closed() method to asyncio.BaseEventLoop. - run_forever() and run_until_complete() methods of asyncio.BaseEventLoop now - raise an exception if the event loop was closed. - -- Issue #21766: Prevent a security hole in CGIHTTPServer by URL unquoting paths - before checking for a CGI script at that path. - -- Issue #21310: Fixed possible resource leak in failed open(). - -- Issue #21256: Printout of keyword args should be in deterministic order in - a mock function call. This will help to write better doctests. - -- Issue #21677: Fixed chaining nonnormalized exceptions in io close() methods. - -- Issue #11709: Fix the pydoc.help function to not fail when sys.stdin is not a - valid file. - -- Issue #21515: tempfile.TemporaryFile now uses os.O_TMPFILE flag is available. - -- Issue #13223: Fix pydoc.writedoc so that the HTML documentation for methods - that use 'self' in the example code is generated correctly. - -- Issue #21463: In urllib.request, fix pruning of the FTP cache. - -- Issue #21618: The subprocess module could fail to close open fds that were - inherited by the calling process and already higher than POSIX resource - limits would otherwise allow. On systems with a functioning /proc/self/fd - or /dev/fd interface the max is now ignored and all fds are closed. - -- Issue #20383: Introduce importlib.util.module_from_spec() as the preferred way - to create a new module. - -- Issue #21552: Fixed possible integer overflow of too long string lengths in - the tkinter module on 64-bit platforms. - -- Issue #14315: The zipfile module now ignores extra fields in the central - directory that are too short to be parsed instead of letting a struct.unpack - error bubble up as this "bad data" appears in many real world zip files in - the wild and is ignored by other zip tools. - -- Issue #13742: Added "key" and "reverse" parameters to heapq.merge(). - (First draft of patch contributed by Simon Sapin.) - -- Issue #21402: tkinter.ttk now works when default root window is not set. - -- Issue #3015: _tkinter.create() now creates tkapp object with wantobject=1 by - default. - -- Issue #10203: sqlite3.Row now truly supports sequence protocol. In particular - it supports reverse() and negative indices. Original patch by Claudiu Popa. - -- Issue #18807: If copying (no symlinks) specified for a venv, then the python - interpreter aliases (python, python3) are now created by copying rather than - symlinking. - -- Issue #20197: Added support for the WebP image type in the imghdr module. - Patch by Fabrice Aneche and Claudiu Popa. - -- Issue #21513: Speedup some properties of IP addresses (IPv4Address, - IPv6Address) such as .is_private or .is_multicast. - -- Issue #21137: Improve the repr for threading.Lock() and its variants - by showing the "locked" or "unlocked" status. Patch by Berker Peksag. - -- Issue #21538: The plistlib module now supports loading of binary plist files - when reference or offset size is not a power of two. - -- Issue #21455: Add a default backlog to socket.listen(). - -- Issue #21525: Most Tkinter methods which accepted tuples now accept lists too. - -- Issue #22166: With the assistance of a new internal _codecs._forget_codec - helping function, test_codecs now clears the encoding caches to avoid the - appearance of a reference leak - -- Issue #22236: Tkinter tests now don't reuse default root window. New root - window is created for every test class. - -- Issue #10744: Fix PEP 3118 format strings on ctypes objects with a nontrivial - shape. - -- Issue #20826: Optimize ipaddress.collapse_addresses(). - -- Issue #21487: Optimize ipaddress.summarize_address_range() and - ipaddress.{IPv4Network,IPv6Network}.subnets(). - -- Issue #21486: Optimize parsing of netmasks in ipaddress.IPv4Network and - ipaddress.IPv6Network. - -- Issue #13916: Disallowed the surrogatepass error handler for non UTF-\* - encodings. - -- Issue #20998: Fixed re.fullmatch() of repeated single character pattern - with ignore case. Original patch by Matthew Barnett. - -- Issue #21075: fileinput.FileInput now reads bytes from standard stream if - binary mode is specified. Patch by Sam Kimbrel. - -- Issue #19775: Add a samefile() method to pathlib Path objects. Initial - patch by Vajrasky Kok. - -- Issue #21226: Set up modules properly in PyImport_ExecCodeModuleObject - (and friends). - -- Issue #21398: Fix a unicode error in the pydoc pager when the documentation - contains characters not encodable to the stdout encoding. - -- Issue #16531: ipaddress.IPv4Network and ipaddress.IPv6Network now accept - an (address, netmask) tuple argument, so as to easily construct network - objects from existing addresses. - -- Issue #21156: importlib.abc.InspectLoader.source_to_code() is now a - staticmethod. - -- Issue #21424: Simplified and optimized heaqp.nlargest() and nmsmallest() - to make fewer tuple comparisons. - -- Issue #21396: Fix TextIOWrapper(..., write_through=True) to not force a - flush() on the underlying binary stream. Patch by akira. - -- Issue #18314: Unlink now removes junctions on Windows. Patch by Kim Gräsman - -- Issue #21088: Bugfix for curses.window.addch() regression in 3.4.0. - In porting to Argument Clinic, the first two arguments were reversed. - -- Issue #21407: _decimal: The module now supports function signatures. - -- Issue #10650: Remove the non-standard 'watchexp' parameter from the - Decimal.quantize() method in the Python version. It had never been - present in the C version. - -- Issue #21469: Reduced the risk of false positives in robotparser by - checking to make sure that robots.txt has been read or does not exist - prior to returning True in can_fetch(). - -- Issue #19414: Have the OrderedDict mark deleted links as unusable. - This gives an early failure if the link is deleted during iteration. - -- Issue #21421: Add __slots__ to the MappingViews ABC. - Patch by Josh Rosenberg. - -- Issue #21101: Eliminate double hashing in the C speed-up code for - collections.Counter(). - -- Issue #21321: itertools.islice() now releases the reference to the source - iterator when the slice is exhausted. Patch by Anton Afanasyev. - -- Issue #21057: TextIOWrapper now allows the underlying binary stream's - read() or read1() method to return an arbitrary bytes-like object - (such as a memoryview). Patch by Nikolaus Rath. - -- Issue #20951: SSLSocket.send() now raises either SSLWantReadError or - SSLWantWriteError on a non-blocking socket if the operation would block. - Previously, it would return 0. Patch by Nikolaus Rath. - -- Issue #13248: removed previously deprecated asyncore.dispatcher __getattr__ - cheap inheritance hack. - -- Issue #9815: assertRaises now tries to clear references to local variables - in the exception's traceback. - -- Issue #19940: ssl.cert_time_to_seconds() now interprets the given time - string in the UTC timezone (as specified in RFC 5280), not the local - timezone. - -- Issue #13204: Calling sys.flags.__new__ would crash the interpreter, - now it raises a TypeError. - -- Issue #19385: Make operations on a closed dbm.dumb database always raise the - same exception. - -- Issue #21207: Detect when the os.urandom cached fd has been closed or - replaced, and open it anew. - -- Issue #21291: subprocess's Popen.wait() is now thread safe so that - multiple threads may be calling wait() or poll() on a Popen instance - at the same time without losing the Popen.returncode value. - -- Issue #21127: Path objects can now be instantiated from str subclass - instances (such as ``numpy.str_``). - -- Issue #15002: urllib.response object to use _TemporaryFileWrapper (and - _TemporaryFileCloser) facility. Provides a better way to handle file - descriptor close. Patch contributed by Christian Theune. - -- Issue #12220: mindom now raises a custom ValueError indicating it doesn't - support spaces in URIs instead of letting a 'split' ValueError bubble up. - -- Issue #21068: The ssl.PROTOCOL* constants are now enum members. - -- Issue #21276: posixmodule: Don't define USE_XATTRS on KFreeBSD and the Hurd. - -- Issue #21262: New method assert_not_called for Mock. - It raises AssertionError if the mock has been called. - -- Issue #21238: New keyword argument `unsafe` to Mock. It raises - `AttributeError` incase of an attribute startswith assert or assret. - -- Issue #20896: ssl.get_server_certificate() now uses PROTOCOL_SSLv23, not - PROTOCOL_SSLv3, for maximum compatibility. - -- Issue #21239: patch.stopall() didn't work deterministically when the same - name was patched more than once. - -- Issue #21203: Updated fileConfig and dictConfig to remove inconsistencies. - Thanks to Jure Koren for the patch. - -- Issue #21222: Passing name keyword argument to mock.create_autospec now - works. - -- Issue #21197: Add lib64 -> lib symlink in venvs on 64-bit non-OS X POSIX. - -- Issue #17498: Some SMTP servers disconnect after certain errors, violating - strict RFC conformance. Instead of losing the error code when we issue the - subsequent RSET, smtplib now returns the error code and defers raising the - SMTPServerDisconnected error until the next command is issued. - -- Issue #17826: setting an iterable side_effect on a mock function created by - create_autospec now works. Patch by Kushal Das. - -- Issue #7776: Fix ``Host:`` header and reconnection when using - http.client.HTTPConnection.set_tunnel(). Patch by Nikolaus Rath. - -- Issue #20968: unittest.mock.MagicMock now supports division. - Patch by Johannes Baiter. - -- Issue #21529 (CVE-2014-4616): Fix arbitrary memory access in - JSONDecoder.raw_decode with a negative second parameter. Bug reported by Guido - Vranken. - -- Issue #21169: getpass now handles non-ascii characters that the - input stream encoding cannot encode by re-encoding using the - replace error handler. - -- Issue #21171: Fixed undocumented filter API of the rot13 codec. - Patch by Berker Peksag. - -- Issue #20539: Improved math.factorial error message for large positive inputs - and changed exception type (OverflowError -> ValueError) for large negative - inputs. - -- Issue #21172: isinstance check relaxed from dict to collections.Mapping. - -- Issue #21155: asyncio.EventLoop.create_unix_server() now raises a ValueError - if path and sock are specified at the same time. - -- Issue #21136: Avoid unnecessary normalization of Fractions resulting from - power and other operations. Patch by Raymond Hettinger. - -- Issue #17621: Introduce importlib.util.LazyLoader. - -- Issue #21076: signal module constants were turned into enums. - Patch by Giampaolo Rodola'. - -- Issue #20636: Improved the repr of Tkinter widgets. - -- Issue #19505: The items, keys, and values views of OrderedDict now support - reverse iteration using reversed(). - -- Issue #21149: Improved thread-safety in logging cleanup during interpreter - shutdown. Thanks to Devin Jeanpierre for the patch. - -- Issue #21058: Fix a leak of file descriptor in - :func:`tempfile.NamedTemporaryFile`, close the file descriptor if - :func:`io.open` fails - -- Issue #21200: Return None from pkgutil.get_loader() when __spec__ is missing. - -- Issue #21013: Enhance ssl.create_default_context() when used for server side - sockets to provide better security by default. - -- Issue #20145: `assertRaisesRegex` and `assertWarnsRegex` now raise a - TypeError if the second argument is not a string or compiled regex. - -- Issue #20633: Replace relative import by absolute import. - -- Issue #20980: Stop wrapping exception when using ThreadPool. - -- Issue #21082: In os.makedirs, do not set the process-wide umask. Note this - changes behavior of makedirs when exist_ok=True. - -- Issue #20990: Fix issues found by pyflakes for multiprocessing. - -- Issue #21015: SSL contexts will now automatically select an elliptic - curve for ECDH key exchange on OpenSSL 1.0.2 and later, and otherwise - default to "prime256v1". - -- Issue #21000: Improve the command-line interface of json.tool. - -- Issue #20995: Enhance default ciphers used by the ssl module to enable - better security and prioritize perfect forward secrecy. - -- Issue #20884: Don't assume that __file__ is defined on importlib.__init__. - -- Issue #21499: Ignore __builtins__ in several test_importlib.test_api tests. - -- Issue #20627: xmlrpc.client.ServerProxy is now a context manager. - -- Issue #19165: The formatter module now raises DeprecationWarning instead of - PendingDeprecationWarning. - -- Issue #13936: Remove the ability of datetime.time instances to be considered - false in boolean contexts. - -- Issue #18931: selectors module now supports /dev/poll on Solaris. - Patch by Giampaolo Rodola'. - -- Issue #19977: When the ``LC_TYPE`` locale is the POSIX locale (``C`` locale), - :py:data:`sys.stdin` and :py:data:`sys.stdout` are now using the - ``surrogateescape`` error handler, instead of the ``strict`` error handler. - -- Issue #20574: Implement incremental decoder for cp65001 code (Windows code - page 65001, Microsoft UTF-8). - -- Issue #20879: Delay the initialization of encoding and decoding tables for - base32, ascii85 and base85 codecs in the base64 module, and delay the - initialization of the unquote_to_bytes() table of the urllib.parse module, to - not waste memory if these modules are not used. - -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 - in ipaddress. - -- Issue #11599: When an external command (e.g. compiler) fails, distutils now - prints out the whole command line (instead of just the command name) if the - environment variable DISTUTILS_DEBUG is set. - -- Issue #4931: distutils should not produce unhelpful "error: None" messages - anymore. distutils.util.grok_environment_error is kept but doc-deprecated. - -- Issue #20875: Prevent possible gzip "'read' is not defined" NameError. - Patch by Claudiu Popa. - -- Issue #11558: ``email.message.Message.attach`` now returns a more - useful error message if ``attach`` is called on a message for which - ``is_multipart`` is False. - -- Issue #20283: RE pattern methods now accept the string keyword parameters - as documented. The pattern and source keyword parameters are left as - deprecated aliases. - -- Issue #20778: Fix modulefinder to work with bytecode-only modules. - -- Issue #20791: copy.copy() now doesn't make a copy when the input is - a bytes object. Initial patch by Peter Otten. - -- Issue #19748: On AIX, time.mktime() now raises an OverflowError for year - outsize range [1902; 2037]. - -- Issue #19573: inspect.signature: Use enum for parameter kind constants. - -- Issue #20726: inspect.signature: Make Signature and Parameter picklable. - -- Issue #17373: Add inspect.Signature.from_callable method. - -- Issue #20378: Improve repr of inspect.Signature and inspect.Parameter. - -- Issue #20816: Fix inspect.getcallargs() to raise correct TypeError for - missing keyword-only arguments. Patch by Jeremiah Lowin. - -- Issue #20817: Fix inspect.getcallargs() to fail correctly if more - than 3 arguments are missing. Patch by Jeremiah Lowin. - -- Issue #6676: Ensure a meaningful exception is raised when attempting - to parse more than one XML document per pyexpat xmlparser instance. - (Original patches by Hirokazu Yamamoto and Amaury Forgeot d'Arc, with - suggested wording by David Gutteridge) - -- Issue #21117: Fix inspect.signature to better support functools.partial. - Due to the specifics of functools.partial implementation, - positional-or-keyword arguments passed as keyword arguments become - keyword-only. - -- Issue #20334: inspect.Signature and inspect.Parameter are now hashable. - Thanks to Antony Lee for bug reports and suggestions. - -- Issue #15916: doctest.DocTestSuite returns an empty unittest.TestSuite instead - of raising ValueError if it finds no tests - -- Issue #21209: Fix asyncio.tasks.CoroWrapper to workaround a bug - in yield-from implementation in CPythons prior to 3.4.1. - -- asyncio: Add gi_{frame,running,code} properties to CoroWrapper - (upstream issue #163). - -- Issue #21311: Avoid exception in _osx_support with non-standard compiler - configurations. Patch by John Szakmeister. - -- Issue #11571: Ensure that the turtle window becomes the topmost window - when launched on OS X. - -- Issue #21801: Validate that __signature__ is None or an instance of Signature. - -- Issue #21923: Prevent AttributeError in distutils.sysconfig.customize_compiler - due to possible uninitialized _config_vars. - -- Issue #21323: Fix http.server to again handle scripts in CGI subdirectories, - broken by the fix for security issue #19435. Patch by Zach Byrne. - -- Issue #22733: Fix ffi_prep_args not zero-extending argument values correctly - on 64-bit Windows. - -- Issue #23302: Default to TCP_NODELAY=1 upon establishing an HTTPConnection. - Removed use of hard-coded MSS as it's an optimization that's no longer needed - with Nagle disabled. - -IDLE ----- - -- Issue #20577: Configuration of the max line length for the FormatParagraph - extension has been moved from the General tab of the Idle preferences dialog - to the FormatParagraph tab of the Config Extensions dialog. - Patch by Tal Einat. - -- Issue #16893: Update Idle doc chapter to match current Idle and add new - information. - -- Issue #3068: Add Idle extension configuration dialog to Options menu. - Changes are written to HOME/.idlerc/config-extensions.cfg. - Original patch by Tal Einat. - -- Issue #16233: A module browser (File : Class Browser, Alt+C) requires an - editor window with a filename. When Class Browser is requested otherwise, - from a shell, output window, or 'Untitled' editor, Idle no longer displays - an error box. It now pops up an Open Module box (Alt+M). If a valid name - is entered and a module is opened, a corresponding browser is also opened. - -- Issue #4832: Save As to type Python files automatically adds .py to the - name you enter (even if your system does not display it). Some systems - automatically add .txt when type is Text files. - -- Issue #21986: Code objects are not normally pickled by the pickle module. - To match this, they are no longer pickled when running under Idle. - -- Issue #17390: Adjust Editor window title; remove 'Python', - move version to end. - -- Issue #14105: Idle debugger breakpoints no longer disappear - when inserting or deleting lines. - -- Issue #17172: Turtledemo can now be run from Idle. - Currently, the entry is on the Help menu, but it may move to Run. - Patch by Ramchandra Apt and Lita Cho. - -- Issue #21765: Add support for non-ascii identifiers to HyperParser. - -- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav - Heblikar. - -- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. - -- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. - -- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav - Heblikar. - -- Issue #12387: Add missing upper(lower)case versions of default Windows key - bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. - -- Issue #21695: Closing a Find-in-files output window while the search is - still in progress no longer closes Idle. - -- Issue #18910: Add unittest for textView. Patch by Phil Webster. - -- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. - -- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. - -- Issue #21477: htest.py - Improve framework, complete set of tests. - Patches by Saimadhav Heblikar - -- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin - consolidating and improving human-validated tests of Idle. Change other files - as needed to work with htest. Running the module as __main__ runs all tests. - -- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. - -- Issue #21284: Paragraph reformat test passes after user changes reformat width. - -- Issue #17654: Ensure IDLE menus are customized properly on OS X for - non-framework builds and for all variants of Tk. - -- Issue #23180: Rename IDLE "Windows" menu item to "Window". - Patch by Al Sweigart. - -Build ------ - -- Issue #15506: Use standard PKG_PROG_PKG_CONFIG autoconf macro in the configure - script. - -- Issue #22935: Allow the ssl module to be compiled if openssl doesn't support - SSL 3. - -- Issue #22592: Drop support of the Borland C compiler to build Python. The - distutils module still supports it to build extensions. - -- Issue #22591: Drop support of MS-DOS, especially of the DJGPP compiler - (MS-DOS port of GCC). - -- Issue #16537: Check whether self.extensions is empty in setup.py. Patch by - Jonathan Hosmer. - -- Issue #22359: Remove incorrect uses of recursive make. Patch by Jonas - Wagner. - -- Issue #21958: Define HAVE_ROUND when building with Visual Studio 2013 and - above. Patch by Zachary Turner. - -- Issue #18093: the programs that embed the CPython runtime are now in a - separate "Programs" directory, rather than being kept in the Modules - directory. - -- Issue #15759: "make suspicious", "make linkcheck" and "make doctest" in Doc/ - now display special message when and only when there are failures. - -- Issue #21141: The Windows build process no longer attempts to find Perl, - instead relying on OpenSSL source being configured and ready to build. The - ``PCbuild\build_ssl.py`` script has been re-written and re-named to - ``PCbuild\prepare_ssl.py``, and takes care of configuring OpenSSL source - for both 32 and 64 bit platforms. OpenSSL sources obtained from - svn.python.org will always be pre-configured and ready to build. - -- Issue #21037: Add a build option to enable AddressSanitizer support. - -- Issue #19962: The Windows build process now creates "python.bat" in the - root of the source tree, which passes all arguments through to the most - recently built interpreter. - -- Issue #21285: Refactor and fix curses configure check to always search - in a ncursesw directory. - -- Issue #15234: For BerkelyDB and Sqlite, only add the found library and - include directories if they aren't already being searched. This avoids - an explicit runtime library dependency. - -- Issue #17861: Tools/scripts/generate_opcode_h.py automatically regenerates - Include/opcode.h from Lib/opcode.py if the latter gets any change. - -- Issue #20644: OS X installer build support for documentation build changes - in 3.4.1: assume externally supplied sphinx-build is available in /usr/bin. - -- Issue #20022: Eliminate use of deprecated bundlebuilder in OS X builds. - -- Issue #15968: Incorporated Tcl, Tk, and Tix builds into the Windows build - solution. - -- Issue #17095: Fix Modules/Setup *shared* support. - -- Issue #21811: Anticipated fixes to support OS X versions > 10.9. - -- Issue #21166: Prevent possible segfaults and other random failures of - python --generate-posix-vars in pybuilddir.txt build target. - -- Issue #18096: Fix library order returned by python-config. - -- Issue #17219: Add library build dir for Python extension cross-builds. - -- Issue #22919: Windows build updated to support VC 14.0 (Visual Studio 2015), - which will be used for the official release. - -- Issue #21236: Build _msi.pyd with cabinet.lib instead of fci.lib - -- Issue #17128: Use private version of OpenSSL for OS X 10.5+ installer. - -C API ------ - -- Issue #14203: Remove obsolete support for view==NULL in PyBuffer_FillInfo(), - bytearray_getbuffer(), bytesiobuf_getbuffer() and array_buffer_getbuf(). - All functions now raise BufferError in that case. - -- Issue #22445: PyBuffer_IsContiguous() now implements precise contiguity - tests, compatible with NumPy's NPY_RELAXED_STRIDES_CHECKING compilation - flag. Previously the function reported false negatives for corner cases. - -- Issue #22079: PyType_Ready() now checks that statically allocated type has - no dynamically allocated bases. - -- Issue #22453: Removed non-documented macro PyObject_REPR(). - -- Issue #18395: Rename ``_Py_char2wchar()`` to :c:func:`Py_DecodeLocale`, - rename ``_Py_wchar2char()`` to :c:func:`Py_EncodeLocale`, and document - these functions. - -- Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(), - PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) is now using - ``calloc()`` instead of ``malloc()`` for large objects which is faster and - use less memory. - -- Issue #20942: PyImport_ImportFrozenModuleObject() no longer sets __file__ to - match what importlib does; this affects _frozen_importlib as well as any - module loaded using imp.init_frozen(). - -Documentation -------------- - -- Issue #19548: Update the codecs module documentation to better cover the - distinction between text encodings and other codecs, together with other - clarifications. Patch by Martin Panter. - -- Issue #22394: Doc/Makefile now supports ``make venv PYTHON=../python`` to - create a venv for generating the documentation, e.g., - ``make html PYTHON=venv/bin/python3``. - -- Issue #21514: The documentation of the json module now refers to new JSON RFC - 7159 instead of obsoleted RFC 4627. - -- Issue #21777: The binary sequence methods on bytes and bytearray are now - documented explicitly, rather than assuming users will be able to derive - the expected behaviour from the behaviour of the corresponding str methods. - -- Issue #6916: undocument deprecated asynchat.fifo class. - -- Issue #17386: Expanded functionality of the ``Doc/make.bat`` script to make - it much more comparable to ``Doc/Makefile``. - -- Issue #21312: Update the thread_foobar.h template file to include newer - threading APIs. Patch by Jack McCracken. - -- Issue #21043: Remove the recommendation for specific CA organizations and to - mention the ability to load the OS certificates. - -- Issue #20765: Add missing documentation for PurePath.with_name() and - PurePath.with_suffix(). - -- Issue #19407: New package installation and distribution guides based on - the Python Packaging Authority tools. Existing guides have been retained - as legacy links from the distutils docs, as they still contain some - required reference material for tool developers that isn't recorded - anywhere else. - -- Issue #19697: Document cases where __main__.__spec__ is None. - -Tests ------ - -- Issue #18982: Add tests for CLI of the calendar module. - -- Issue #19548: Added some additional checks to test_codecs to ensure that - statements in the updated documentation remain accurate. Patch by Martin - Panter. - -- Issue #22838: All test_re tests now work with unittest test discovery. - -- Issue #22173: Update lib2to3 tests to use unittest test discovery. - -- Issue #16000: Convert test_curses to use unittest. - -- Issue #21456: Skip two tests in test_urllib2net.py if _ssl module not - present. Patch by Remi Pointel. - -- Issue #20746: Fix test_pdb to run in refleak mode (-R). Patch by Xavier - de Gaye. - -- Issue #22060: test_ctypes has been somewhat cleaned up and simplified; it - now uses unittest test discovery to find its tests. - -- Issue #22104: regrtest.py no longer holds a reference to the suite of tests - loaded from test modules that don't define test_main(). - -- Issue #22111: Assorted cleanups in test_imaplib. Patch by Milan Oberkirch. - -- Issue #22002: Added ``load_package_tests`` function to test.support and used - it to implement/augment test discovery in test_asyncio, test_email, - test_importlib, test_json, and test_tools. - -- Issue #21976: Fix test_ssl to accept LibreSSL version strings. Thanks - to William Orr. - -- Issue #21918: Converted test_tools from a module to a package containing - separate test files for each tested script. - -- Issue #9554: Use modern unittest features in test_argparse. Initial patch by - Denver Coneybeare and Radu Voicilas. - -- Issue #20155: Changed HTTP method names in failing tests in test_httpservers - so that packet filtering software (specifically Windows Base Filtering Engine) - does not interfere with the transaction semantics expected by the tests. - -- Issue #19493: Refactored the ctypes test package to skip tests explicitly - rather than silently. - -- Issue #18492: All resources are now allowed when tests are not run by - regrtest.py. - -- Issue #21634: Fix pystone micro-benchmark: use floor division instead of true - division to benchmark integers instead of floating point numbers. Set pystone - version to 1.2. Patch written by Lennart Regebro. - -- Issue #21605: Added tests for Tkinter images. - -- Issue #21493: Added test for ntpath.expanduser(). Original patch by - Claudiu Popa. - -- Issue #19925: Added tests for the spwd module. Original patch by Vajrasky Kok. - -- Issue #21522: Added Tkinter tests for Listbox.itemconfigure(), - PanedWindow.paneconfigure(), and Menu.entryconfigure(). - -- Issue #17756: Fix test_code test when run from the installed location. - -- Issue #17752: Fix distutils tests when run from the installed location. - -- Issue #18604: Consolidated checks for GUI availability. All platforms now - at least check whether Tk can be instantiated when the GUI resource is - requested. - -- Issue #21275: Fix a socket test on KFreeBSD. - -- Issue #21223: Pass test_site/test_startup_imports when some of the extensions - are built as builtins. - -- Issue #20635: Added tests for Tk geometry managers. - -- Add test case for freeze. - -- Issue #20743: Fix a reference leak in test_tcl. - -- Issue #21097: Move test_namespace_pkgs into test_importlib. - -- Issue #21503: Use test_both() consistently in test_importlib. - -- Issue #20939: Avoid various network test failures due to new - redirect of http://www.python.org/ to https://www.python.org: - use http://www.example.com instead. - -- Issue #20668: asyncio tests no longer rely on tests.txt file. - (Patch by Vajrasky Kok) - -- Issue #21093: Prevent failures of ctypes test_macholib on OS X if a - copy of libz exists in $HOME/lib or /usr/local/lib. - -- Issue #22770: Prevent some Tk segfaults on OS X when running gui tests. - -- Issue #23211: Workaround test_logging failure on some OS X 10.6 systems. - -- Issue #23345: Prevent test_ssl failures with large OpenSSL patch level - values (like 0.9.8zc). - -Tools/Demos ------------ - -- Issue #22314: pydoc now works when the LINES environment variable is set. - -- Issue #22615: Argument Clinic now supports the "type" argument for the - int converter. This permits using the int converter with enums and - typedefs. - -- Issue #20076: The makelocalealias.py script no longer ignores UTF-8 mapping. - -- Issue #20079: The makelocalealias.py script now can parse the SUPPORTED file - from glibc sources and supports command line options for source paths. - -- Issue #22201: Command-line interface of the zipfile module now correctly - extracts ZIP files with directory entries. Patch by Ryan Wilson. - -- Issue #22120: For functions using an unsigned integer return converter, - Argument Clinic now generates a cast to that type for the comparison - to -1 in the generated code. (This suppresses a compilation warning.) - -- Issue #18974: Tools/scripts/diff.py now uses argparse instead of optparse. - -- Issue #21906: Make Tools/scripts/md5sum.py work in Python 3. - Patch by Zachary Ware. - -- Issue #21629: Fix Argument Clinic's "--converters" feature. - -- Add support for ``yield from`` to 2to3. - -- Add support for the PEP 465 matrix multiplication operator to 2to3. - -- Issue #16047: Fix module exception list and __file__ handling in freeze. - Patch by Meador Inge. - -- Issue #11824: Consider ABI tags in freeze. Patch by Meador Inge. - -- Issue #20535: PYTHONWARNING no longer affects the run_tests.py script. - Patch by Arfrever Frehtes Taifersar Arahesis. - -Windows -------- - -- Issue #23260: Update Windows installer - -- The bundled version of Tcl/Tk has been updated to 8.6.3. The most visible - result of this change is the addition of new native file dialogs when - running on Windows Vista or newer. See Tcl/Tk's TIP 432 for more - information. Also, this version of Tcl/Tk includes support for Windows 10. - -- Issue #17896: The Windows build scripts now expect external library sources - to be in ``PCbuild\..\externals`` rather than ``PCbuild\..\..``. - -- Issue #17717: The Windows build scripts now use a copy of NASM pulled from - svn.python.org to build OpenSSL. - -- Issue #21907: Improved the batch scripts provided for building Python. - -- Issue #22644: The bundled version of OpenSSL has been updated to 1.0.1j. - -- Issue #10747: Use versioned labels in the Windows start menu. - Patch by Olive Kilburn. - -- Issue #22980: .pyd files with a version and platform tag (for example, - ".cp35-win32.pyd") will now be loaded in preference to those without tags. - - -**(For information about older versions, consult the HISTORY file.)** diff --git a/Misc/NEWS.d/3.5.0.rst b/Misc/NEWS.d/3.5.0.rst new file mode 100644 index 00000000000000..130b6781df274f --- /dev/null +++ b/Misc/NEWS.d/3.5.0.rst @@ -0,0 +1,8 @@ +.. bpo: 25071 +.. date: 8265 +.. nonce: EwjXl1 +.. release date: 2015-09-13 +.. section: Build + +Windows installer should not require TargetDir parameter when installing +quietly. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst new file mode 100644 index 00000000000000..97fe42d18fbaf9 --- /dev/null +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -0,0 +1,5869 @@ +.. bpo: 23285 +.. date: 7908 +.. nonce: bJJA8B +.. release date: 2015-02-08 +.. section: Core and Builtins + +PEP 475 - EINTR handling. + +.. + +.. bpo: 22735 +.. date: 7907 +.. nonce: mFEX9n +.. section: Core and Builtins + +Fix many edge cases (including crashes) involving custom mro() +implementations. + +.. + +.. bpo: 22896 +.. date: 7906 +.. nonce: xSDAHK +.. section: Core and Builtins + +Avoid using PyObject_AsCharBuffer(), PyObject_AsReadBuffer() and +PyObject_AsWriteBuffer(). + +.. + +.. bpo: 21295 +.. date: 7905 +.. nonce: LYq9nF +.. section: Core and Builtins + +Revert some changes (issue #16795) to AST line numbers and column offsets +that constituted a regression. + +.. + +.. bpo: 22986 +.. date: 7904 +.. nonce: yay2Lv +.. section: Core and Builtins + +Allow changing an object's __class__ between a dynamic type and static type +in some cases. + +.. + +.. bpo: 15859 +.. date: 7903 +.. nonce: Fs5mE2 +.. section: Core and Builtins + +PyUnicode_EncodeFSDefault(), PyUnicode_EncodeMBCS() and +PyUnicode_EncodeCodePage() now raise an exception if the object is not a +Unicode object. For PyUnicode_EncodeFSDefault(), it was already the case on +platforms other than Windows. Patch written by Campbell Barton. + +.. + +.. bpo: 21408 +.. date: 7902 +.. nonce: Lz6P3P +.. section: Core and Builtins + +The default __ne__() now returns NotImplemented if __eq__() returned +NotImplemented. Original patch by Martin Panter. + +.. + +.. bpo: 23321 +.. date: 7901 +.. nonce: HQelge +.. section: Core and Builtins + +Fixed a crash in str.decode() when error handler returned replacment string +longer than mailformed input data. + +.. + +.. bpo: 22286 +.. date: 7900 +.. nonce: l6Qyy1 +.. section: Core and Builtins + +The "backslashreplace" error handlers now works with decoding and +translating. + +.. + +.. bpo: 23253 +.. date: 7899 +.. nonce: p4B1H- +.. section: Core and Builtins + +Delay-load ShellExecute[AW] in os.startfile for reduced startup overhead on +Windows. + +.. + +.. bpo: 22038 +.. date: 7898 +.. nonce: BMZUHx +.. section: Core and Builtins + +pyatomic.h now uses stdatomic.h or GCC built-in functions for atomic memory +access if available. Patch written by Vitor de Lima and Gustavo Temple. + +.. + +.. bpo: 20284 +.. date: 7897 +.. nonce: CH8wpD +.. section: Core and Builtins + +%-interpolation (aka printf) formatting added for bytes and bytearray. + +.. + +.. bpo: 23048 +.. date: 7896 +.. nonce: X5BUd3 +.. section: Core and Builtins + +Fix jumping out of an infinite while loop in the pdb. + +.. + +.. bpo: 20335 +.. date: 7895 +.. nonce: YcAPOs +.. section: Core and Builtins + +bytes constructor now raises TypeError when encoding or errors is specified +with non-string argument. Based on patch by Renaud Blanch. + +.. + +.. bpo: 22834 +.. date: 7894 +.. nonce: N1kAXN +.. section: Core and Builtins + +If the current working directory ends up being set to a non-existent +directory then import will no longer raise FileNotFoundError. + +.. + +.. bpo: 22869 +.. date: 7893 +.. nonce: rAWg-V +.. section: Core and Builtins + +Move the interpreter startup & shutdown code to a new dedicated +pylifecycle.c module + +.. + +.. bpo: 22847 +.. date: 7892 +.. nonce: 6baj9f +.. section: Core and Builtins + +Improve method cache efficiency. + +.. + +.. bpo: 22335 +.. date: 7891 +.. nonce: DWsXiy +.. section: Core and Builtins + +Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit +platform. + +.. + +.. bpo: 22653 +.. date: 7890 +.. nonce: pCNlpv +.. section: Core and Builtins + +Fix an assertion failure in debug mode when doing a reentrant dict insertion +in debug mode. + +.. + +.. bpo: 22643 +.. date: 7889 +.. nonce: xv8xev +.. section: Core and Builtins + +Fix integer overflow in Unicode case operations (upper, lower, title, +swapcase, casefold). + +.. + +.. bpo: 17636 +.. date: 7888 +.. nonce: wiqnhw +.. section: Core and Builtins + +Circular imports involving relative imports are now supported. + +.. + +.. bpo: 22604 +.. date: 7887 +.. nonce: yii-It +.. section: Core and Builtins + +Fix assertion error in debug mode when dividing a complex number by +(nan+0j). + +.. + +.. bpo: 21052 +.. date: 7886 +.. nonce: -sf3tp +.. section: Core and Builtins + +Do not raise ImportWarning when sys.path_hooks or sys.meta_path are set to +None. + +.. + +.. bpo: 16518 +.. date: 7885 +.. nonce: UADwcN +.. section: Core and Builtins + +Use 'bytes-like object required' in error messages that previously used the +far more cryptic "'x' does not support the buffer protocol. + +.. + +.. bpo: 22470 +.. date: 7884 +.. nonce: igrgN2 +.. section: Core and Builtins + +Fixed integer overflow issues in "backslashreplace", "xmlcharrefreplace", +and "surrogatepass" error handlers. + +.. + +.. bpo: 22540 +.. date: 7883 +.. nonce: FM72m- +.. section: Core and Builtins + +speed up `PyObject_IsInstance` and `PyObject_IsSubclass` in the common case +that the second argument has metaclass `type`. + +.. + +.. bpo: 18711 +.. date: 7882 +.. nonce: ds5wQa +.. section: Core and Builtins + +Add a new `PyErr_FormatV` function, similar to `PyErr_Format` but accepting +a `va_list` argument. + +.. + +.. bpo: 22520 +.. date: 7881 +.. nonce: ZPJXSq +.. section: Core and Builtins + +Fix overflow checking when generating the repr of a unicode object. + +.. + +.. bpo: 22519 +.. date: 7880 +.. nonce: xvJVg0 +.. section: Core and Builtins + +Fix overflow checking in PyBytes_Repr. + +.. + +.. bpo: 22518 +.. date: 7879 +.. nonce: C9T6ed +.. section: Core and Builtins + +Fix integer overflow issues in latin-1 encoding. + +.. + +.. bpo: 16324 +.. date: 7878 +.. nonce: YfrBNz +.. section: Core and Builtins + +_charset parameter of MIMEText now also accepts email.charset.Charset +instances. Initial patch by Claude Paroz. + +.. + +.. bpo: 1764286 +.. date: 7877 +.. nonce: L4seL2 +.. section: Core and Builtins + +Fix inspect.getsource() to support decorated functions. Patch by Claudiu +Popa. + +.. + +.. bpo: 18554 +.. date: 7876 +.. nonce: hxnaui +.. section: Core and Builtins + +os.__all__ includes posix functions. + +.. + +.. bpo: 21391 +.. date: 7875 +.. nonce: 3jntPd +.. section: Core and Builtins + +Use os.path.abspath in the shutil module. + +.. + +.. bpo: 11471 +.. date: 7874 +.. nonce: Uu752F +.. section: Core and Builtins + +avoid generating a JUMP_FORWARD instruction at the end of an if-block if +there is no else-clause. Original patch by Eugene Toder. + +.. + +.. bpo: 22215 +.. date: 7873 +.. nonce: IBFi6H +.. section: Core and Builtins + +Now ValueError is raised instead of TypeError when str or bytes argument +contains not permitted null character or byte. + +.. + +.. bpo: 22258 +.. date: 7872 +.. nonce: 4FszMt +.. section: Core and Builtins + +Fix the internal function set_inheritable() on Illumos. This platform +exposes the function ``ioctl(FIOCLEX)``, but calling it fails with errno is +ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back +to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). + +.. + +.. bpo: 21389 +.. date: 7871 +.. nonce: dnWZBn +.. section: Core and Builtins + +Displaying the __qualname__ of the underlying function in the repr of a +bound method. + +.. + +.. bpo: 22206 +.. date: 7870 +.. nonce: 0i_ihB +.. section: Core and Builtins + +Using pthread, PyThread_create_key() now sets errno to ENOMEM and returns -1 +(error) on integer overflow. + +.. + +.. bpo: 20184 +.. date: 7869 +.. nonce: bb3uHY +.. section: Core and Builtins + +Argument Clinic based signature introspection added for 30 of the builtin +functions. + +.. + +.. bpo: 22116 +.. date: 7868 +.. nonce: auVmIt +.. section: Core and Builtins + +C functions and methods (of the 'builtin_function_or_method' type) can now +be weakref'ed. Patch by Wei Wu. + +.. + +.. bpo: 22077 +.. date: 7867 +.. nonce: KZUDR- +.. section: Core and Builtins + +Improve index error messages for bytearrays, bytes, lists, and tuples by +adding 'or slices'. Added ', not ' for bytearrays. Original patch +by Claudiu Popa. + +.. + +.. bpo: 20179 +.. date: 7866 +.. nonce: Nvhffc +.. section: Core and Builtins + +Apply Argument Clinic to bytes and bytearray. Patch by Tal Einat. + +.. + +.. bpo: 22082 +.. date: 7865 +.. nonce: 6X8Qmg +.. section: Core and Builtins + +Clear interned strings in slotdefs. + +.. + +.. bpo: 0 +.. date: 7864 +.. nonce: tuMnCc +.. section: Core and Builtins + +Upgrade Unicode database to Unicode 7.0.0. + +.. + +.. bpo: 21897 +.. date: 7863 +.. nonce: kiOGHe +.. section: Core and Builtins + +Fix a crash with the f_locals attribute with closure variables when +frame.clear() has been called. + +.. + +.. bpo: 21205 +.. date: 7862 +.. nonce: wZsx1K +.. section: Core and Builtins + +Add a new ``__qualname__`` attribute to generator, the qualified name, and +use it in the representation of a generator (``repr(gen)``). The default +name of the generator (``__name__`` attribute) is now get from the function +instead of the code. Use ``gen.gi_code.co_name`` to get the name of the +code. + +.. + +.. bpo: 21669 +.. date: 7861 +.. nonce: DFDrBA +.. section: Core and Builtins + +With the aid of heuristics in SyntaxError.__init__, the parser now attempts +to generate more meaningful (or at least more search engine friendly) error +messages when "exec" and "print" are used as statements. + +.. + +.. bpo: 21642 +.. date: 7860 +.. nonce: -lWoKz +.. section: Core and Builtins + +In the conditional if-else expression, allow an integer written with no +space between itself and the ``else`` keyword (e.g. ``True if 42else +False``) to be valid syntax. + +.. + +.. bpo: 21523 +.. date: 7859 +.. nonce: f_PPYO +.. section: Core and Builtins + +Fix over-pessimistic computation of the stack effect of some opcodes in the +compiler. This also fixes a quadratic compilation time issue noticeable +when compiling code with a large number of "and" and "or" operators. + +.. + +.. bpo: 21418 +.. date: 7858 +.. nonce: z9jp1_ +.. section: Core and Builtins + +Fix a crash in the builtin function super() when called without argument and +without current frame (ex: embedded Python). + +.. + +.. bpo: 21425 +.. date: 7857 +.. nonce: i3Teb8 +.. section: Core and Builtins + +Fix flushing of standard streams in the interactive interpreter. + +.. + +.. bpo: 21435 +.. date: 7856 +.. nonce: ZojVOT +.. section: Core and Builtins + +In rare cases, when running finalizers on objects in cyclic trash a bad +pointer dereference could occur due to a subtle flaw in internal iteration +logic. + +.. + +.. bpo: 21377 +.. date: 7855 +.. nonce: OawYfl +.. section: Core and Builtins + +PyBytes_Concat() now tries to concatenate in-place when the first argument +has a reference count of 1. Patch by Nikolaus Rath. + +.. + +.. bpo: 20355 +.. date: 7854 +.. nonce: OrCNkZ +.. section: Core and Builtins + +-W command line options now have higher priority than the PYTHONWARNINGS +environment variable. Patch by Arfrever. + +.. + +.. bpo: 21274 +.. date: 7853 +.. nonce: fVGfwq +.. section: Core and Builtins + +Define PATH_MAX for GNU/Hurd in Python/pythonrun.c. + +.. + +.. bpo: 20904 +.. date: 7852 +.. nonce: fAGdj2 +.. section: Core and Builtins + +Support setting FPU precision on m68k. + +.. + +.. bpo: 21209 +.. date: 7851 +.. nonce: nMljFr +.. section: Core and Builtins + +Fix sending tuples to custom generator objects with the yield from syntax. + +.. + +.. bpo: 21193 +.. date: 7850 +.. nonce: Dg98Oo +.. section: Core and Builtins + +pow(a, b, c) now raises ValueError rather than TypeError when b is negative. +Patch by Josh Rosenberg. + +.. + +.. bpo: 21176 +.. date: 7849 +.. nonce: mitDhW +.. section: Core and Builtins + +PEP 465: Add the '@' operator for matrix multiplication. + +.. + +.. bpo: 21134 +.. date: 7848 +.. nonce: ZL4SKo +.. section: Core and Builtins + +Fix segfault when str is called on an uninitialized UnicodeEncodeError, +UnicodeDecodeError, or UnicodeTranslateError object. + +.. + +.. bpo: 19537 +.. date: 7847 +.. nonce: AkuC_J +.. section: Core and Builtins + +Fix PyUnicode_DATA() alignment under m68k. Patch by Andreas Schwab. + +.. + +.. bpo: 20929 +.. date: 7846 +.. nonce: 9NlUR7 +.. section: Core and Builtins + +Add a type cast to avoid shifting a negative number. + +.. + +.. bpo: 20731 +.. date: 7845 +.. nonce: _03SZg +.. section: Core and Builtins + +Properly position in source code files even if they are opened in text mode. +Patch by Serhiy Storchaka. + +.. + +.. bpo: 20637 +.. date: 7844 +.. nonce: ppYU0o +.. section: Core and Builtins + +Key-sharing now also works for instance dictionaries of subclasses. Patch +by Peter Ingebretson. + +.. + +.. bpo: 8297 +.. date: 7843 +.. nonce: _XdGON +.. section: Core and Builtins + +Attributes missing from modules now include the module name in the error +text. Original patch by ysj.ray. + +.. + +.. bpo: 19995 +.. date: 7842 +.. nonce: mnHEzX +.. section: Core and Builtins + +%c, %o, %x, and %X now raise TypeError on non-integer input. + +.. + +.. bpo: 19655 +.. date: 7841 +.. nonce: JgVdes +.. section: Core and Builtins + +The ASDL parser - used by the build process to generate code for managing +the Python AST in C - was rewritten. The new parser is self contained and +does not require to carry long the spark.py parser-generator library; +spark.py was removed from the source base. + +.. + +.. bpo: 12546 +.. date: 7840 +.. nonce: 09naZ9 +.. section: Core and Builtins + +Allow ``\x00`` to be used as a fill character when using str, int, float, +and complex __format__ methods. + +.. + +.. bpo: 20480 +.. date: 7839 +.. nonce: TIYPLo +.. section: Core and Builtins + +Add ipaddress.reverse_pointer. Patch by Leon Weber. + +.. + +.. bpo: 13598 +.. date: 7838 +.. nonce: GJelrw +.. section: Core and Builtins + +Modify string.Formatter to support auto-numbering of replacement fields. It +now matches the behavior of str.format() in this regard. Patches by Phil +Elson and Ramchandra Apte. + +.. + +.. bpo: 8931 +.. date: 7837 +.. nonce: M05x4f +.. section: Core and Builtins + +Make alternate formatting ('#') for type 'c' raise an exception. In versions +prior to 3.5, '#' with 'c' had no effect. Now specifying it is an error. +Patch by Torsten Landschoff. + +.. + +.. bpo: 23165 +.. date: 7836 +.. nonce: lk8uCE +.. section: Core and Builtins + +Perform overflow checks before allocating memory in the _Py_char2wchar +function. + +.. + +.. bpo: 23399 +.. date: 7835 +.. nonce: hXMYgA +.. section: Library + +pyvenv creates relative symlinks where possible. + +.. + +.. bpo: 20289 +.. date: 7834 +.. nonce: nio1N- +.. section: Library + +cgi.FieldStorage() now supports the context management protocol. + +.. + +.. bpo: 13128 +.. date: 7833 +.. nonce: vqEcsy +.. section: Library + +Print response headers for CONNECT requests when debuglevel > 0. Patch by +Demian Brecht. + +.. + +.. bpo: 15381 +.. date: 7832 +.. nonce: Xv-wu8 +.. section: Library + +Optimized io.BytesIO to make less allocations and copyings. + +.. + +.. bpo: 22818 +.. date: 7831 +.. nonce: NYdAc9 +.. section: Library + +Splitting on a pattern that could match an empty string now raises a +warning. Patterns that can only match empty strings are now rejected. + +.. + +.. bpo: 23099 +.. date: 7830 +.. nonce: ZASrUo +.. section: Library + +Closing io.BytesIO with exported buffer is rejected now to prevent +corrupting exported buffer. + +.. + +.. bpo: 23326 +.. date: 7829 +.. nonce: 8VzlZD +.. section: Library + +Removed __ne__ implementations. Since fixing default __ne__ implementation +in issue #21408 they are redundant. + +.. + +.. bpo: 23363 +.. date: 7828 +.. nonce: -koaol +.. section: Library + +Fix possible overflow in itertools.permutations. + +.. + +.. bpo: 23364 +.. date: 7827 +.. nonce: 3yBV-6 +.. section: Library + +Fix possible overflow in itertools.product. + +.. + +.. bpo: 23366 +.. date: 7826 +.. nonce: tyAfm8 +.. section: Library + +Fixed possible integer overflow in itertools.combinations. + +.. + +.. bpo: 23369 +.. date: 7825 +.. nonce: nqChyE +.. section: Library + +Fixed possible integer overflow in _json.encode_basestring_ascii. + +.. + +.. bpo: 23353 +.. date: 7824 +.. nonce: Iytkpc +.. section: Library + +Fix the exception handling of generators in PyEval_EvalFrameEx(). At entry, +save or swap the exception state even if PyEval_EvalFrameEx() is called with +throwflag=0. At exit, the exception state is now always restored or swapped, +not only if why is WHY_YIELD or WHY_RETURN. Patch co-written with Antoine +Pitrou. + +.. + +.. bpo: 14099 +.. date: 7823 +.. nonce: t9-HVE +.. section: Library + +Restored support of writing ZIP files to tellable but non-seekable streams. + +.. + +.. bpo: 14099 +.. date: 7822 +.. nonce: Myxxww +.. section: Library + +Writing to ZipFile and reading multiple ZipExtFiles is threadsafe now. + +.. + +.. bpo: 19361 +.. date: 7821 +.. nonce: 2mvrV3 +.. section: Library + +JSON decoder now raises JSONDecodeError instead of ValueError. + +.. + +.. bpo: 18518 +.. date: 7820 +.. nonce: JXgicC +.. section: Library + +timeit now rejects statements which can't be compiled outside a function or +a loop (e.g. "return" or "break"). + +.. + +.. bpo: 23094 +.. date: 7819 +.. nonce: -8AXSi +.. section: Library + +Fixed readline with frames in Python implementation of pickle. + +.. + +.. bpo: 23268 +.. date: 7818 +.. nonce: ATtRa5 +.. section: Library + +Fixed bugs in the comparison of ipaddress classes. + +.. + +.. bpo: 21408 +.. date: 7817 +.. nonce: 0rI6tx +.. section: Library + +Removed incorrect implementations of __ne__() which didn't returned +NotImplemented if __eq__() returned NotImplemented. The default __ne__() +now works correctly. + +.. + +.. bpo: 19996 +.. date: 7816 +.. nonce: 2-SiMf +.. section: Library + +:class:`email.feedparser.FeedParser` now handles (malformed) headers with no +key rather than assuming the body has started. + +.. + +.. bpo: 20188 +.. date: 7815 +.. nonce: xocY-2 +.. section: Library + +Support Application-Layer Protocol Negotiation (ALPN) in the ssl module. + +.. + +.. bpo: 23133 +.. date: 7814 +.. nonce: 8p2Wnl +.. section: Library + +Pickling of ipaddress objects now produces more compact and portable +representation. + +.. + +.. bpo: 23248 +.. date: 7813 +.. nonce: FjcyCP +.. section: Library + +Update ssl error codes from latest OpenSSL git master. + +.. + +.. bpo: 23266 +.. date: 7812 +.. nonce: Mo7alR +.. section: Library + +Much faster implementation of ipaddress.collapse_addresses() when there are +many non-consecutive addresses. + +.. + +.. bpo: 23098 +.. date: 7811 +.. nonce: 7VwF3K +.. section: Library + +64-bit dev_t is now supported in the os module. + +.. + +.. bpo: 21817 +.. date: 7810 +.. nonce: xYUW-9 +.. section: Library + +When an exception is raised in a task submitted to a ProcessPoolExecutor, +the remote traceback is now displayed in the parent process. Patch by +Claudiu Popa. + +.. + +.. bpo: 15955 +.. date: 7809 +.. nonce: uvpBL4 +.. section: Library + +Add an option to limit output size when decompressing LZMA data. Patch by +Nikolaus Rath and Martin Panter. + +.. + +.. bpo: 23250 +.. date: 7808 +.. nonce: qNGAUf +.. section: Library + +In the http.cookies module, capitalize "HttpOnly" and "Secure" as they are +written in the standard. + +.. + +.. bpo: 23063 +.. date: 7807 +.. nonce: 9-UJRs +.. section: Library + +In the disutils' check command, fix parsing of reST with code or code-block +directives. + +.. + +.. bpo: 23209 +.. date: 7806 +.. nonce: I0bCCH +.. section: Library + +selectors.BaseSelector.get_key() now raises a RuntimeError if the selector +is closed. And selectors.BaseSelector.close() now clears its internal +reference to the selector mapping to break a reference cycle. Initial patch +written by Martin Richard. (See also: bpo-23225) + +.. + +.. bpo: 17911 +.. date: 7805 +.. nonce: yg65Iu +.. section: Library + +Provide a way to seed the linecache for a PEP-302 module without actually +loading the code. + +.. + +.. bpo: 17911 +.. date: 7804 +.. nonce: qeTePa +.. section: Library + +Provide a new object API for traceback, including the ability to not lookup +lines at all until the traceback is actually rendered, without any trace of +the original objects being kept alive. + +.. + +.. bpo: 19777 +.. date: 7803 +.. nonce: H_NDIA +.. section: Library + +Provide a home() classmethod on Path objects. Contributed by Victor Salgado +and Mayank Tripathi. + +.. + +.. bpo: 23206 +.. date: 7802 +.. nonce: xSiYwq +.. section: Library + +Make ``json.dumps(..., ensure_ascii=False)`` as fast as the default case of +``ensure_ascii=True``. Patch by Naoki Inada. + +.. + +.. bpo: 23185 +.. date: 7801 +.. nonce: KHyoSO +.. section: Library + +Add math.inf and math.nan constants. + +.. + +.. bpo: 23186 +.. date: 7800 +.. nonce: KzWLP2 +.. section: Library + +Add ssl.SSLObject.shared_ciphers() and ssl.SSLSocket.shared_ciphers() to +fetch the client's list ciphers sent at handshake. + +.. + +.. bpo: 23143 +.. date: 7799 +.. nonce: AWxJXV +.. section: Library + +Remove compatibility with OpenSSLs older than 0.9.8. + +.. + +.. bpo: 23132 +.. date: 7798 +.. nonce: pbQcut +.. section: Library + +Improve performance and introspection support of comparison methods created +by functool.total_ordering. + +.. + +.. bpo: 19776 +.. date: 7797 +.. nonce: BxNgxd +.. section: Library + +Add an expanduser() method on Path objects. + +.. + +.. bpo: 23112 +.. date: 7796 +.. nonce: dZGf82 +.. section: Library + +Fix SimpleHTTPServer to correctly carry the query string and fragment when +it redirects to add a trailing slash. + +.. + +.. bpo: 21793 +.. date: 7795 +.. nonce: T1kQBL +.. section: Library + +Added http.HTTPStatus enums (i.e. HTTPStatus.OK, HTTPStatus.NOT_FOUND). +Patch by Demian Brecht. + +.. + +.. bpo: 23093 +.. date: 7794 +.. nonce: cP7OqD +.. section: Library + +In the io, module allow more operations to work on detached streams. + +.. + +.. bpo: 23111 +.. date: 7793 +.. nonce: A34IA4 +.. section: Library + +In the ftplib, make ssl.PROTOCOL_SSLv23 the default protocol version. + +.. + +.. bpo: 22585 +.. date: 7792 +.. nonce: F4BkNo +.. section: Library + +On OpenBSD 5.6 and newer, os.urandom() now calls getentropy(), instead of +reading /dev/urandom, to get pseudo-random bytes. + +.. + +.. bpo: 19104 +.. date: 7791 +.. nonce: _eIThy +.. section: Library + +pprint now produces evaluable output for wrapped strings. + +.. + +.. bpo: 23071 +.. date: 7790 +.. nonce: 3BSqF7 +.. section: Library + +Added missing names to codecs.__all__. Patch by Martin Panter. + +.. + +.. bpo: 22783 +.. date: 7789 +.. nonce: OfYxBd +.. section: Library + +Pickling now uses the NEWOBJ opcode instead of the NEWOBJ_EX opcode if +possible. + +.. + +.. bpo: 15513 +.. date: 7788 +.. nonce: 7yVnRE +.. section: Library + +Added a __sizeof__ implementation for pickle classes. + +.. + +.. bpo: 19858 +.. date: 7787 +.. nonce: cqOlIt +.. section: Library + +pickletools.optimize() now aware of the MEMOIZE opcode, can produce more +compact result and no longer produces invalid output if input data contains +MEMOIZE opcodes together with PUT or BINPUT opcodes. + +.. + +.. bpo: 22095 +.. date: 7786 +.. nonce: iISzxM +.. section: Library + +Fixed HTTPConnection.set_tunnel with default port. The port value in the +host header was set to "None". Patch by Demian Brecht. + +.. + +.. bpo: 23016 +.. date: 7785 +.. nonce: LyrPd_ +.. section: Library + +A warning no longer produces an AttributeError when the program is run with +pythonw.exe. + +.. + +.. bpo: 21775 +.. date: 7784 +.. nonce: ELR_Al +.. section: Library + +shutil.copytree(): fix crash when copying to VFAT. An exception handler +assumed that OSError objects always have a 'winerror' attribute. That is not +the case, so the exception handler itself raised AttributeError when run on +Linux (and, presumably, any other non-Windows OS). Patch by Greg Ward. + +.. + +.. bpo: 1218234 +.. date: 7783 +.. nonce: 4GcoQK +.. section: Library + +Fix inspect.getsource() to load updated source of reloaded module. Initial +patch by Berker Peksag. + +.. + +.. bpo: 21740 +.. date: 7782 +.. nonce: TtAApO +.. section: Library + +Support wrapped callables in doctest. Patch by Claudiu Popa. + +.. + +.. bpo: 23009 +.. date: 7781 +.. nonce: -sW7gk +.. section: Library + +Make sure selectors.EpollSelecrtor.select() works when no FD is registered. + +.. + +.. bpo: 22959 +.. date: 7780 +.. nonce: Vxt3EP +.. section: Library + +In the constructor of http.client.HTTPSConnection, prefer the context's +check_hostname attribute over the *check_hostname* parameter. + +.. + +.. bpo: 22696 +.. date: 7779 +.. nonce: pvdcxs +.. section: Library + +Add function :func:`sys.is_finalizing` to know about interpreter shutdown. + +.. + +.. bpo: 16043 +.. date: 7778 +.. nonce: TGIC7t +.. section: Library + +Add a default limit for the amount of data xmlrpclib.gzip_decode will +return. This resolves CVE-2013-1753. + +.. + +.. bpo: 14099 +.. date: 7777 +.. nonce: GJ5meQ +.. section: Library + +ZipFile.open() no longer reopen the underlying file. Objects returned by +ZipFile.open() can now operate independently of the ZipFile even if the +ZipFile was created by passing in a file-like object as the first argument +to the constructor. + +.. + +.. bpo: 22966 +.. date: 7776 +.. nonce: zIxDrT +.. section: Library + +Fix __pycache__ pyc file name clobber when pyc_compile is asked to compile a +source file containing multiple dots in the source file name. + +.. + +.. bpo: 21971 +.. date: 7775 +.. nonce: XlTc22 +.. section: Library + +Update turtledemo doc and add module to the index. + +.. + +.. bpo: 21032 +.. date: 7774 +.. nonce: wxT_41 +.. section: Library + +Fixed socket leak if HTTPConnection.getresponse() fails. Original patch by +Martin Panter. + +.. + +.. bpo: 22407 +.. date: 7773 +.. nonce: CWi1wX +.. section: Library + +Deprecated the use of re.LOCALE flag with str patterns or re.ASCII. It was +newer worked. + +.. + +.. bpo: 22902 +.. date: 7772 +.. nonce: ZqXriA +.. section: Library + +The "ip" command is now used on Linux to determine MAC address in +uuid.getnode(). Pach by Bruno Cauet. + +.. + +.. bpo: 22960 +.. date: 7771 +.. nonce: 2VDILT +.. section: Library + +Add a context argument to xmlrpclib.ServerProxy constructor. + +.. + +.. bpo: 22389 +.. date: 7770 +.. nonce: 82DuwD +.. section: Library + +Add contextlib.redirect_stderr(). + +.. + +.. bpo: 21356 +.. date: 7769 +.. nonce: 8NY75J +.. section: Library + +Make ssl.RAND_egd() optional to support LibreSSL. The availability of the +function is checked during the compilation. Patch written by Bernard Spil. + +.. + +.. bpo: 22915 +.. date: 7768 +.. nonce: 709UAo +.. section: Library + +SAX parser now supports files opened with file descriptor or bytes path. + +.. + +.. bpo: 22609 +.. date: 7767 +.. nonce: mmLoeb +.. section: Library + +Constructors and update methods of mapping classes in the collections module +now accept the self keyword argument. + +.. + +.. bpo: 22940 +.. date: 7766 +.. nonce: SP99Nf +.. section: Library + +Add readline.append_history_file. + +.. + +.. bpo: 19676 +.. date: 7765 +.. nonce: Wijwr8 +.. section: Library + +Added the "namereplace" error handler. + +.. + +.. bpo: 22788 +.. date: 7764 +.. nonce: vofL9e +.. section: Library + +Add *context* parameter to logging.handlers.HTTPHandler. + +.. + +.. bpo: 22921 +.. date: 7763 +.. nonce: a4wx1C +.. section: Library + +Allow SSLContext to take the *hostname* parameter even if OpenSSL doesn't +support SNI. + +.. + +.. bpo: 22894 +.. date: 7762 +.. nonce: 4AkwPA +.. section: Library + +TestCase.subTest() would cause the test suite to be stopped when in failfast +mode, even in the absence of failures. + +.. + +.. bpo: 22796 +.. date: 7761 +.. nonce: _pFPFA +.. section: Library + +HTTP cookie parsing is now stricter, in order to protect against potential +injection attacks. + +.. + +.. bpo: 22370 +.. date: 7760 +.. nonce: j4y21u +.. section: Library + +Windows detection in pathlib is now more robust. + +.. + +.. bpo: 22841 +.. date: 7759 +.. nonce: 8wpk7T +.. section: Library + +Reject coroutines in asyncio add_signal_handler(). Patch by Ludovic.Gasc. + +.. + +.. bpo: 19494 +.. date: 7758 +.. nonce: 7O5O8k +.. section: Library + +Added urllib.request.HTTPBasicPriorAuthHandler. Patch by Matej Cepl. + +.. + +.. bpo: 22578 +.. date: 7757 +.. nonce: 6XZ0Jf +.. section: Library + +Added attributes to the re.error class. + +.. + +.. bpo: 22849 +.. date: 7756 +.. nonce: AqBPyj +.. section: Library + +Fix possible double free in the io.TextIOWrapper constructor. + +.. + +.. bpo: 12728 +.. date: 7755 +.. nonce: rHZmXO +.. section: Library + +Different Unicode characters having the same uppercase but different +lowercase are now matched in case-insensitive regular expressions. + +.. + +.. bpo: 22821 +.. date: 7754 +.. nonce: 30cQ-U +.. section: Library + +Fixed fcntl() with integer argument on 64-bit big-endian platforms. + +.. + +.. bpo: 21650 +.. date: 7753 +.. nonce: 62MLqr +.. section: Library + +Add an `--sort-keys` option to json.tool CLI. + +.. + +.. bpo: 22824 +.. date: 7752 +.. nonce: d5Txvr +.. section: Library + +Updated reprlib output format for sets to use set literals. Patch +contributed by Berker Peksag. + +.. + +.. bpo: 22824 +.. date: 7751 +.. nonce: H_r9uH +.. section: Library + +Updated reprlib output format for arrays to display empty arrays without an +unnecessary empty list. Suggested by Serhiy Storchaka. + +.. + +.. bpo: 22406 +.. date: 7750 +.. nonce: sPlVbI +.. section: Library + +Fixed the uu_codec codec incorrectly ported to 3.x. Based on patch by Martin +Panter. + +.. + +.. bpo: 17293 +.. date: 7749 +.. nonce: Hk06bO +.. section: Library + +uuid.getnode() now determines MAC address on AIX using netstat. Based on +patch by Aivars Kalvāns. + +.. + +.. bpo: 22769 +.. date: 7748 +.. nonce: PunnvQ +.. section: Library + +Fixed ttk.Treeview.tag_has() when called without arguments. + +.. + +.. bpo: 22417 +.. date: 7747 +.. nonce: To4b7U +.. section: Library + +Verify certificates by default in httplib (PEP 476). + +.. + +.. bpo: 22775 +.. date: 7746 +.. nonce: V5aCUz +.. section: Library + +Fixed unpickling of http.cookies.SimpleCookie with protocol 2 and above. +Patch by Tim Graham. + +.. + +.. bpo: 22776 +.. date: 7745 +.. nonce: xNcRse +.. section: Library + +Brought excluded code into the scope of a try block in SysLogHandler.emit(). + +.. + +.. bpo: 22665 +.. date: 7744 +.. nonce: j6Jlp8 +.. section: Library + +Add missing get_terminal_size and SameFileError to shutil.__all__. + +.. + +.. bpo: 6623 +.. date: 7743 +.. nonce: 6LOidS +.. section: Library + +Remove deprecated Netrc class in the ftplib module. Patch by Matt Chaput. + +.. + +.. bpo: 17381 +.. date: 7742 +.. nonce: 4J5yv7 +.. section: Library + +Fixed handling of case-insensitive ranges in regular expressions. + +.. + +.. bpo: 22410 +.. date: 7741 +.. nonce: 99YFdd +.. section: Library + +Module level functions in the re module now cache compiled locale-dependent +regular expressions taking into account the locale. + +.. + +.. bpo: 22759 +.. date: 7740 +.. nonce: BJPdiL +.. section: Library + +Query methods on pathlib.Path() (exists(), is_dir(), etc.) now return False +when the underlying stat call raises NotADirectoryError. + +.. + +.. bpo: 8876 +.. date: 7739 +.. nonce: A83Av4 +.. section: Library + +distutils now falls back to copying files when hard linking doesn't work. +This allows use with special filesystems such as VirtualBox shared folders. + +.. + +.. bpo: 22217 +.. date: 7738 +.. nonce: nXzGur +.. section: Library + +Implemented reprs of classes in the zipfile module. + +.. + +.. bpo: 22457 +.. date: 7737 +.. nonce: Xd2Mk- +.. section: Library + +Honour load_tests in the start_dir of discovery. + +.. + +.. bpo: 18216 +.. date: 7736 +.. nonce: trTZw4 +.. section: Library + +gettext now raises an error when a .mo file has an unsupported major version +number. Patch by Aaron Hill. + +.. + +.. bpo: 13918 +.. date: 7735 +.. nonce: -OnUhD +.. section: Library + +Provide a locale.delocalize() function which can remove locale-specific +number formatting from a string representing a number, without then +converting it to a specific type. Patch by Cédric Krier. + +.. + +.. bpo: 22676 +.. date: 7734 +.. nonce: d2v8QM +.. section: Library + +Make the pickling of global objects which don't have a __module__ attribute +less slow. + +.. + +.. bpo: 18853 +.. date: 7733 +.. nonce: 76DrPD +.. section: Library + +Fixed ResourceWarning in shlex.__nain__. + +.. + +.. bpo: 9351 +.. date: 7732 +.. nonce: u5UI-6 +.. section: Library + +Defaults set with set_defaults on an argparse subparser are no longer +ignored when also set on the parent parser. + +.. + +.. bpo: 7559 +.. date: 7731 +.. nonce: QG35ZP +.. section: Library + +unittest test loading ImportErrors are reported as import errors with their +import exception rather than as attribute errors after the import has +already failed. + +.. + +.. bpo: 19746 +.. date: 7730 +.. nonce: S1dg1K +.. section: Library + +Make it possible to examine the errors from unittest discovery without +executing the test suite. The new `errors` attribute on TestLoader exposes +these non-fatal errors encountered during discovery. + +.. + +.. bpo: 21991 +.. date: 7729 +.. nonce: Mkm0IN +.. section: Library + +Make email.headerregistry's header 'params' attributes be read-only +(MappingProxyType). Previously the dictionary was modifiable but a new one +was created on each access of the attribute. + +.. + +.. bpo: 22638 +.. date: 7728 +.. nonce: Ur73gJ +.. section: Library + +SSLv3 is now disabled throughout the standard library. It can still be +enabled by instantiating a SSLContext manually. + +.. + +.. bpo: 22641 +.. date: 7727 +.. nonce: m0ldtl +.. section: Library + +In asyncio, the default SSL context for client connections is now created +using ssl.create_default_context(), for stronger security. + +.. + +.. bpo: 17401 +.. date: 7726 +.. nonce: SZd19P +.. section: Library + +Include closefd in io.FileIO repr. + +.. + +.. bpo: 21338 +.. date: 7725 +.. nonce: evDyHD +.. section: Library + +Add silent mode for compileall. quiet parameters of compile_{dir, file, +path} functions now have a multilevel value. Also, -q option of the CLI now +have a multilevel value. Patch by Thomas Kluyver. + +.. + +.. bpo: 20152 +.. date: 7724 +.. nonce: 9_o92A +.. section: Library + +Convert the array and cmath modules to Argument Clinic. + +.. + +.. bpo: 18643 +.. date: 7723 +.. nonce: 6Qdc0J +.. section: Library + +Add socket.socketpair() on Windows. + +.. + +.. bpo: 22435 +.. date: 7722 +.. nonce: s2U7Zm +.. section: Library + +Fix a file descriptor leak when socketserver bind fails. + +.. + +.. bpo: 13096 +.. date: 7721 +.. nonce: rsailB +.. section: Library + +Fixed segfault in CTypes POINTER handling of large values. + +.. + +.. bpo: 11694 +.. date: 7720 +.. nonce: JuDrch +.. section: Library + +Raise ConversionError in xdrlib as documented. Patch by Filip Gruszczyński +and Claudiu Popa. + +.. + +.. bpo: 19380 +.. date: 7719 +.. nonce: nqgoRQ +.. section: Library + +Optimized parsing of regular expressions. + +.. + +.. bpo: 1519638 +.. date: 7718 +.. nonce: 2pbuog +.. section: Library + +Now unmatched groups are replaced with empty strings in re.sub() and +re.subn(). + +.. + +.. bpo: 18615 +.. date: 7717 +.. nonce: 65TxnY +.. section: Library + +sndhdr.what/whathdr now return a namedtuple. + +.. + +.. bpo: 22462 +.. date: 7716 +.. nonce: 1h4Kpr +.. section: Library + +Fix pyexpat's creation of a dummy frame to make it appear in exception +tracebacks. + +.. + +.. bpo: 21965 +.. date: 7715 +.. nonce: n_jnXs +.. section: Library + +Add support for in-memory SSL to the ssl module. Patch by Geert Jansen. + +.. + +.. bpo: 21173 +.. date: 7714 +.. nonce: egkbEx +.. section: Library + +Fix len() on a WeakKeyDictionary when .clear() was called with an iterator +alive. + +.. + +.. bpo: 11866 +.. date: 7713 +.. nonce: xrvbIC +.. section: Library + +Eliminated race condition in the computation of names for new threads. + +.. + +.. bpo: 21905 +.. date: 7712 +.. nonce: coKyRo +.. section: Library + +Avoid RuntimeError in pickle.whichmodule() when sys.modules is mutated while +iterating. Patch by Olivier Grisel. + +.. + +.. bpo: 11271 +.. date: 7711 +.. nonce: ZYiJru +.. section: Library + +concurrent.futures.Executor.map() now takes a *chunksize* argument to allow +batching of tasks in child processes and improve performance of +ProcessPoolExecutor. Patch by Dan O'Reilly. + +.. + +.. bpo: 21883 +.. date: 7710 +.. nonce: qpuQu6 +.. section: Library + +os.path.join() and os.path.relpath() now raise a TypeError with more helpful +error message for unsupported or mismatched types of arguments. + +.. + +.. bpo: 22219 +.. date: 7709 +.. nonce: l9Enh9 +.. section: Library + +The zipfile module CLI now adds entries for directories (including empty +directories) in ZIP file. + +.. + +.. bpo: 22449 +.. date: 7708 +.. nonce: nFW_Fl +.. section: Library + +In the ssl.SSLContext.load_default_certs, consult the environmental +variables SSL_CERT_DIR and SSL_CERT_FILE on Windows. + +.. + +.. bpo: 22508 +.. date: 7707 +.. nonce: 2LbnGQ +.. section: Library + +The email.__version__ variable has been removed; the email code is no longer +shipped separately from the stdlib, and __version__ hasn't been updated in +several releases. + +.. + +.. bpo: 20076 +.. date: 7706 +.. nonce: -7OIVB +.. section: Library + +Added non derived UTF-8 aliases to locale aliases table. + +.. + +.. bpo: 20079 +.. date: 7705 +.. nonce: qM949O +.. section: Library + +Added locales supported in glibc 2.18 to locale alias table. + +.. + +.. bpo: 20218 +.. date: 7704 +.. nonce: CMgOyE +.. section: Library + +Added convenience methods read_text/write_text and read_bytes/ write_bytes +to pathlib.Path objects. + +.. + +.. bpo: 22396 +.. date: 7703 +.. nonce: cQSizA +.. section: Library + +On 32-bit AIX platform, don't expose os.posix_fadvise() nor +os.posix_fallocate() because their prototypes in system headers are wrong. + +.. + +.. bpo: 22517 +.. date: 7702 +.. nonce: qT6-aB +.. section: Library + +When an io.BufferedRWPair object is deallocated, clear its weakrefs. + +.. + +.. bpo: 22437 +.. date: 7701 +.. nonce: MRVnmQ +.. section: Library + +Number of capturing groups in regular expression is no longer limited by +100. + +.. + +.. bpo: 17442 +.. date: 7700 +.. nonce: rnc87D +.. section: Library + +InteractiveInterpreter now displays the full chained traceback in its +showtraceback method, to match the built in interactive interpreter. + +.. + +.. bpo: 23392 +.. date: 7699 +.. nonce: Pe7_WK +.. section: Library + +Added tests for marshal C API that works with FILE*. + +.. + +.. bpo: 10510 +.. date: 7698 +.. nonce: N-ntcD +.. section: Library + +distutils register and upload methods now use HTML standards compliant CRLF +line endings. + +.. + +.. bpo: 9850 +.. date: 7697 +.. nonce: D-UnVi +.. section: Library + +Fixed macpath.join() for empty first component. Patch by Oleg Oshmyan. + +.. + +.. bpo: 5309 +.. date: 7696 +.. nonce: pVMmQ8 +.. section: Library + +distutils' build and build_ext commands now accept a ``-j`` option to enable +parallel building of extension modules. + +.. + +.. bpo: 22448 +.. date: 7695 +.. nonce: fAapvE +.. section: Library + +Improve canceled timer handles cleanup to prevent unbound memory usage. +Patch by Joshua Moore-Oliva. + +.. + +.. bpo: 22427 +.. date: 7694 +.. nonce: TZ5S_u +.. section: Library + +TemporaryDirectory no longer attempts to clean up twice when used in the +with statement in generator. + +.. + +.. bpo: 22362 +.. date: 7693 +.. nonce: xIBThN +.. section: Library + +Forbidden ambiguous octal escapes out of range 0-0o377 in regular +expressions. + +.. + +.. bpo: 20912 +.. date: 7692 +.. nonce: cAq3mZ +.. section: Library + +Now directories added to ZIP file have correct Unix and MS-DOS directory +attributes. + +.. + +.. bpo: 21866 +.. date: 7691 +.. nonce: hSc4wM +.. section: Library + +ZipFile.close() no longer writes ZIP64 central directory records if +allowZip64 is false. + +.. + +.. bpo: 22278 +.. date: 7690 +.. nonce: abqBXZ +.. section: Library + +Fix urljoin problem with relative urls, a regression observed after changes +to issue22118 were submitted. + +.. + +.. bpo: 22415 +.. date: 7689 +.. nonce: xJLAvI +.. section: Library + +Fixed debugging output of the GROUPREF_EXISTS opcode in the re module. +Removed trailing spaces in debugging output. + +.. + +.. bpo: 22423 +.. date: 7688 +.. nonce: Rtb4oT +.. section: Library + +Unhandled exception in thread no longer causes unhandled AttributeError when +sys.stderr is None. + +.. + +.. bpo: 21332 +.. date: 7687 +.. nonce: Gwxwlr +.. section: Library + +Ensure that ``bufsize=1`` in subprocess.Popen() selects line buffering, +rather than block buffering. Patch by Akira Li. + +.. + +.. bpo: 21091 +.. date: 7686 +.. nonce: M5hAtT +.. section: Library + +Fix API bug: email.message.EmailMessage.is_attachment is now a method. + +.. + +.. bpo: 21079 +.. date: 7685 +.. nonce: czVcL8 +.. section: Library + +Fix email.message.EmailMessage.is_attachment to return the correct result +when the header has parameters as well as a value. + +.. + +.. bpo: 22247 +.. date: 7684 +.. nonce: sGIpR3 +.. section: Library + +Add NNTPError to nntplib.__all__. + +.. + +.. bpo: 22366 +.. date: 7683 +.. nonce: Dd1eFj +.. section: Library + +urllib.request.urlopen will accept a context object (SSLContext) as an +argument which will then be used for HTTPS connection. Patch by Alex Gaynor. + +.. + +.. bpo: 4180 +.. date: 7682 +.. nonce: QBx0JK +.. section: Library + +The warnings registries are now reset when the filters are modified. + +.. + +.. bpo: 22419 +.. date: 7681 +.. nonce: FqH4aC +.. section: Library + +Limit the length of incoming HTTP request in wsgiref server to 65536 bytes +and send a 414 error code for higher lengths. Patch contributed by Devin +Cook. + +.. + +.. bpo: 0 +.. date: 7680 +.. nonce: y7r3O2 +.. section: Library + +Lax cookie parsing in http.cookies could be a security issue when combined +with non-standard cookie handling in some Web browsers. Reported by Sergey +Bobrov. + +.. + +.. bpo: 20537 +.. date: 7679 +.. nonce: E0CE54 +.. section: Library + +logging methods now accept an exception instance as well as a Boolean value +or exception tuple. Thanks to Yury Selivanov for the patch. + +.. + +.. bpo: 22384 +.. date: 7678 +.. nonce: -Nl4He +.. section: Library + +An exception in Tkinter callback no longer crashes the program when it is +run with pythonw.exe. + +.. + +.. bpo: 22168 +.. date: 7677 +.. nonce: vLeKWC +.. section: Library + +Prevent turtle AttributeError with non-default Canvas on OS X. + +.. + +.. bpo: 21147 +.. date: 7676 +.. nonce: w9DE17 +.. section: Library + +sqlite3 now raises an exception if the request contains a null character +instead of truncating it. Based on patch by Victor Stinner. + +.. + +.. bpo: 13968 +.. date: 7675 +.. nonce: 1okGqm +.. section: Library + +The glob module now supports recursive search in subdirectories using the +``**`` pattern. + +.. + +.. bpo: 21951 +.. date: 7674 +.. nonce: 3vS4LK +.. section: Library + +Fixed a crash in Tkinter on AIX when called Tcl command with empty string or +tuple argument. + +.. + +.. bpo: 21951 +.. date: 7673 +.. nonce: _CCC4v +.. section: Library + +Tkinter now most likely raises MemoryError instead of crash if the memory +allocation fails. + +.. + +.. bpo: 22338 +.. date: 7672 +.. nonce: rKlCMz +.. section: Library + +Fix a crash in the json module on memory allocation failure. + +.. + +.. bpo: 12410 +.. date: 7671 +.. nonce: oFf-cB +.. section: Library + +imaplib.IMAP4 now supports the context management protocol. Original patch +by Tarek Ziadé. + +.. + +.. bpo: 21270 +.. date: 7670 +.. nonce: qMBaY- +.. section: Library + +We now override tuple methods in mock.call objects so that they can be used +as normal call attributes. + +.. + +.. bpo: 16662 +.. date: 7669 +.. nonce: Nghn-Y +.. section: Library + +load_tests() is now unconditionally run when it is present in a package's +__init__.py. TestLoader.loadTestsFromModule() still accepts use_load_tests, +but it is deprecated and ignored. A new keyword-only attribute `pattern` is +added and documented. Patch given by Robert Collins, tweaked by Barry +Warsaw. + +.. + +.. bpo: 22226 +.. date: 7668 +.. nonce: T1ZMPY +.. section: Library + +First letter no longer is stripped from the "status" key in the result of +Treeview.heading(). + +.. + +.. bpo: 19524 +.. date: 7667 +.. nonce: EQJjlF +.. section: Library + +Fixed resource leak in the HTTP connection when an invalid response is +received. Patch by Martin Panter. + +.. + +.. bpo: 20421 +.. date: 7666 +.. nonce: iR0S1s +.. section: Library + +Add a .version() method to SSL sockets exposing the actual protocol version +in use. + +.. + +.. bpo: 19546 +.. date: 7665 +.. nonce: 8VdYBK +.. section: Library + +configparser exceptions no longer expose implementation details. Chained +KeyErrors are removed, which leads to cleaner tracebacks. Patch by Claudiu +Popa. + +.. + +.. bpo: 22051 +.. date: 7664 +.. nonce: cUjFqL +.. section: Library + +turtledemo no longer reloads examples to re-run them. Initialization of +variables and gui setup should be done in main(), which is called each time +a demo is run, but not on import. + +.. + +.. bpo: 21933 +.. date: 7663 +.. nonce: IhMjN1 +.. section: Library + +Turtledemo users can change the code font size with a menu selection or +control(command) '-' or '+' or control-mousewheel. Original patch by Lita +Cho. + +.. + +.. bpo: 21597 +.. date: 7662 +.. nonce: aPTCWJ +.. section: Library + +The separator between the turtledemo text pane and the drawing canvas can +now be grabbed and dragged with a mouse. The code text pane can be widened +to easily view or copy the full width of the text. The canvas can be +widened on small screens. Original patches by Jan Kanis and Lita Cho. + +.. + +.. bpo: 18132 +.. date: 7661 +.. nonce: 2R2nwM +.. section: Library + +Turtledemo buttons no longer disappear when the window is shrunk. Original +patches by Jan Kanis and Lita Cho. + +.. + +.. bpo: 22043 +.. date: 7660 +.. nonce: Q6RvGL +.. section: Library + +time.monotonic() is now always available. ``threading.Lock.acquire()``, +``threading.RLock.acquire()`` and socket operations now use a monotonic +clock, instead of the system clock, when a timeout is used. + +.. + +.. bpo: 21527 +.. date: 7659 +.. nonce: N5WPxr +.. section: Library + +Add a default number of workers to ThreadPoolExecutor equal to 5 times the +number of CPUs. Patch by Claudiu Popa. + +.. + +.. bpo: 22216 +.. date: 7658 +.. nonce: Cmalu6 +.. section: Library + +smtplib now resets its state more completely after a quit. The most obvious +consequence of the previous behavior was a STARTTLS failure during a +connect/starttls/quit/connect/starttls sequence. + +.. + +.. bpo: 22098 +.. date: 7657 +.. nonce: 5JYiQN +.. section: Library + +ctypes' BigEndianStructure and LittleEndianStructure now define an empty +__slots__ so that subclasses don't always get an instance dict. Patch by +Claudiu Popa. + +.. + +.. bpo: 22185 +.. date: 7656 +.. nonce: 1SCCIK +.. section: Library + +Fix an occasional RuntimeError in threading.Condition.wait() caused by +mutation of the waiters queue without holding the lock. Patch by Doug +Zongker. + +.. + +.. bpo: 22287 +.. date: 7655 +.. nonce: awH2AI +.. section: Library + +On UNIX, _PyTime_gettimeofday() now uses clock_gettime(CLOCK_REALTIME) if +available. As a side effect, Python now depends on the librt library on +Solaris and on Linux (only with glibc older than 2.17). + +.. + +.. bpo: 22182 +.. date: 7654 +.. nonce: 5EG1Bc +.. section: Library + +Use e.args to unpack exceptions correctly in distutils.file_util.move_file. +Patch by Claudiu Popa. + +.. + +.. bpo: 0 +.. date: 7653 +.. nonce: zBfe8J +.. section: Library + +The webbrowser module now uses subprocess's start_new_session=True rather +than a potentially risky preexec_fn=os.setsid call. + +.. + +.. bpo: 22042 +.. date: 7652 +.. nonce: WZvb8s +.. section: Library + +signal.set_wakeup_fd(fd) now raises an exception if the file descriptor is +in blocking mode. + +.. + +.. bpo: 16808 +.. date: 7651 +.. nonce: kPy_5U +.. section: Library + +inspect.stack() now returns a named tuple instead of a tuple. Patch by +Daniel Shahaf. + +.. + +.. bpo: 22236 +.. date: 7650 +.. nonce: 1utXkg +.. section: Library + +Fixed Tkinter images copying operations in NoDefaultRoot mode. + +.. + +.. bpo: 2527 +.. date: 7649 +.. nonce: fR2GS6 +.. section: Library + +Add a *globals* argument to timeit functions, in order to override the +globals namespace in which the timed code is executed. Patch by Ben Roberts. + +.. + +.. bpo: 22118 +.. date: 7648 +.. nonce: 3gdkOF +.. section: Library + +Switch urllib.parse to use RFC 3986 semantics for the resolution of relative +URLs, rather than RFCs 1808 and 2396. Patch by Demian Brecht. + +.. + +.. bpo: 21549 +.. date: 7647 +.. nonce: i1LVvg +.. section: Library + +Added the "members" parameter to TarFile.list(). + +.. + +.. bpo: 19628 +.. date: 7646 +.. nonce: ssQVP8 +.. section: Library + +Allow compileall recursion depth to be specified with a -r option. + +.. + +.. bpo: 15696 +.. date: 7645 +.. nonce: PTwXYJ +.. section: Library + +Add a __sizeof__ implementation for mmap objects on Windows. + +.. + +.. bpo: 22068 +.. date: 7644 +.. nonce: wCdaW0 +.. section: Library + +Avoided reference loops with Variables and Fonts in Tkinter. + +.. + +.. bpo: 22165 +.. date: 7643 +.. nonce: J1np4o +.. section: Library + +SimpleHTTPRequestHandler now supports undecodable file names. + +.. + +.. bpo: 15381 +.. date: 7642 +.. nonce: Ia8pf6 +.. section: Library + +Optimized line reading in io.BytesIO. + +.. + +.. bpo: 8797 +.. date: 7641 +.. nonce: aJcIPu +.. section: Library + +Raise HTTPError on failed Basic Authentication immediately. Initial patch by +Sam Bull. + +.. + +.. bpo: 20729 +.. date: 7640 +.. nonce: I-1Lap +.. section: Library + +Restored the use of lazy iterkeys()/itervalues()/iteritems() in the mailbox +module. + +.. + +.. bpo: 21448 +.. date: 7639 +.. nonce: THJSYB +.. section: Library + +Changed FeedParser feed() to avoid O(N**2) behavior when parsing long line. +Original patch by Raymond Hettinger. + +.. + +.. bpo: 22184 +.. date: 7638 +.. nonce: UCbSOt +.. section: Library + +The functools LRU Cache decorator factory now gives an earlier and clearer +error message when the user forgets the required parameters. + +.. + +.. bpo: 17923 +.. date: 7637 +.. nonce: YI_QjG +.. section: Library + +glob() patterns ending with a slash no longer match non-dirs on AIX. Based +on patch by Delhallt. + +.. + +.. bpo: 21725 +.. date: 7636 +.. nonce: eIu-2N +.. section: Library + +Added support for RFC 6531 (SMTPUTF8) in smtpd. + +.. + +.. bpo: 22176 +.. date: 7635 +.. nonce: rgbRyg +.. section: Library + +Update the ctypes module's libffi to v3.1. This release adds support for +the Linux AArch64 and POWERPC ELF ABIv2 little endian architectures. + +.. + +.. bpo: 5411 +.. date: 7634 +.. nonce: 5Utapn +.. section: Library + +Added support for the "xztar" format in the shutil module. + +.. + +.. bpo: 21121 +.. date: 7633 +.. nonce: ZLsRil +.. section: Library + +Don't force 3rd party C extensions to be built with -Werror=declaration- +after-statement. + +.. + +.. bpo: 21975 +.. date: 7632 +.. nonce: MI8ntO +.. section: Library + +Fixed crash when using uninitialized sqlite3.Row (in particular when +unpickling pickled sqlite3.Row). sqlite3.Row is now initialized in the +__new__() method. + +.. + +.. bpo: 20170 +.. date: 7631 +.. nonce: 8QfhN7 +.. section: Library + +Convert posixmodule to use Argument Clinic. + +.. + +.. bpo: 21539 +.. date: 7630 +.. nonce: YccmZF +.. section: Library + +Add an *exists_ok* argument to `Pathlib.mkdir()` to mimic `mkdir -p` and +`os.makedirs()` functionality. When true, ignore FileExistsErrors. Patch +by Berker Peksag. + +.. + +.. bpo: 22127 +.. date: 7629 +.. nonce: 0l2OO5 +.. section: Library + +Bypass IDNA for pure-ASCII host names in the socket module (in particular +for numeric IPs). + +.. + +.. bpo: 21047 +.. date: 7628 +.. nonce: XfUQG3 +.. section: Library + +set the default value for the *convert_charrefs* argument of HTMLParser to +True. Patch by Berker Peksag. + +.. + +.. bpo: 0 +.. date: 7627 +.. nonce: 56bAnQ +.. section: Library + +Add an __all__ to html.entities. + +.. + +.. bpo: 15114 +.. date: 7626 +.. nonce: jXwseC +.. section: Library + +the strict mode and argument of HTMLParser, HTMLParser.error, and the +HTMLParserError exception have been removed. + +.. + +.. bpo: 22085 +.. date: 7625 +.. nonce: 3JM_Aw +.. section: Library + +Dropped support of Tk 8.3 in Tkinter. + +.. + +.. bpo: 21580 +.. date: 7624 +.. nonce: 3ssycS +.. section: Library + +Now Tkinter correctly handles bytes arguments passed to Tk. In particular +this allows initializing images from binary data. + +.. + +.. bpo: 22003 +.. date: 7623 +.. nonce: 4ZIDS1 +.. section: Library + +When initialized from a bytes object, io.BytesIO() now defers making a copy +until it is mutated, improving performance and memory use on some use cases. +Patch by David Wilson. + +.. + +.. bpo: 22018 +.. date: 7622 +.. nonce: 6ApxSH +.. section: Library + +On Windows, signal.set_wakeup_fd() now also supports sockets. A side effect +is that Python depends to the WinSock library. + +.. + +.. bpo: 22054 +.. date: 7621 +.. nonce: zp6Svw +.. section: Library + +Add os.get_blocking() and os.set_blocking() functions to get and set the +blocking mode of a file descriptor (False if the O_NONBLOCK flag is set, +True otherwise). These functions are not available on Windows. + +.. + +.. bpo: 17172 +.. date: 7620 +.. nonce: R_LI_2 +.. section: Library + +Make turtledemo start as active on OS X even when run with subprocess. +Patch by Lita Cho. + +.. + +.. bpo: 21704 +.. date: 7619 +.. nonce: gL3ikj +.. section: Library + +Fix build error for _multiprocessing when semaphores are not available. +Patch by Arfrever Frehtes Taifersar Arahesis. + +.. + +.. bpo: 20173 +.. date: 7618 +.. nonce: FAL-4L +.. section: Library + +Convert sha1, sha256, sha512 and md5 to ArgumentClinic. Patch by Vajrasky +Kok. + +.. + +.. bpo: 0 +.. date: 7617 +.. nonce: G25tq3 +.. section: Library + +Fix repr(_socket.socket) on Windows 64-bit: don't fail with OverflowError on +closed socket. repr(socket.socket) already works fine. + +.. + +.. bpo: 22033 +.. date: 7616 +.. nonce: nkBNci +.. section: Library + +Reprs of most Python implemened classes now contain actual class name +instead of hardcoded one. + +.. + +.. bpo: 21947 +.. date: 7615 +.. nonce: mlisu- +.. section: Library + +The dis module can now disassemble generator-iterator objects based on their +gi_code attribute. Patch by Clement Rouault. + +.. + +.. bpo: 16133 +.. date: 7614 +.. nonce: tYuYQF +.. section: Library + +The asynchat.async_chat.handle_read() method now ignores BlockingIOError +exceptions. + +.. + +.. bpo: 22044 +.. date: 7613 +.. nonce: t09GRU +.. section: Library + +Fixed premature DECREF in call_tzinfo_method. Patch by Tom Flanagan. + +.. + +.. bpo: 19884 +.. date: 7612 +.. nonce: v73gSn +.. section: Library + +readline: Disable the meta modifier key if stdout is not a terminal to not +write the ANSI sequence ``"\033[1034h"`` into stdout. This sequence is used +on some terminal (ex: TERM=xterm-256color") to enable support of 8 bit +characters. + +.. + +.. bpo: 4350 +.. date: 7611 +.. nonce: nrTzJn +.. section: Library + +Removed a number of out-of-dated and non-working for a long time Tkinter +methods. + +.. + +.. bpo: 6167 +.. date: 7610 +.. nonce: n9dV_D +.. section: Library + +Scrollbar.activate() now returns the name of active element if the argument +is not specified. Scrollbar.set() now always accepts only 2 arguments. + +.. + +.. bpo: 15275 +.. date: 7609 +.. nonce: jk0tTI +.. section: Library + +Clean up and speed up the ntpath module. + +.. + +.. bpo: 21888 +.. date: 7608 +.. nonce: danlpz +.. section: Library + +plistlib's load() and loads() now work if the fmt parameter is specified. + +.. + +.. bpo: 22032 +.. date: 7607 +.. nonce: UklzQW +.. section: Library + +__qualname__ instead of __name__ is now always used to format fully +qualified class names of Python implemented classes. + +.. + +.. bpo: 22031 +.. date: 7606 +.. nonce: 9aazp1 +.. section: Library + +Reprs now always use hexadecimal format with the "0x" prefix when contain an +id in form " at 0x...". + +.. + +.. bpo: 22018 +.. date: 7605 +.. nonce: b_JTHH +.. section: Library + +signal.set_wakeup_fd() now raises an OSError instead of a ValueError on +``fstat()`` failure. + +.. + +.. bpo: 21044 +.. date: 7604 +.. nonce: 16xo9u +.. section: Library + +tarfile.open() now handles fileobj with an integer 'name' attribute. Based +on patch by Antoine Pietri. + +.. + +.. bpo: 21966 +.. date: 7603 +.. nonce: hHD9MK +.. section: Library + +Respect -q command-line option when code module is ran. + +.. + +.. bpo: 19076 +.. date: 7602 +.. nonce: xCoIai +.. section: Library + +Don't pass the redundant 'file' argument to self.error(). + +.. + +.. bpo: 16382 +.. date: 7601 +.. nonce: -XBK7z +.. section: Library + +Improve exception message of warnings.warn() for bad category. Initial patch +by Phil Elson. + +.. + +.. bpo: 21932 +.. date: 7600 +.. nonce: LK_5S1 +.. section: Library + +os.read() now uses a :c:func:`Py_ssize_t` type instead of :c:type:`int` for +the size to support reading more than 2 GB at once. On Windows, the size is +truncted to INT_MAX. As any call to os.read(), the OS may read less bytes +than the number of requested bytes. + +.. + +.. bpo: 21942 +.. date: 7599 +.. nonce: TLOS41 +.. section: Library + +Fixed source file viewing in pydoc's server mode on Windows. + +.. + +.. bpo: 11259 +.. date: 7598 +.. nonce: GxfYnE +.. section: Library + +asynchat.async_chat().set_terminator() now raises a ValueError if the number +of received bytes is negative. + +.. + +.. bpo: 12523 +.. date: 7597 +.. nonce: XBdAky +.. section: Library + +asynchat.async_chat.push() now raises a TypeError if it doesn't get a bytes +string + +.. + +.. bpo: 21707 +.. date: 7596 +.. nonce: rrY_wd +.. section: Library + +Add missing kwonlyargcount argument to ModuleFinder.replace_paths_in_code(). + +.. + +.. bpo: 20639 +.. date: 7595 +.. nonce: YdvOpp +.. section: Library + +calling Path.with_suffix('') allows removing the suffix again. Patch by +July Tikhonov. + +.. + +.. bpo: 21714 +.. date: 7594 +.. nonce: HhkGXW +.. section: Library + +Disallow the construction of invalid paths using Path.with_name(). Original +patch by Antony Lee. + +.. + +.. bpo: 15014 +.. date: 7593 +.. nonce: dB50zb +.. section: Library + +Added 'auth' method to smtplib to make implementing auth mechanisms simpler, +and used it internally in the login method. + +.. + +.. bpo: 21151 +.. date: 7592 +.. nonce: o7IuiD +.. section: Library + +Fixed a segfault in the winreg module when ``None`` is passed as a +``REG_BINARY`` value to SetValueEx. Patch by John Ehresman. + +.. + +.. bpo: 21090 +.. date: 7591 +.. nonce: 20Ooif +.. section: Library + +io.FileIO.readall() does not ignore I/O errors anymore. Before, it ignored +I/O errors if at least the first C call read() succeed. + +.. + +.. bpo: 5800 +.. date: 7590 +.. nonce: ZJiLZP +.. section: Library + +headers parameter of wsgiref.headers.Headers is now optional. Initial patch +by Pablo Torres Navarrete and SilentGhost. + +.. + +.. bpo: 21781 +.. date: 7589 +.. nonce: u_oiv9 +.. section: Library + +ssl.RAND_add() now supports strings longer than 2 GB. + +.. + +.. bpo: 21679 +.. date: 7588 +.. nonce: CTVT9A +.. section: Library + +Prevent extraneous fstat() calls during open(). Patch by Bohuslav Kabrda. + +.. + +.. bpo: 21863 +.. date: 7587 +.. nonce: BzbwSL +.. section: Library + +cProfile now displays the module name of C extension functions, in addition +to their own name. + +.. + +.. bpo: 11453 +.. date: 7586 +.. nonce: 53Gr_R +.. section: Library + +asyncore: emit a ResourceWarning when an unclosed file_wrapper object is +destroyed. The destructor now closes the file if needed. The close() method +can now be called twice: the second call does nothing. + +.. + +.. bpo: 21858 +.. date: 7585 +.. nonce: 0hbFBG +.. section: Library + +Better handling of Python exceptions in the sqlite3 module. + +.. + +.. bpo: 21476 +.. date: 7584 +.. nonce: VN-5pW +.. section: Library + +Make sure the email.parser.BytesParser TextIOWrapper is discarded after +parsing, so the input file isn't unexpectedly closed. + +.. + +.. bpo: 20295 +.. date: 7583 +.. nonce: U1MPhw +.. section: Library + +imghdr now recognizes OpenEXR format images. + +.. + +.. bpo: 21729 +.. date: 7582 +.. nonce: dk7o_U +.. section: Library + +Used the "with" statement in the dbm.dumb module to ensure files closing. +Patch by Claudiu Popa. + +.. + +.. bpo: 21491 +.. date: 7581 +.. nonce: Zxmut- +.. section: Library + +socketserver: Fix a race condition in child processes reaping. + +.. + +.. bpo: 21719 +.. date: 7580 +.. nonce: DhQz3I +.. section: Library + +Added the ``st_file_attributes`` field to os.stat_result on Windows. + +.. + +.. bpo: 21832 +.. date: 7579 +.. nonce: PBA0Uu +.. section: Library + +Require named tuple inputs to be exact strings. + +.. + +.. bpo: 21722 +.. date: 7578 +.. nonce: WTHuRy +.. section: Library + +The distutils "upload" command now exits with a non-zero return code when +uploading fails. Patch by Martin Dengler. + +.. + +.. bpo: 21723 +.. date: 7577 +.. nonce: r86fwb +.. section: Library + +asyncio.Queue: support any type of number (ex: float) for the maximum size. +Patch written by Vajrasky Kok. + +.. + +.. bpo: 21711 +.. date: 7576 +.. nonce: JWPFQZ +.. section: Library + +support for "site-python" directories has now been removed from the site +module (it was deprecated in 3.4). + +.. + +.. bpo: 17552 +.. date: 7575 +.. nonce: NunErD +.. section: Library + +new socket.sendfile() method allowing a file to be sent over a socket by +using high-performance os.sendfile() on UNIX. Patch by Giampaolo Rodola'. + +.. + +.. bpo: 18039 +.. date: 7574 +.. nonce: vC9hNy +.. section: Library + +dbm.dump.open() now always creates a new database when the flag has the +value 'n'. Patch by Claudiu Popa. + +.. + +.. bpo: 21326 +.. date: 7573 +.. nonce: Y6iW3s +.. section: Library + +Add a new is_closed() method to asyncio.BaseEventLoop. run_forever() and +run_until_complete() methods of asyncio.BaseEventLoop now raise an exception +if the event loop was closed. + +.. + +.. bpo: 21766 +.. date: 7572 +.. nonce: 0xk_xC +.. section: Library + +Prevent a security hole in CGIHTTPServer by URL unquoting paths before +checking for a CGI script at that path. + +.. + +.. bpo: 21310 +.. date: 7571 +.. nonce: 2mjByJ +.. section: Library + +Fixed possible resource leak in failed open(). + +.. + +.. bpo: 21256 +.. date: 7570 +.. nonce: dGq6cw +.. section: Library + +Printout of keyword args should be in deterministic order in a mock function +call. This will help to write better doctests. + +.. + +.. bpo: 21677 +.. date: 7569 +.. nonce: 58CDDD +.. section: Library + +Fixed chaining nonnormalized exceptions in io close() methods. + +.. + +.. bpo: 11709 +.. date: 7568 +.. nonce: JdObvL +.. section: Library + +Fix the pydoc.help function to not fail when sys.stdin is not a valid file. + +.. + +.. bpo: 21515 +.. date: 7567 +.. nonce: D9TLJF +.. section: Library + +tempfile.TemporaryFile now uses os.O_TMPFILE flag is available. + +.. + +.. bpo: 13223 +.. date: 7566 +.. nonce: 9AzEbN +.. section: Library + +Fix pydoc.writedoc so that the HTML documentation for methods that use +'self' in the example code is generated correctly. + +.. + +.. bpo: 21463 +.. date: 7565 +.. nonce: 09PsgH +.. section: Library + +In urllib.request, fix pruning of the FTP cache. + +.. + +.. bpo: 21618 +.. date: 7564 +.. nonce: 3Z7WS3 +.. section: Library + +The subprocess module could fail to close open fds that were inherited by +the calling process and already higher than POSIX resource limits would +otherwise allow. On systems with a functioning /proc/self/fd or /dev/fd +interface the max is now ignored and all fds are closed. + +.. + +.. bpo: 20383 +.. date: 7563 +.. nonce: pSPFpW +.. section: Library + +Introduce importlib.util.module_from_spec() as the preferred way to create a +new module. + +.. + +.. bpo: 21552 +.. date: 7562 +.. nonce: uVy4tM +.. section: Library + +Fixed possible integer overflow of too long string lengths in the tkinter +module on 64-bit platforms. + +.. + +.. bpo: 14315 +.. date: 7561 +.. nonce: YzZzS8 +.. section: Library + +The zipfile module now ignores extra fields in the central directory that +are too short to be parsed instead of letting a struct.unpack error bubble +up as this "bad data" appears in many real world zip files in the wild and +is ignored by other zip tools. + +.. + +.. bpo: 13742 +.. date: 7560 +.. nonce: QJiVSC +.. section: Library + +Added "key" and "reverse" parameters to heapq.merge(). (First draft of patch +contributed by Simon Sapin.) + +.. + +.. bpo: 21402 +.. date: 7559 +.. nonce: 51vDXt +.. section: Library + +tkinter.ttk now works when default root window is not set. + +.. + +.. bpo: 3015 +.. date: 7558 +.. nonce: FE_PII +.. section: Library + +_tkinter.create() now creates tkapp object with wantobject=1 by default. + +.. + +.. bpo: 10203 +.. date: 7557 +.. nonce: zgr0hh +.. section: Library + +sqlite3.Row now truly supports sequence protocol. In particular it supports +reverse() and negative indices. Original patch by Claudiu Popa. + +.. + +.. bpo: 18807 +.. date: 7556 +.. nonce: XP7p8B +.. section: Library + +If copying (no symlinks) specified for a venv, then the python interpreter +aliases (python, python3) are now created by copying rather than symlinking. + +.. + +.. bpo: 20197 +.. date: 7555 +.. nonce: nYR9fq +.. section: Library + +Added support for the WebP image type in the imghdr module. Patch by Fabrice +Aneche and Claudiu Popa. + +.. + +.. bpo: 21513 +.. date: 7554 +.. nonce: ro4AOe +.. section: Library + +Speedup some properties of IP addresses (IPv4Address, IPv6Address) such as +.is_private or .is_multicast. + +.. + +.. bpo: 21137 +.. date: 7553 +.. nonce: wgHb_F +.. section: Library + +Improve the repr for threading.Lock() and its variants by showing the +"locked" or "unlocked" status. Patch by Berker Peksag. + +.. + +.. bpo: 21538 +.. date: 7552 +.. nonce: Q60FWA +.. section: Library + +The plistlib module now supports loading of binary plist files when +reference or offset size is not a power of two. + +.. + +.. bpo: 21455 +.. date: 7551 +.. nonce: 6-Uvv4 +.. section: Library + +Add a default backlog to socket.listen(). + +.. + +.. bpo: 21525 +.. date: 7550 +.. nonce: hAKOve +.. section: Library + +Most Tkinter methods which accepted tuples now accept lists too. + +.. + +.. bpo: 22166 +.. date: 7549 +.. nonce: sZYhmv +.. section: Library + +With the assistance of a new internal _codecs._forget_codec helping +function, test_codecs now clears the encoding caches to avoid the appearance +of a reference leak + +.. + +.. bpo: 22236 +.. date: 7548 +.. nonce: ginJSI +.. section: Library + +Tkinter tests now don't reuse default root window. New root window is +created for every test class. + +.. + +.. bpo: 10744 +.. date: 7547 +.. nonce: kfV0wm +.. section: Library + +Fix PEP 3118 format strings on ctypes objects with a nontrivial shape. + +.. + +.. bpo: 20826 +.. date: 7546 +.. nonce: 3rXqMC +.. section: Library + +Optimize ipaddress.collapse_addresses(). + +.. + +.. bpo: 21487 +.. date: 7545 +.. nonce: sX8YmK +.. section: Library + +Optimize ipaddress.summarize_address_range() and +ipaddress.{IPv4Network,IPv6Network}.subnets(). + +.. + +.. bpo: 21486 +.. date: 7544 +.. nonce: CeFKRP +.. section: Library + +Optimize parsing of netmasks in ipaddress.IPv4Network and +ipaddress.IPv6Network. + +.. + +.. bpo: 13916 +.. date: 7543 +.. nonce: D77YVH +.. section: Library + +Disallowed the surrogatepass error handler for non UTF-\* encodings. + +.. + +.. bpo: 20998 +.. date: 7542 +.. nonce: fkxpXI +.. section: Library + +Fixed re.fullmatch() of repeated single character pattern with ignore case. +Original patch by Matthew Barnett. + +.. + +.. bpo: 21075 +.. date: 7541 +.. nonce: f_hmEh +.. section: Library + +fileinput.FileInput now reads bytes from standard stream if binary mode is +specified. Patch by Sam Kimbrel. + +.. + +.. bpo: 19775 +.. date: 7540 +.. nonce: yxxD_R +.. section: Library + +Add a samefile() method to pathlib Path objects. Initial patch by Vajrasky +Kok. + +.. + +.. bpo: 21226 +.. date: 7539 +.. nonce: pzGmG1 +.. section: Library + +Set up modules properly in PyImport_ExecCodeModuleObject (and friends). + +.. + +.. bpo: 21398 +.. date: 7538 +.. nonce: guSBXt +.. section: Library + +Fix a unicode error in the pydoc pager when the documentation contains +characters not encodable to the stdout encoding. + +.. + +.. bpo: 16531 +.. date: 7537 +.. nonce: AhrY_v +.. section: Library + +ipaddress.IPv4Network and ipaddress.IPv6Network now accept an (address, +netmask) tuple argument, so as to easily construct network objects from +existing addresses. + +.. + +.. bpo: 21156 +.. date: 7536 +.. nonce: 3dmBEp +.. section: Library + +importlib.abc.InspectLoader.source_to_code() is now a staticmethod. + +.. + +.. bpo: 21424 +.. date: 7535 +.. nonce: 8CJBqW +.. section: Library + +Simplified and optimized heaqp.nlargest() and nmsmallest() to make fewer +tuple comparisons. + +.. + +.. bpo: 21396 +.. date: 7534 +.. nonce: cqO6DN +.. section: Library + +Fix TextIOWrapper(..., write_through=True) to not force a flush() on the +underlying binary stream. Patch by akira. + +.. + +.. bpo: 18314 +.. date: 7533 +.. nonce: NCd_KF +.. section: Library + +Unlink now removes junctions on Windows. Patch by Kim Gräsman + +.. + +.. bpo: 21088 +.. date: 7532 +.. nonce: WOg7Xy +.. section: Library + +Bugfix for curses.window.addch() regression in 3.4.0. In porting to Argument +Clinic, the first two arguments were reversed. + +.. + +.. bpo: 21407 +.. date: 7531 +.. nonce: cZjFde +.. section: Library + +_decimal: The module now supports function signatures. + +.. + +.. bpo: 10650 +.. date: 7530 +.. nonce: HYT4Oe +.. section: Library + +Remove the non-standard 'watchexp' parameter from the Decimal.quantize() +method in the Python version. It had never been present in the C version. + +.. + +.. bpo: 21469 +.. date: 7529 +.. nonce: _fFGuq +.. section: Library + +Reduced the risk of false positives in robotparser by checking to make sure +that robots.txt has been read or does not exist prior to returning True in +can_fetch(). + +.. + +.. bpo: 19414 +.. date: 7528 +.. nonce: bAAw4D +.. section: Library + +Have the OrderedDict mark deleted links as unusable. This gives an early +failure if the link is deleted during iteration. + +.. + +.. bpo: 21421 +.. date: 7527 +.. nonce: 5AKAat +.. section: Library + +Add __slots__ to the MappingViews ABC. Patch by Josh Rosenberg. + +.. + +.. bpo: 21101 +.. date: 7526 +.. nonce: Lj-_P4 +.. section: Library + +Eliminate double hashing in the C speed-up code for collections.Counter(). + +.. + +.. bpo: 21321 +.. date: 7525 +.. nonce: wUkTON +.. section: Library + +itertools.islice() now releases the reference to the source iterator when +the slice is exhausted. Patch by Anton Afanasyev. + +.. + +.. bpo: 21057 +.. date: 7524 +.. nonce: 0TC4Xl +.. section: Library + +TextIOWrapper now allows the underlying binary stream's read() or read1() +method to return an arbitrary bytes-like object (such as a memoryview). +Patch by Nikolaus Rath. + +.. + +.. bpo: 20951 +.. date: 7523 +.. nonce: tF0dJi +.. section: Library + +SSLSocket.send() now raises either SSLWantReadError or SSLWantWriteError on +a non-blocking socket if the operation would block. Previously, it would +return 0. Patch by Nikolaus Rath. + +.. + +.. bpo: 13248 +.. date: 7522 +.. nonce: 7vtGj0 +.. section: Library + +removed previously deprecated asyncore.dispatcher __getattr__ cheap +inheritance hack. + +.. + +.. bpo: 9815 +.. date: 7521 +.. nonce: 52FPlI +.. section: Library + +assertRaises now tries to clear references to local variables in the +exception's traceback. + +.. + +.. bpo: 19940 +.. date: 7520 +.. nonce: 2qtBQ8 +.. section: Library + +ssl.cert_time_to_seconds() now interprets the given time string in the UTC +timezone (as specified in RFC 5280), not the local timezone. + +.. + +.. bpo: 13204 +.. date: 7519 +.. nonce: ZPKA5g +.. section: Library + +Calling sys.flags.__new__ would crash the interpreter, now it raises a +TypeError. + +.. + +.. bpo: 19385 +.. date: 7518 +.. nonce: PexO_g +.. section: Library + +Make operations on a closed dbm.dumb database always raise the same +exception. + +.. + +.. bpo: 21207 +.. date: 7517 +.. nonce: Hr72AB +.. section: Library + +Detect when the os.urandom cached fd has been closed or replaced, and open +it anew. + +.. + +.. bpo: 21291 +.. date: 7516 +.. nonce: 5sSLWN +.. section: Library + +subprocess's Popen.wait() is now thread safe so that multiple threads may be +calling wait() or poll() on a Popen instance at the same time without losing +the Popen.returncode value. + +.. + +.. bpo: 21127 +.. date: 7515 +.. nonce: A1aBjG +.. section: Library + +Path objects can now be instantiated from str subclass instances (such as +``numpy.str_``). + +.. + +.. bpo: 15002 +.. date: 7514 +.. nonce: qorYDe +.. section: Library + +urllib.response object to use _TemporaryFileWrapper (and +_TemporaryFileCloser) facility. Provides a better way to handle file +descriptor close. Patch contributed by Christian Theune. + +.. + +.. bpo: 12220 +.. date: 7513 +.. nonce: U25uE9 +.. section: Library + +mindom now raises a custom ValueError indicating it doesn't support spaces +in URIs instead of letting a 'split' ValueError bubble up. + +.. + +.. bpo: 21068 +.. date: 7512 +.. nonce: 9k6N9m +.. section: Library + +The ssl.PROTOCOL* constants are now enum members. + +.. + +.. bpo: 21276 +.. date: 7511 +.. nonce: JkfhvQ +.. section: Library + +posixmodule: Don't define USE_XATTRS on KFreeBSD and the Hurd. + +.. + +.. bpo: 21262 +.. date: 7510 +.. nonce: 1J5ylk +.. section: Library + +New method assert_not_called for Mock. It raises AssertionError if the mock +has been called. + +.. + +.. bpo: 21238 +.. date: 7509 +.. nonce: 5CDoox +.. section: Library + +New keyword argument `unsafe` to Mock. It raises `AttributeError` incase of +an attribute startswith assert or assret. + +.. + +.. bpo: 20896 +.. date: 7508 +.. nonce: oWwAb1 +.. section: Library + +ssl.get_server_certificate() now uses PROTOCOL_SSLv23, not PROTOCOL_SSLv3, +for maximum compatibility. + +.. + +.. bpo: 21239 +.. date: 7507 +.. nonce: EalCNt +.. section: Library + +patch.stopall() didn't work deterministically when the same name was patched +more than once. + +.. + +.. bpo: 21203 +.. date: 7506 +.. nonce: 1IMs-Z +.. section: Library + +Updated fileConfig and dictConfig to remove inconsistencies. Thanks to Jure +Koren for the patch. + +.. + +.. bpo: 21222 +.. date: 7505 +.. nonce: G6MQBP +.. section: Library + +Passing name keyword argument to mock.create_autospec now works. + +.. + +.. bpo: 21197 +.. date: 7504 +.. nonce: Gzfqdl +.. section: Library + +Add lib64 -> lib symlink in venvs on 64-bit non-OS X POSIX. + +.. + +.. bpo: 17498 +.. date: 7503 +.. nonce: LR9xyb +.. section: Library + +Some SMTP servers disconnect after certain errors, violating strict RFC +conformance. Instead of losing the error code when we issue the subsequent +RSET, smtplib now returns the error code and defers raising the +SMTPServerDisconnected error until the next command is issued. + +.. + +.. bpo: 17826 +.. date: 7502 +.. nonce: z0zMRV +.. section: Library + +setting an iterable side_effect on a mock function created by +create_autospec now works. Patch by Kushal Das. + +.. + +.. bpo: 7776 +.. date: 7501 +.. nonce: K5S2Pe +.. section: Library + +Fix ``Host:`` header and reconnection when using +http.client.HTTPConnection.set_tunnel(). Patch by Nikolaus Rath. + +.. + +.. bpo: 20968 +.. date: 7500 +.. nonce: 53Aagz +.. section: Library + +unittest.mock.MagicMock now supports division. Patch by Johannes Baiter. + +.. + +.. bpo: 21529 +.. date: 7499 +.. nonce: 57R_Fc +.. section: Library + +Fix arbitrary memory access in JSONDecoder.raw_decode with a negative second +parameter. Bug reported by Guido Vranken. (See also: CVE-2014-4616) + +.. + +.. bpo: 21169 +.. date: 7498 +.. nonce: KE7B0M +.. section: Library + +getpass now handles non-ascii characters that the input stream encoding +cannot encode by re-encoding using the replace error handler. + +.. + +.. bpo: 21171 +.. date: 7497 +.. nonce: iUbV9S +.. section: Library + +Fixed undocumented filter API of the rot13 codec. Patch by Berker Peksag. + +.. + +.. bpo: 20539 +.. date: 7496 +.. nonce: 62nbEb +.. section: Library + +Improved math.factorial error message for large positive inputs and changed +exception type (OverflowError -> ValueError) for large negative inputs. + +.. + +.. bpo: 21172 +.. date: 7495 +.. nonce: dQ7yY7 +.. section: Library + +isinstance check relaxed from dict to collections.Mapping. + +.. + +.. bpo: 21155 +.. date: 7494 +.. nonce: JSKEE7 +.. section: Library + +asyncio.EventLoop.create_unix_server() now raises a ValueError if path and +sock are specified at the same time. + +.. + +.. bpo: 21136 +.. date: 7493 +.. nonce: JZAKv3 +.. section: Library + +Avoid unnecessary normalization of Fractions resulting from power and other +operations. Patch by Raymond Hettinger. + +.. + +.. bpo: 17621 +.. date: 7492 +.. nonce: 1x0mvJ +.. section: Library + +Introduce importlib.util.LazyLoader. + +.. + +.. bpo: 21076 +.. date: 7491 +.. nonce: upxQc6 +.. section: Library + +signal module constants were turned into enums. Patch by Giampaolo Rodola'. + +.. + +.. bpo: 20636 +.. date: 7490 +.. nonce: KGh-BD +.. section: Library + +Improved the repr of Tkinter widgets. + +.. + +.. bpo: 19505 +.. date: 7489 +.. nonce: VEtIE6 +.. section: Library + +The items, keys, and values views of OrderedDict now support reverse +iteration using reversed(). + +.. + +.. bpo: 21149 +.. date: 7488 +.. nonce: cnjwMR +.. section: Library + +Improved thread-safety in logging cleanup during interpreter shutdown. +Thanks to Devin Jeanpierre for the patch. + +.. + +.. bpo: 21058 +.. date: 7487 +.. nonce: IhluPP +.. section: Library + +Fix a leak of file descriptor in :func:`tempfile.NamedTemporaryFile`, close +the file descriptor if :func:`io.open` fails + +.. + +.. bpo: 21200 +.. date: 7486 +.. nonce: Kht8yD +.. section: Library + +Return None from pkgutil.get_loader() when __spec__ is missing. + +.. + +.. bpo: 21013 +.. date: 7485 +.. nonce: 3s8Ic0 +.. section: Library + +Enhance ssl.create_default_context() when used for server side sockets to +provide better security by default. + +.. + +.. bpo: 20145 +.. date: 7484 +.. nonce: FP5FY0 +.. section: Library + +`assertRaisesRegex` and `assertWarnsRegex` now raise a TypeError if the +second argument is not a string or compiled regex. + +.. + +.. bpo: 20633 +.. date: 7483 +.. nonce: 6kaPjT +.. section: Library + +Replace relative import by absolute import. + +.. + +.. bpo: 20980 +.. date: 7482 +.. nonce: cYszHY +.. section: Library + +Stop wrapping exception when using ThreadPool. + +.. + +.. bpo: 21082 +.. date: 7481 +.. nonce: GLzGlV +.. section: Library + +In os.makedirs, do not set the process-wide umask. Note this changes +behavior of makedirs when exist_ok=True. + +.. + +.. bpo: 20990 +.. date: 7480 +.. nonce: PBfjW3 +.. section: Library + +Fix issues found by pyflakes for multiprocessing. + +.. + +.. bpo: 21015 +.. date: 7479 +.. nonce: xnwWAH +.. section: Library + +SSL contexts will now automatically select an elliptic curve for ECDH key +exchange on OpenSSL 1.0.2 and later, and otherwise default to "prime256v1". + +.. + +.. bpo: 21000 +.. date: 7478 +.. nonce: JUyyVV +.. section: Library + +Improve the command-line interface of json.tool. + +.. + +.. bpo: 20995 +.. date: 7477 +.. nonce: KSORJT +.. section: Library + +Enhance default ciphers used by the ssl module to enable better security and +prioritize perfect forward secrecy. + +.. + +.. bpo: 20884 +.. date: 7476 +.. nonce: qNmub_ +.. section: Library + +Don't assume that __file__ is defined on importlib.__init__. + +.. + +.. bpo: 21499 +.. date: 7475 +.. nonce: wU4OBi +.. section: Library + +Ignore __builtins__ in several test_importlib.test_api tests. + +.. + +.. bpo: 20627 +.. date: 7474 +.. nonce: fgfQ1x +.. section: Library + +xmlrpc.client.ServerProxy is now a context manager. + +.. + +.. bpo: 19165 +.. date: 7473 +.. nonce: sAkUjU +.. section: Library + +The formatter module now raises DeprecationWarning instead of +PendingDeprecationWarning. + +.. + +.. bpo: 13936 +.. date: 7472 +.. nonce: _Q0Yog +.. section: Library + +Remove the ability of datetime.time instances to be considered false in +boolean contexts. + +.. + +.. bpo: 18931 +.. date: 7471 +.. nonce: mq4Mud +.. section: Library + +selectors module now supports /dev/poll on Solaris. Patch by Giampaolo +Rodola'. + +.. + +.. bpo: 19977 +.. date: 7470 +.. nonce: A-sQ_V +.. section: Library + +When the ``LC_TYPE`` locale is the POSIX locale (``C`` locale), +:py:data:`sys.stdin` and :py:data:`sys.stdout` are now using the +``surrogateescape`` error handler, instead of the ``strict`` error handler. + +.. + +.. bpo: 20574 +.. date: 7469 +.. nonce: KaKqSs +.. section: Library + +Implement incremental decoder for cp65001 code (Windows code page 65001, +Microsoft UTF-8). + +.. + +.. bpo: 20879 +.. date: 7468 +.. nonce: myeYdq +.. section: Library + +Delay the initialization of encoding and decoding tables for base32, ascii85 +and base85 codecs in the base64 module, and delay the initialization of the +unquote_to_bytes() table of the urllib.parse module, to not waste memory if +these modules are not used. + +.. + +.. bpo: 19157 +.. date: 7467 +.. nonce: V1-XhC +.. section: Library + +Include the broadcast address in the usuable hosts for IPv6 in ipaddress. + +.. + +.. bpo: 11599 +.. date: 7466 +.. nonce: 9QOXf4 +.. section: Library + +When an external command (e.g. compiler) fails, distutils now prints out the +whole command line (instead of just the command name) if the environment +variable DISTUTILS_DEBUG is set. + +.. + +.. bpo: 4931 +.. date: 7465 +.. nonce: uF10hr +.. section: Library + +distutils should not produce unhelpful "error: None" messages anymore. +distutils.util.grok_environment_error is kept but doc-deprecated. + +.. + +.. bpo: 20875 +.. date: 7464 +.. nonce: IjfI5V +.. section: Library + +Prevent possible gzip "'read' is not defined" NameError. Patch by Claudiu +Popa. + +.. + +.. bpo: 11558 +.. date: 7463 +.. nonce: pxrsmq +.. section: Library + +``email.message.Message.attach`` now returns a more useful error message if +``attach`` is called on a message for which ``is_multipart`` is False. + +.. + +.. bpo: 20283 +.. date: 7462 +.. nonce: v0Vs9V +.. section: Library + +RE pattern methods now accept the string keyword parameters as documented. +The pattern and source keyword parameters are left as deprecated aliases. + +.. + +.. bpo: 20778 +.. date: 7461 +.. nonce: g_fAGI +.. section: Library + +Fix modulefinder to work with bytecode-only modules. + +.. + +.. bpo: 20791 +.. date: 7460 +.. nonce: n_zrkc +.. section: Library + +copy.copy() now doesn't make a copy when the input is a bytes object. +Initial patch by Peter Otten. + +.. + +.. bpo: 19748 +.. date: 7459 +.. nonce: kiA171 +.. section: Library + +On AIX, time.mktime() now raises an OverflowError for year outsize range +[1902; 2037]. + +.. + +.. bpo: 19573 +.. date: 7458 +.. nonce: QJvX_V +.. section: Library + +inspect.signature: Use enum for parameter kind constants. + +.. + +.. bpo: 20726 +.. date: 7457 +.. nonce: 0yfRDI +.. section: Library + +inspect.signature: Make Signature and Parameter picklable. + +.. + +.. bpo: 17373 +.. date: 7456 +.. nonce: ECwuJO +.. section: Library + +Add inspect.Signature.from_callable method. + +.. + +.. bpo: 20378 +.. date: 7455 +.. nonce: l9M3H- +.. section: Library + +Improve repr of inspect.Signature and inspect.Parameter. + +.. + +.. bpo: 20816 +.. date: 7454 +.. nonce: DFMEgN +.. section: Library + +Fix inspect.getcallargs() to raise correct TypeError for missing keyword- +only arguments. Patch by Jeremiah Lowin. + +.. + +.. bpo: 20817 +.. date: 7453 +.. nonce: O5XyZB +.. section: Library + +Fix inspect.getcallargs() to fail correctly if more than 3 arguments are +missing. Patch by Jeremiah Lowin. + +.. + +.. bpo: 6676 +.. date: 7452 +.. nonce: CJu5On +.. section: Library + +Ensure a meaningful exception is raised when attempting to parse more than +one XML document per pyexpat xmlparser instance. (Original patches by +Hirokazu Yamamoto and Amaury Forgeot d'Arc, with suggested wording by David +Gutteridge) + +.. + +.. bpo: 21117 +.. date: 7451 +.. nonce: hyH7EK +.. section: Library + +Fix inspect.signature to better support functools.partial. Due to the +specifics of functools.partial implementation, positional-or-keyword +arguments passed as keyword arguments become keyword-only. + +.. + +.. bpo: 20334 +.. date: 7450 +.. nonce: 0yFmfQ +.. section: Library + +inspect.Signature and inspect.Parameter are now hashable. Thanks to Antony +Lee for bug reports and suggestions. + +.. + +.. bpo: 15916 +.. date: 7449 +.. nonce: _vhKPn +.. section: Library + +doctest.DocTestSuite returns an empty unittest.TestSuite instead of raising +ValueError if it finds no tests + +.. + +.. bpo: 21209 +.. date: 7448 +.. nonce: wRE7Dn +.. section: Library + +Fix asyncio.tasks.CoroWrapper to workaround a bug in yield-from +implementation in CPythons prior to 3.4.1. + +.. + +.. bpo: 0 +.. date: 7447 +.. nonce: Q1I78Z +.. section: Library + +asyncio: Add gi_{frame,running,code} properties to CoroWrapper (upstream +issue #163). + +.. + +.. bpo: 21311 +.. date: 7446 +.. nonce: JsDF8H +.. section: Library + +Avoid exception in _osx_support with non-standard compiler configurations. +Patch by John Szakmeister. + +.. + +.. bpo: 11571 +.. date: 7445 +.. nonce: RPeGNo +.. section: Library + +Ensure that the turtle window becomes the topmost window when launched on OS +X. + +.. + +.. bpo: 21801 +.. date: 7444 +.. nonce: rzfhYl +.. section: Library + +Validate that __signature__ is None or an instance of Signature. + +.. + +.. bpo: 21923 +.. date: 7443 +.. nonce: hXnoZa +.. section: Library + +Prevent AttributeError in distutils.sysconfig.customize_compiler due to +possible uninitialized _config_vars. + +.. + +.. bpo: 21323 +.. date: 7442 +.. nonce: quiWfl +.. section: Library + +Fix http.server to again handle scripts in CGI subdirectories, broken by the +fix for security issue #19435. Patch by Zach Byrne. + +.. + +.. bpo: 22733 +.. date: 7441 +.. nonce: 21gJBp +.. section: Library + +Fix ffi_prep_args not zero-extending argument values correctly on 64-bit +Windows. + +.. + +.. bpo: 23302 +.. date: 7440 +.. nonce: X2dabK +.. section: Library + +Default to TCP_NODELAY=1 upon establishing an HTTPConnection. Removed use of +hard-coded MSS as it's an optimization that's no longer needed with Nagle +disabled. + +.. + +.. bpo: 20577 +.. date: 7439 +.. nonce: Y71IMj +.. section: IDLE + +Configuration of the max line length for the FormatParagraph extension has +been moved from the General tab of the Idle preferences dialog to the +FormatParagraph tab of the Config Extensions dialog. Patch by Tal Einat. + +.. + +.. bpo: 16893 +.. date: 7438 +.. nonce: JfHAA4 +.. section: IDLE + +Update Idle doc chapter to match current Idle and add new information. + +.. + +.. bpo: 3068 +.. date: 7437 +.. nonce: TYjXTA +.. section: IDLE + +Add Idle extension configuration dialog to Options menu. Changes are written +to HOME/.idlerc/config-extensions.cfg. Original patch by Tal Einat. + +.. + +.. bpo: 16233 +.. date: 7436 +.. nonce: sOadNo +.. section: IDLE + +A module browser (File : Class Browser, Alt+C) requires an editor window +with a filename. When Class Browser is requested otherwise, from a shell, +output window, or 'Untitled' editor, Idle no longer displays an error box. +It now pops up an Open Module box (Alt+M). If a valid name is entered and a +module is opened, a corresponding browser is also opened. + +.. + +.. bpo: 4832 +.. date: 7435 +.. nonce: GRKi9M +.. section: IDLE + +Save As to type Python files automatically adds .py to the name you enter +(even if your system does not display it). Some systems automatically add +.txt when type is Text files. + +.. + +.. bpo: 21986 +.. date: 7434 +.. nonce: 04GUv2 +.. section: IDLE + +Code objects are not normally pickled by the pickle module. To match this, +they are no longer pickled when running under Idle. + +.. + +.. bpo: 17390 +.. date: 7433 +.. nonce: I4vHFh +.. section: IDLE + +Adjust Editor window title; remove 'Python', move version to end. + +.. + +.. bpo: 14105 +.. date: 7432 +.. nonce: -FZwYH +.. section: IDLE + +Idle debugger breakpoints no longer disappear when inserting or deleting +lines. + +.. + +.. bpo: 17172 +.. date: 7431 +.. nonce: R8jkU1 +.. section: IDLE + +Turtledemo can now be run from Idle. Currently, the entry is on the Help +menu, but it may move to Run. Patch by Ramchandra Apt and Lita Cho. + +.. + +.. bpo: 21765 +.. date: 7430 +.. nonce: JyiDbd +.. section: IDLE + +Add support for non-ascii identifiers to HyperParser. + +.. + +.. bpo: 21940 +.. date: 7429 +.. nonce: VlIRz7 +.. section: IDLE + +Add unittest for WidgetRedirector. Initial patch by Saimadhav Heblikar. + +.. + +.. bpo: 18592 +.. date: 7428 +.. nonce: sMG-SZ +.. section: IDLE + +Add unittest for SearchDialogBase. Patch by Phil Webster. + +.. + +.. bpo: 21694 +.. date: 7427 +.. nonce: 1oLmRo +.. section: IDLE + +Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +.. + +.. bpo: 21686 +.. date: 7426 +.. nonce: TAkFB0 +.. section: IDLE + +add unittest for HyperParser. Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 12387 +.. date: 7425 +.. nonce: XO7Ozk +.. section: IDLE + +Add missing upper(lower)case versions of default Windows key bindings for +Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +.. + +.. bpo: 21695 +.. date: 7424 +.. nonce: g-t0Tm +.. section: IDLE + +Closing a Find-in-files output window while the search is still in progress +no longer closes Idle. + +.. + +.. bpo: 18910 +.. date: 7423 +.. nonce: ke8lMK +.. section: IDLE + +Add unittest for textView. Patch by Phil Webster. + +.. + +.. bpo: 18292 +.. date: 7422 +.. nonce: ks_3wm +.. section: IDLE + +Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +.. + +.. bpo: 18409 +.. date: 7421 +.. nonce: 7fe-aK +.. section: IDLE + +Add unittest for AutoComplete. Patch by Phil Webster. + +.. + +.. bpo: 21477 +.. date: 7420 +.. nonce: 33NOe0 +.. section: IDLE + +htest.py - Improve framework, complete set of tests. Patches by Saimadhav +Heblikar + +.. + +.. bpo: 18104 +.. date: 7419 +.. nonce: 8Fj9Pf +.. section: IDLE + +Add idlelib/idle_test/htest.py with a few sample tests to begin +consolidating and improving human-validated tests of Idle. Change other +files as needed to work with htest. Running the module as __main__ runs all +tests. + +.. + +.. bpo: 21139 +.. date: 7418 +.. nonce: kqetng +.. section: IDLE + +Change default paragraph width to 72, the PEP 8 recommendation. + +.. + +.. bpo: 21284 +.. date: 7417 +.. nonce: KKJfmv +.. section: IDLE + +Paragraph reformat test passes after user changes reformat width. + +.. + +.. bpo: 17654 +.. date: 7416 +.. nonce: NbzhNS +.. section: IDLE + +Ensure IDLE menus are customized properly on OS X for non-framework builds +and for all variants of Tk. + +.. + +.. bpo: 23180 +.. date: 7415 +.. nonce: cE_89F +.. section: IDLE + +Rename IDLE "Windows" menu item to "Window". Patch by Al Sweigart. + +.. + +.. bpo: 15506 +.. date: 7414 +.. nonce: nh8KlR +.. section: Build + +Use standard PKG_PROG_PKG_CONFIG autoconf macro in the configure script. + +.. + +.. bpo: 22935 +.. date: 7413 +.. nonce: -vY3lc +.. section: Build + +Allow the ssl module to be compiled if openssl doesn't support SSL 3. + +.. + +.. bpo: 22592 +.. date: 7412 +.. nonce: O_IE9W +.. section: Build + +Drop support of the Borland C compiler to build Python. The distutils module +still supports it to build extensions. + +.. + +.. bpo: 22591 +.. date: 7411 +.. nonce: wwBlG8 +.. section: Build + +Drop support of MS-DOS, especially of the DJGPP compiler (MS-DOS port of +GCC). + +.. + +.. bpo: 16537 +.. date: 7410 +.. nonce: llFo71 +.. section: Build + +Check whether self.extensions is empty in setup.py. Patch by Jonathan +Hosmer. + +.. + +.. bpo: 22359 +.. date: 7409 +.. nonce: YYFOFG +.. section: Build + +Remove incorrect uses of recursive make. Patch by Jonas Wagner. + +.. + +.. bpo: 21958 +.. date: 7408 +.. nonce: 3rq4qR +.. section: Build + +Define HAVE_ROUND when building with Visual Studio 2013 and above. Patch by +Zachary Turner. + +.. + +.. bpo: 18093 +.. date: 7407 +.. nonce: gnZieo +.. section: Build + +the programs that embed the CPython runtime are now in a separate "Programs" +directory, rather than being kept in the Modules directory. + +.. + +.. bpo: 15759 +.. date: 7406 +.. nonce: iGLR6O +.. section: Build + +"make suspicious", "make linkcheck" and "make doctest" in Doc/ now display +special message when and only when there are failures. + +.. + +.. bpo: 21141 +.. date: 7405 +.. nonce: 669LzK +.. section: Build + +The Windows build process no longer attempts to find Perl, instead relying +on OpenSSL source being configured and ready to build. The +``PCbuild\build_ssl.py`` script has been re-written and re-named to +``PCbuild\prepare_ssl.py``, and takes care of configuring OpenSSL source for +both 32 and 64 bit platforms. OpenSSL sources obtained from svn.python.org +will always be pre-configured and ready to build. + +.. + +.. bpo: 21037 +.. date: 7404 +.. nonce: v1rZzo +.. section: Build + +Add a build option to enable AddressSanitizer support. + +.. + +.. bpo: 19962 +.. date: 7403 +.. nonce: HDlwsE +.. section: Build + +The Windows build process now creates "python.bat" in the root of the source +tree, which passes all arguments through to the most recently built +interpreter. + +.. + +.. bpo: 21285 +.. date: 7402 +.. nonce: cU9p2E +.. section: Build + +Refactor and fix curses configure check to always search in a ncursesw +directory. + +.. + +.. bpo: 15234 +.. date: 7401 +.. nonce: vlM720 +.. section: Build + +For BerkelyDB and Sqlite, only add the found library and include directories +if they aren't already being searched. This avoids an explicit runtime +library dependency. + +.. + +.. bpo: 17861 +.. date: 7400 +.. nonce: jCi44U +.. section: Build + +Tools/scripts/generate_opcode_h.py automatically regenerates +Include/opcode.h from Lib/opcode.py if the latter gets any change. + +.. + +.. bpo: 20644 +.. date: 7399 +.. nonce: aV0zq7 +.. section: Build + +OS X installer build support for documentation build changes in 3.4.1: +assume externally supplied sphinx-build is available in /usr/bin. + +.. + +.. bpo: 20022 +.. date: 7398 +.. nonce: EqSCTW +.. section: Build + +Eliminate use of deprecated bundlebuilder in OS X builds. + +.. + +.. bpo: 15968 +.. date: 7397 +.. nonce: vxUxHK +.. section: Build + +Incorporated Tcl, Tk, and Tix builds into the Windows build solution. + +.. + +.. bpo: 17095 +.. date: 7396 +.. nonce: -XEBIU +.. section: Build + +Fix Modules/Setup *shared* support. + +.. + +.. bpo: 21811 +.. date: 7395 +.. nonce: 3_Xyr- +.. section: Build + +Anticipated fixes to support OS X versions > 10.9. + +.. + +.. bpo: 21166 +.. date: 7394 +.. nonce: KAl7aO +.. section: Build + +Prevent possible segfaults and other random failures of python --generate- +posix-vars in pybuilddir.txt build target. + +.. + +.. bpo: 18096 +.. date: 7393 +.. nonce: ELyAUJ +.. section: Build + +Fix library order returned by python-config. + +.. + +.. bpo: 17219 +.. date: 7392 +.. nonce: q8ueQ0 +.. section: Build + +Add library build dir for Python extension cross-builds. + +.. + +.. bpo: 22919 +.. date: 7391 +.. nonce: 1XThL9 +.. section: Build + +Windows build updated to support VC 14.0 (Visual Studio 2015), which will be +used for the official release. + +.. + +.. bpo: 21236 +.. date: 7390 +.. nonce: 84LXxj +.. section: Build + +Build _msi.pyd with cabinet.lib instead of fci.lib + +.. + +.. bpo: 17128 +.. date: 7389 +.. nonce: U2biLA +.. section: Build + +Use private version of OpenSSL for OS X 10.5+ installer. + +.. + +.. bpo: 14203 +.. date: 7388 +.. nonce: 3hv0TX +.. section: C API + +Remove obsolete support for view==NULL in PyBuffer_FillInfo(), +bytearray_getbuffer(), bytesiobuf_getbuffer() and array_buffer_getbuf(). All +functions now raise BufferError in that case. + +.. + +.. bpo: 22445 +.. date: 7387 +.. nonce: s0AOAS +.. section: C API + +PyBuffer_IsContiguous() now implements precise contiguity tests, compatible +with NumPy's NPY_RELAXED_STRIDES_CHECKING compilation flag. Previously the +function reported false negatives for corner cases. + +.. + +.. bpo: 22079 +.. date: 7386 +.. nonce: zhs2qM +.. section: C API + +PyType_Ready() now checks that statically allocated type has no dynamically +allocated bases. + +.. + +.. bpo: 22453 +.. date: 7385 +.. nonce: XoO4ns +.. section: C API + +Removed non-documented macro PyObject_REPR(). + +.. + +.. bpo: 18395 +.. date: 7384 +.. nonce: YC9B06 +.. section: C API + +Rename ``_Py_char2wchar()`` to :c:func:`Py_DecodeLocale`, rename +``_Py_wchar2char()`` to :c:func:`Py_EncodeLocale`, and document these +functions. + +.. + +.. bpo: 21233 +.. date: 7383 +.. nonce: 98hZAt +.. section: C API + +Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(), PyObject_Calloc(), +_PyObject_GC_Calloc(). bytes(int) is now using ``calloc()`` instead of +``malloc()`` for large objects which is faster and use less memory. + +.. + +.. bpo: 20942 +.. date: 7382 +.. nonce: qHLJ5- +.. section: C API + +PyImport_ImportFrozenModuleObject() no longer sets __file__ to match what +importlib does; this affects _frozen_importlib as well as any module loaded +using imp.init_frozen(). + +.. + +.. bpo: 19548 +.. date: 7381 +.. nonce: yOX8sS +.. section: Documentation + +Update the codecs module documentation to better cover the distinction +between text encodings and other codecs, together with other clarifications. +Patch by Martin Panter. + +.. + +.. bpo: 22394 +.. date: 7380 +.. nonce: 6bJywY +.. section: Documentation + +Doc/Makefile now supports ``make venv PYTHON=../python`` to create a venv +for generating the documentation, e.g., ``make html +PYTHON=venv/bin/python3``. + +.. + +.. bpo: 21514 +.. date: 7379 +.. nonce: 1H16T6 +.. section: Documentation + +The documentation of the json module now refers to new JSON RFC 7159 instead +of obsoleted RFC 4627. + +.. + +.. bpo: 21777 +.. date: 7378 +.. nonce: dtQCWV +.. section: Documentation + +The binary sequence methods on bytes and bytearray are now documented +explicitly, rather than assuming users will be able to derive the expected +behaviour from the behaviour of the corresponding str methods. + +.. + +.. bpo: 6916 +.. date: 7377 +.. nonce: 4sm3nE +.. section: Documentation + +undocument deprecated asynchat.fifo class. + +.. + +.. bpo: 17386 +.. date: 7376 +.. nonce: ivaGLb +.. section: Documentation + +Expanded functionality of the ``Doc/make.bat`` script to make it much more +comparable to ``Doc/Makefile``. + +.. + +.. bpo: 21312 +.. date: 7375 +.. nonce: 6IqcV4 +.. section: Documentation + +Update the thread_foobar.h template file to include newer threading APIs. +Patch by Jack McCracken. + +.. + +.. bpo: 21043 +.. date: 7374 +.. nonce: oEOC8O +.. section: Documentation + +Remove the recommendation for specific CA organizations and to mention the +ability to load the OS certificates. + +.. + +.. bpo: 20765 +.. date: 7373 +.. nonce: Rv3GgV +.. section: Documentation + +Add missing documentation for PurePath.with_name() and +PurePath.with_suffix(). + +.. + +.. bpo: 19407 +.. date: 7372 +.. nonce: mRyNnG +.. section: Documentation + +New package installation and distribution guides based on the Python +Packaging Authority tools. Existing guides have been retained as legacy +links from the distutils docs, as they still contain some required reference +material for tool developers that isn't recorded anywhere else. + +.. + +.. bpo: 19697 +.. date: 7371 +.. nonce: 2jMQBP +.. section: Documentation + +Document cases where __main__.__spec__ is None. + +.. + +.. bpo: 18982 +.. date: 7370 +.. nonce: TynSM6 +.. section: Tests + +Add tests for CLI of the calendar module. + +.. + +.. bpo: 19548 +.. date: 7369 +.. nonce: 25Kxq_ +.. section: Tests + +Added some additional checks to test_codecs to ensure that statements in the +updated documentation remain accurate. Patch by Martin Panter. + +.. + +.. bpo: 22838 +.. date: 7368 +.. nonce: VZBtZg +.. section: Tests + +All test_re tests now work with unittest test discovery. + +.. + +.. bpo: 22173 +.. date: 7367 +.. nonce: dxIIVx +.. section: Tests + +Update lib2to3 tests to use unittest test discovery. + +.. + +.. bpo: 16000 +.. date: 7366 +.. nonce: Y7O6TP +.. section: Tests + +Convert test_curses to use unittest. + +.. + +.. bpo: 21456 +.. date: 7365 +.. nonce: Axsw43 +.. section: Tests + +Skip two tests in test_urllib2net.py if _ssl module not present. Patch by +Remi Pointel. + +.. + +.. bpo: 20746 +.. date: 7364 +.. nonce: N2pzAY +.. section: Tests + +Fix test_pdb to run in refleak mode (-R). Patch by Xavier de Gaye. + +.. + +.. bpo: 22060 +.. date: 7363 +.. nonce: TduJNO +.. section: Tests + +test_ctypes has been somewhat cleaned up and simplified; it now uses +unittest test discovery to find its tests. + +.. + +.. bpo: 22104 +.. date: 7362 +.. nonce: -YYDup +.. section: Tests + +regrtest.py no longer holds a reference to the suite of tests loaded from +test modules that don't define test_main(). + +.. + +.. bpo: 22111 +.. date: 7361 +.. nonce: 0XlFAU +.. section: Tests + +Assorted cleanups in test_imaplib. Patch by Milan Oberkirch. + +.. + +.. bpo: 22002 +.. date: 7360 +.. nonce: jpiaA2 +.. section: Tests + +Added ``load_package_tests`` function to test.support and used it to +implement/augment test discovery in test_asyncio, test_email, +test_importlib, test_json, and test_tools. + +.. + +.. bpo: 21976 +.. date: 7359 +.. nonce: Slq6se +.. section: Tests + +Fix test_ssl to accept LibreSSL version strings. Thanks to William Orr. + +.. + +.. bpo: 21918 +.. date: 7358 +.. nonce: QTFFSj +.. section: Tests + +Converted test_tools from a module to a package containing separate test +files for each tested script. + +.. + +.. bpo: 9554 +.. date: 7357 +.. nonce: VsP0Ve +.. section: Tests + +Use modern unittest features in test_argparse. Initial patch by Denver +Coneybeare and Radu Voicilas. + +.. + +.. bpo: 20155 +.. date: 7356 +.. nonce: nphzS3 +.. section: Tests + +Changed HTTP method names in failing tests in test_httpservers so that +packet filtering software (specifically Windows Base Filtering Engine) does +not interfere with the transaction semantics expected by the tests. + +.. + +.. bpo: 19493 +.. date: 7355 +.. nonce: SwbzLQ +.. section: Tests + +Refactored the ctypes test package to skip tests explicitly rather than +silently. + +.. + +.. bpo: 18492 +.. date: 7354 +.. nonce: ylPRU7 +.. section: Tests + +All resources are now allowed when tests are not run by regrtest.py. + +.. + +.. bpo: 21634 +.. date: 7353 +.. nonce: Eng06F +.. section: Tests + +Fix pystone micro-benchmark: use floor division instead of true division to +benchmark integers instead of floating point numbers. Set pystone version to +1.2. Patch written by Lennart Regebro. + +.. + +.. bpo: 21605 +.. date: 7352 +.. nonce: qsLV8d +.. section: Tests + +Added tests for Tkinter images. + +.. + +.. bpo: 21493 +.. date: 7351 +.. nonce: NqhRsy +.. section: Tests + +Added test for ntpath.expanduser(). Original patch by Claudiu Popa. + +.. + +.. bpo: 19925 +.. date: 7350 +.. nonce: dhMx08 +.. section: Tests + +Added tests for the spwd module. Original patch by Vajrasky Kok. + +.. + +.. bpo: 21522 +.. date: 7349 +.. nonce: b-VwFW +.. section: Tests + +Added Tkinter tests for Listbox.itemconfigure(), +PanedWindow.paneconfigure(), and Menu.entryconfigure(). + +.. + +.. bpo: 17756 +.. date: 7348 +.. nonce: LLfbfU +.. section: Tests + +Fix test_code test when run from the installed location. + +.. + +.. bpo: 17752 +.. date: 7347 +.. nonce: P8iG44 +.. section: Tests + +Fix distutils tests when run from the installed location. + +.. + +.. bpo: 18604 +.. date: 7346 +.. nonce: Q00Xrj +.. section: Tests + +Consolidated checks for GUI availability. All platforms now at least check +whether Tk can be instantiated when the GUI resource is requested. + +.. + +.. bpo: 21275 +.. date: 7345 +.. nonce: lI5FkX +.. section: Tests + +Fix a socket test on KFreeBSD. + +.. + +.. bpo: 21223 +.. date: 7344 +.. nonce: lMY6ka +.. section: Tests + +Pass test_site/test_startup_imports when some of the extensions are built as +builtins. + +.. + +.. bpo: 20635 +.. date: 7343 +.. nonce: mzWmoS +.. section: Tests + +Added tests for Tk geometry managers. + +.. + +.. bpo: 0 +.. date: 7342 +.. nonce: E5XNqr +.. section: Tests + +Add test case for freeze. + +.. + +.. bpo: 20743 +.. date: 7341 +.. nonce: hxZQUf +.. section: Tests + +Fix a reference leak in test_tcl. + +.. + +.. bpo: 21097 +.. date: 7340 +.. nonce: gsUesm +.. section: Tests + +Move test_namespace_pkgs into test_importlib. + +.. + +.. bpo: 21503 +.. date: 7339 +.. nonce: H9TPCg +.. section: Tests + +Use test_both() consistently in test_importlib. + +.. + +.. bpo: 20939 +.. date: 7338 +.. nonce: x3KQ35 +.. section: Tests + +Avoid various network test failures due to new redirect of +http://www.python.org/ to https://www.python.org: use http://www.example.com +instead. + +.. + +.. bpo: 20668 +.. date: 7337 +.. nonce: IWjOSC +.. section: Tests + +asyncio tests no longer rely on tests.txt file. (Patch by Vajrasky Kok) + +.. + +.. bpo: 21093 +.. date: 7336 +.. nonce: CcpRim +.. section: Tests + +Prevent failures of ctypes test_macholib on OS X if a copy of libz exists in +$HOME/lib or /usr/local/lib. + +.. + +.. bpo: 22770 +.. date: 7335 +.. nonce: FxAh91 +.. section: Tests + +Prevent some Tk segfaults on OS X when running gui tests. + +.. + +.. bpo: 23211 +.. date: 7334 +.. nonce: Bc-QfJ +.. section: Tests + +Workaround test_logging failure on some OS X 10.6 systems. + +.. + +.. bpo: 23345 +.. date: 7333 +.. nonce: HIGBKx +.. section: Tests + +Prevent test_ssl failures with large OpenSSL patch level values (like +0.9.8zc). + +.. + +.. bpo: 22314 +.. date: 7332 +.. nonce: ws6xsH +.. section: Tools/Demos + +pydoc now works when the LINES environment variable is set. + +.. + +.. bpo: 22615 +.. date: 7331 +.. nonce: My3DWN +.. section: Tools/Demos + +Argument Clinic now supports the "type" argument for the int converter. +This permits using the int converter with enums and typedefs. + +.. + +.. bpo: 20076 +.. date: 7330 +.. nonce: ZNuBrC +.. section: Tools/Demos + +The makelocalealias.py script no longer ignores UTF-8 mapping. + +.. + +.. bpo: 20079 +.. date: 7329 +.. nonce: ogPXcK +.. section: Tools/Demos + +The makelocalealias.py script now can parse the SUPPORTED file from glibc +sources and supports command line options for source paths. + +.. + +.. bpo: 22201 +.. date: 7328 +.. nonce: k1Awbh +.. section: Tools/Demos + +Command-line interface of the zipfile module now correctly extracts ZIP +files with directory entries. Patch by Ryan Wilson. + +.. + +.. bpo: 22120 +.. date: 7327 +.. nonce: KmBUj- +.. section: Tools/Demos + +For functions using an unsigned integer return converter, Argument Clinic +now generates a cast to that type for the comparison to -1 in the generated +code. (This suppresses a compilation warning.) + +.. + +.. bpo: 18974 +.. date: 7326 +.. nonce: I3DdAo +.. section: Tools/Demos + +Tools/scripts/diff.py now uses argparse instead of optparse. + +.. + +.. bpo: 21906 +.. date: 7325 +.. nonce: ZsKy9v +.. section: Tools/Demos + +Make Tools/scripts/md5sum.py work in Python 3. Patch by Zachary Ware. + +.. + +.. bpo: 21629 +.. date: 7324 +.. nonce: 9kZmQl +.. section: Tools/Demos + +Fix Argument Clinic's "--converters" feature. + +.. + +.. bpo: 0 +.. date: 7323 +.. nonce: _-ge-g +.. section: Tools/Demos + +Add support for ``yield from`` to 2to3. + +.. + +.. bpo: 0 +.. date: 7322 +.. nonce: dpFbyZ +.. section: Tools/Demos + +Add support for the PEP 465 matrix multiplication operator to 2to3. + +.. + +.. bpo: 16047 +.. date: 7321 +.. nonce: IsgTzm +.. section: Tools/Demos + +Fix module exception list and __file__ handling in freeze. Patch by Meador +Inge. + +.. + +.. bpo: 11824 +.. date: 7320 +.. nonce: OBWc3T +.. section: Tools/Demos + +Consider ABI tags in freeze. Patch by Meador Inge. + +.. + +.. bpo: 20535 +.. date: 7319 +.. nonce: 0qkvZZ +.. section: Tools/Demos + +PYTHONWARNING no longer affects the run_tests.py script. Patch by Arfrever +Frehtes Taifersar Arahesis. + +.. + +.. bpo: 23260 +.. date: 7318 +.. nonce: aZ5VLH +.. section: Windows + +Update Windows installer + +.. + +.. bpo: 0 +.. date: 7317 +.. nonce: _aEUNt +.. section: Windows + +The bundled version of Tcl/Tk has been updated to 8.6.3. The most visible +result of this change is the addition of new native file dialogs when +running on Windows Vista or newer. See Tcl/Tk's TIP 432 for more +information. Also, this version of Tcl/Tk includes support for Windows 10. + +.. + +.. bpo: 17896 +.. date: 7316 +.. nonce: o79rHM +.. section: Windows + +The Windows build scripts now expect external library sources to be in +``PCbuild\..\externals`` rather than ``PCbuild\..\..``. + +.. + +.. bpo: 17717 +.. date: 7315 +.. nonce: y1zoye +.. section: Windows + +The Windows build scripts now use a copy of NASM pulled from svn.python.org +to build OpenSSL. + +.. + +.. bpo: 21907 +.. date: 7314 +.. nonce: jm1smN +.. section: Windows + +Improved the batch scripts provided for building Python. + +.. + +.. bpo: 22644 +.. date: 7313 +.. nonce: gosBki +.. section: Windows + +The bundled version of OpenSSL has been updated to 1.0.1j. + +.. + +.. bpo: 10747 +.. date: 7312 +.. nonce: LTWhLn +.. section: Windows + +Use versioned labels in the Windows start menu. Patch by Olive Kilburn. + +.. + +.. bpo: 22980 +.. date: 7311 +.. nonce: -UypE5 +.. section: Windows + +.pyd files with a version and platform tag (for example, ".cp35-win32.pyd") +will now be loaded in preference to those without tags. diff --git a/Misc/NEWS.d/3.5.0a2.rst b/Misc/NEWS.d/3.5.0a2.rst new file mode 100644 index 00000000000000..d66e4cbdeb58ea --- /dev/null +++ b/Misc/NEWS.d/3.5.0a2.rst @@ -0,0 +1,406 @@ +.. bpo: 23571 +.. date: 7950 +.. nonce: GTkAkq +.. release date: 2015-03-09 +.. section: Core and Builtins + +PyObject_Call() and PyCFunction_Call() now raise a SystemError if a function +returns a result and raises an exception. The SystemError is chained to the +previous exception. + +.. + +.. bpo: 22524 +.. date: 7949 +.. nonce: Ks6_2x +.. section: Library + +New os.scandir() function, part of the PEP 471: "os.scandir() function -- a +better and faster directory iterator". Patch written by Ben Hoyt. + +.. + +.. bpo: 23103 +.. date: 7948 +.. nonce: I3RLIV +.. section: Library + +Reduced the memory consumption of IPv4Address and IPv6Address. + +.. + +.. bpo: 21793 +.. date: 7947 +.. nonce: GQtYMM +.. section: Library + +BaseHTTPRequestHandler again logs response code as numeric, not as +stringified enum. Patch by Demian Brecht. + +.. + +.. bpo: 23476 +.. date: 7946 +.. nonce: 82QV9I +.. section: Library + +In the ssl module, enable OpenSSL's X509_V_FLAG_TRUSTED_FIRST flag on +certificate stores when it is available. + +.. + +.. bpo: 23576 +.. date: 7945 +.. nonce: 98F-PP +.. section: Library + +Avoid stalling in SSL reads when EOF has been reached in the SSL layer but +the underlying connection hasn't been closed. + +.. + +.. bpo: 23504 +.. date: 7944 +.. nonce: o31h5I +.. section: Library + +Added an __all__ to the types module. + +.. + +.. bpo: 23563 +.. date: 7943 +.. nonce: iQB-ba +.. section: Library + +Optimized utility functions in urllib.parse. + +.. + +.. bpo: 7830 +.. date: 7942 +.. nonce: irvPdC +.. section: Library + +Flatten nested functools.partial. + +.. + +.. bpo: 20204 +.. date: 7941 +.. nonce: DorA4b +.. section: Library + +Added the __module__ attribute to _tkinter classes. + +.. + +.. bpo: 19980 +.. date: 7940 +.. nonce: whwzL_ +.. section: Library + +Improved help() for non-recognized strings. help('') now shows the help on +str. help('help') now shows the help on help(). Original patch by Mark +Lawrence. + +.. + +.. bpo: 23521 +.. date: 7939 +.. nonce: HvwFfd +.. section: Library + +Corrected pure python implementation of timedelta division. + +Eliminated OverflowError from ``timedelta * float`` for some floats; +Corrected rounding in timedlta true division. + +.. + +.. bpo: 21619 +.. date: 7938 +.. nonce: uL0SZh +.. section: Library + +Popen objects no longer leave a zombie after exit in the with statement if +the pipe was broken. Patch by Martin Panter. + +.. + +.. bpo: 22936 +.. date: 7937 +.. nonce: JrhGYd +.. section: Library + +Make it possible to show local variables in tracebacks for both the +traceback module and unittest. + +.. + +.. bpo: 15955 +.. date: 7936 +.. nonce: _8nYPy +.. section: Library + +Add an option to limit the output size in bz2.decompress(). Patch by +Nikolaus Rath. + +.. + +.. bpo: 6639 +.. date: 7935 +.. nonce: rmjUmG +.. section: Library + +Module-level turtle functions no longer raise TclError after closing the +window. + +.. + +.. bpo: 814253 +.. date: 7934 +.. nonce: icZb-I +.. section: Library + +Group references and conditional group references now work in lookbehind +assertions in regular expressions. (See also: bpo-9179) + +.. + +.. bpo: 23215 +.. date: 7933 +.. nonce: VHVSVX +.. section: Library + +Multibyte codecs with custom error handlers that ignores errors consumed too +much memory and raised SystemError or MemoryError. Original patch by Aleksi +Torhamo. + +.. + +.. bpo: 5700 +.. date: 7932 +.. nonce: iA5yzL +.. section: Library + +io.FileIO() called flush() after closing the file. flush() was not called in +close() if closefd=False. + +.. + +.. bpo: 23374 +.. date: 7931 +.. nonce: 8A9LuZ +.. section: Library + +Fixed pydoc failure with non-ASCII files when stdout encoding differs from +file system encoding (e.g. on Mac OS). + +.. + +.. bpo: 23481 +.. date: 7930 +.. nonce: ZWwliG +.. section: Library + +Remove RC4 from the SSL module's default cipher list. + +.. + +.. bpo: 21548 +.. date: 7929 +.. nonce: CmO_Yh +.. section: Library + +Fix pydoc.synopsis() and pydoc.apropos() on modules with empty docstrings. + +.. + +.. bpo: 22885 +.. date: 7928 +.. nonce: p8FnYk +.. section: Library + +Fixed arbitrary code execution vulnerability in the dbm.dumb module. +Original patch by Claudiu Popa. + +.. + +.. bpo: 23239 +.. date: 7927 +.. nonce: PGUq7T +.. section: Library + +ssl.match_hostname() now supports matching of IP addresses. + +.. + +.. bpo: 23146 +.. date: 7926 +.. nonce: PW-O3u +.. section: Library + +Fix mishandling of absolute Windows paths with forward slashes in pathlib. + +.. + +.. bpo: 23096 +.. date: 7925 +.. nonce: Ftrmf3 +.. section: Library + +Pickle representation of floats with protocol 0 now is the same for both +Python and C implementations. + +.. + +.. bpo: 19105 +.. date: 7924 +.. nonce: ZK07Ff +.. section: Library + +pprint now more efficiently uses free space at the right. + +.. + +.. bpo: 14910 +.. date: 7923 +.. nonce: zueIhP +.. section: Library + +Add allow_abbrev parameter to argparse.ArgumentParser. Patch by Jonathan +Paugh, Steven Bethard, paul j3 and Daniel Eriksson. + +.. + +.. bpo: 21717 +.. date: 7922 +.. nonce: Knut81 +.. section: Library + +tarfile.open() now supports 'x' (exclusive creation) mode. + +.. + +.. bpo: 23344 +.. date: 7921 +.. nonce: ieu8C1 +.. section: Library + +marshal.dumps() is now 20-25% faster on average. + +.. + +.. bpo: 20416 +.. date: 7920 +.. nonce: cwEgkL +.. section: Library + +marshal.dumps() with protocols 3 and 4 is now 40-50% faster on average. + +.. + +.. bpo: 23421 +.. date: 7919 +.. nonce: eckzoV +.. section: Library + +Fixed compression in tarfile CLI. Patch by wdv4758h. + +.. + +.. bpo: 23367 +.. date: 7918 +.. nonce: kHnFiz +.. section: Library + +Fix possible overflows in the unicodedata module. + +.. + +.. bpo: 23361 +.. date: 7917 +.. nonce: I_w0-z +.. section: Library + +Fix possible overflow in Windows subprocess creation code. + +.. + +.. bpo: 0 +.. date: 7916 +.. nonce: sfmjTs +.. section: Library + +logging.handlers.QueueListener now takes a respect_handler_level keyword +argument which, if set to True, will pass messages to handlers taking +handler levels into account. + +.. + +.. bpo: 19705 +.. date: 7915 +.. nonce: WLzTRV +.. section: Library + +turtledemo now has a visual sorting algorithm demo. Original patch from +Jason Yeo. + +.. + +.. bpo: 23801 +.. date: 7914 +.. nonce: jyJK3z +.. section: Library + +Fix issue where cgi.FieldStorage did not always ignore the entire preamble +to a multipart body. + +.. + +.. bpo: 23445 +.. date: 7913 +.. nonce: 7fmkYO +.. section: Build + +pydebug builds now use "gcc -Og" where possible, to make the resulting +executable faster. + +.. + +.. bpo: 23686 +.. date: 7912 +.. nonce: B7jDXY +.. section: Build + +Update OS X 10.5 installer build to use OpenSSL 1.0.2a. + +.. + +.. bpo: 20204 +.. date: 7911 +.. nonce: M_jcNK +.. section: C API + +Deprecation warning is now raised for builtin types without the __module__ +attribute. + +.. + +.. bpo: 23465 +.. date: 7910 +.. nonce: qBauCy +.. section: Windows + +Implement PEP 486 - Make the Python Launcher aware of virtual environments. +Patch by Paul Moore. + +.. + +.. bpo: 23437 +.. date: 7909 +.. nonce: ro9X8r +.. section: Windows + +Make user scripts directory versioned on Windows. Patch by Paul Moore. diff --git a/Misc/NEWS.d/3.5.0a3.rst b/Misc/NEWS.d/3.5.0a3.rst new file mode 100644 index 00000000000000..b69c4c39ca0010 --- /dev/null +++ b/Misc/NEWS.d/3.5.0a3.rst @@ -0,0 +1,518 @@ +.. bpo: 23573 +.. date: 8002 +.. nonce: ZpM4D- +.. release date: 2015-03-28 +.. section: Core and Builtins + +Increased performance of string search operations (str.find, str.index, +str.count, the in operator, str.split, str.partition) with arguments of +different kinds (UCS1, UCS2, UCS4). + +.. + +.. bpo: 23753 +.. date: 8001 +.. nonce: CREjLC +.. section: Core and Builtins + +Python doesn't support anymore platforms without stat() or fstat(), these +functions are always required. + +.. + +.. bpo: 23681 +.. date: 8000 +.. nonce: kh02TF +.. section: Core and Builtins + +The -b option now affects comparisons of bytes with int. + +.. + +.. bpo: 23632 +.. date: 7999 +.. nonce: UVdIZY +.. section: Core and Builtins + +Memoryviews now allow tuple indexing (including for multi-dimensional +memoryviews). + +.. + +.. bpo: 23192 +.. date: 7998 +.. nonce: QKqdow +.. section: Core and Builtins + +Fixed generator lambdas. Patch by Bruno Cauet. + +.. + +.. bpo: 23629 +.. date: 7997 +.. nonce: r9Mt2C +.. section: Core and Builtins + +Fix the default __sizeof__ implementation for variable-sized objects. + +.. + +.. bpo: 14260 +.. date: 7996 +.. nonce: b5M04V +.. section: Library + +The groupindex attribute of regular expression pattern object now is non- +modifiable mapping. + +.. + +.. bpo: 23792 +.. date: 7995 +.. nonce: Kfm9-f +.. section: Library + +Ignore KeyboardInterrupt when the pydoc pager is active. This mimics the +behavior of the standard unix pagers, and prevents pipepager from shutting +down while the pager itself is still running. + +.. + +.. bpo: 23775 +.. date: 7994 +.. nonce: xKGrSQ +.. section: Library + +pprint() of OrderedDict now outputs the same representation as repr(). + +.. + +.. bpo: 23765 +.. date: 7993 +.. nonce: 2ta_C4 +.. section: Library + +Removed IsBadStringPtr calls in ctypes + +.. + +.. bpo: 22364 +.. date: 7992 +.. nonce: ejtoKl +.. section: Library + +Improved some re error messages using regex for hints. + +.. + +.. bpo: 23742 +.. date: 7991 +.. nonce: _EkAIa +.. section: Library + +ntpath.expandvars() no longer loses unbalanced single quotes. + +.. + +.. bpo: 21717 +.. date: 7990 +.. nonce: pKndpx +.. section: Library + +The zipfile.ZipFile.open function now supports 'x' (exclusive creation) +mode. + +.. + +.. bpo: 21802 +.. date: 7989 +.. nonce: ygSM2A +.. section: Library + +The reader in BufferedRWPair now is closed even when closing writer failed +in BufferedRWPair.close(). + +.. + +.. bpo: 23622 +.. date: 7988 +.. nonce: 9-ZRqj +.. section: Library + +Unknown escapes in regular expressions that consist of ``'\'`` and ASCII +letter now raise a deprecation warning and will be forbidden in Python 3.6. + +.. + +.. bpo: 23671 +.. date: 7987 +.. nonce: zWPm-a +.. section: Library + +string.Template now allows specifying the "self" parameter as a keyword +argument. string.Formatter now allows specifying the "self" and the +"format_string" parameters as keyword arguments. + +.. + +.. bpo: 23502 +.. date: 7986 +.. nonce: AH20IQ +.. section: Library + +The pprint module now supports mapping proxies. + +.. + +.. bpo: 17530 +.. date: 7985 +.. nonce: PUp8rL +.. section: Library + +pprint now wraps long bytes objects and bytearrays. + +.. + +.. bpo: 22687 +.. date: 7984 +.. nonce: zEJPd9 +.. section: Library + +Fixed some corner cases in breaking words in tetxtwrap. Got rid of quadratic +complexity in breaking long words. + +.. + +.. bpo: 4727 +.. date: 7983 +.. nonce: iDQSpi +.. section: Library + +The copy module now uses pickle protocol 4 (PEP 3154) and supports copying +of instances of classes whose __new__ method takes keyword-only arguments. + +.. + +.. bpo: 23491 +.. date: 7982 +.. nonce: P_WKrt +.. section: Library + +Added a zipapp module to support creating executable zip file archives of +Python code. Registered ".pyz" and ".pyzw" extensions on Windows for these +archives (PEP 441). + +.. + +.. bpo: 23657 +.. date: 7981 +.. nonce: y1OaV- +.. section: Library + +Avoid explicit checks for str in zipapp, adding support for pathlib.Path +objects as arguments. + +.. + +.. bpo: 23688 +.. date: 7980 +.. nonce: d6LVy3 +.. section: Library + +Added support of arbitrary bytes-like objects and avoided unnecessary +copying of memoryview in gzip.GzipFile.write(). Original patch by Wolfgang +Maier. + +.. + +.. bpo: 23252 +.. date: 7979 +.. nonce: Goi18g +.. section: Library + +Added support for writing ZIP files to unseekable streams. + +.. + +.. bpo: 23647 +.. date: 7978 +.. nonce: pX2qrx +.. section: Library + +Increase impalib's MAXLINE to accommodate modern mailbox sizes. + +.. + +.. bpo: 23539 +.. date: 7977 +.. nonce: 5BVUim +.. section: Library + +If body is None, http.client.HTTPConnection.request now sets Content-Length +to 0 for PUT, POST, and PATCH headers to avoid 411 errors from some web +servers. + +.. + +.. bpo: 22351 +.. date: 7976 +.. nonce: agB8Y3 +.. section: Library + +The nntplib.NNTP constructor no longer leaves the connection and socket open +until the garbage collector cleans them up. Patch by Martin Panter. + +.. + +.. bpo: 23704 +.. date: 7975 +.. nonce: LTyyxL +.. section: Library + +collections.deque() objects now support methods for index(), insert(), and +copy(). This allows deques to be registered as a MutableSequence and it +improves their substitutability for lists. + +.. + +.. bpo: 23715 +.. date: 7974 +.. nonce: Yap3tU +.. section: Library + +:func:`signal.sigwaitinfo` and :func:`signal.sigtimedwait` are now retried +when interrupted by a signal not in the *sigset* parameter, if the signal +handler does not raise an exception. signal.sigtimedwait() recomputes the +timeout with a monotonic clock when it is retried. + +.. + +.. bpo: 23001 +.. date: 7973 +.. nonce: YSFnam +.. section: Library + +Few functions in modules mmap, ossaudiodev, socket, ssl, and codecs, that +accepted only read-only bytes-like object now accept writable bytes-like +object too. + +.. + +.. bpo: 23646 +.. date: 7972 +.. nonce: Tljc1S +.. section: Library + +If time.sleep() is interrupted by a signal, the sleep is now retried with +the recomputed delay, except if the signal handler raises an exception (PEP +475). + +.. + +.. bpo: 23136 +.. date: 7971 +.. nonce: 1bnpnb +.. section: Library + +_strptime now uniformly handles all days in week 0, including Dec 30 of +previous year. Based on patch by Jim Carroll. + +.. + +.. bpo: 23700 +.. date: 7970 +.. nonce: VfnWwi +.. section: Library + +Iterator of NamedTemporaryFile now keeps a reference to NamedTemporaryFile +instance. Patch by Bohuslav Kabrda. + +.. + +.. bpo: 22903 +.. date: 7969 +.. nonce: 2GjTHY +.. section: Library + +The fake test case created by unittest.loader when it fails importing a test +module is now picklable. + +.. + +.. bpo: 22181 +.. date: 7968 +.. nonce: 7mnxea +.. section: Library + +On Linux, os.urandom() now uses the new getrandom() syscall if available, +syscall introduced in the Linux kernel 3.17. It is more reliable and more +secure, because it avoids the need of a file descriptor and waits until the +kernel has enough entropy. + +.. + +.. bpo: 2211 +.. date: 7967 +.. nonce: 17Iz5U +.. section: Library + +Updated the implementation of the http.cookies.Morsel class. Setting +attributes key, value and coded_value directly now is deprecated. update() +and setdefault() now transform and check keys. Comparing for equality now +takes into account attributes key, value and coded_value. copy() now returns +a Morsel, not a dict. repr() now contains all attributes. Optimized +checking keys and quoting values. Added new tests. Original patch by Demian +Brecht. + +.. + +.. bpo: 18983 +.. date: 7966 +.. nonce: vF4i2S +.. section: Library + +Allow selection of output units in timeit. Patch by Julian Gindi. + +.. + +.. bpo: 23631 +.. date: 7965 +.. nonce: GfSqNI +.. section: Library + +Fix traceback.format_list when a traceback has been mutated. + +.. + +.. bpo: 23568 +.. date: 7964 +.. nonce: ffzJc7 +.. section: Library + +Add rdivmod support to MagicMock() objects. Patch by Håkan Lövdahl. + +.. + +.. bpo: 2052 +.. date: 7963 +.. nonce: ujNgna +.. section: Library + +Add charset parameter to HtmlDiff.make_file(). + +.. + +.. bpo: 23668 +.. date: 7962 +.. nonce: nF_jnN +.. section: Library + +Support os.truncate and os.ftruncate on Windows. + +.. + +.. bpo: 23138 +.. date: 7961 +.. nonce: 4vMoMZ +.. section: Library + +Fixed parsing cookies with absent keys or values in cookiejar. Patch by +Demian Brecht. + +.. + +.. bpo: 23051 +.. date: 7960 +.. nonce: Vi5tCZ +.. section: Library + +multiprocessing.Pool methods imap() and imap_unordered() now handle +exceptions raised by an iterator. Patch by Alon Diamant and Davin Potts. + +.. + +.. bpo: 23581 +.. date: 7959 +.. nonce: D4Lknl +.. section: Library + +Add matmul support to MagicMock. Patch by Håkan Lövdahl. + +.. + +.. bpo: 23566 +.. date: 7958 +.. nonce: F6LSyk +.. section: Library + +enable(), register(), dump_traceback() and dump_traceback_later() functions +of faulthandler now accept file descriptors. Patch by Wei Wu. + +.. + +.. bpo: 22928 +.. date: 7957 +.. nonce: q2TmY0 +.. section: Library + +Disabled HTTP header injections in http.client. Original patch by Demian +Brecht. + +.. + +.. bpo: 23615 +.. date: 7956 +.. nonce: 5Kx9k5 +.. section: Library + +Modules bz2, tarfile and tokenize now can be reloaded with imp.reload(). +Patch by Thomas Kluyver. + +.. + +.. bpo: 23605 +.. date: 7955 +.. nonce: JUOA_X +.. section: Library + +os.walk() now calls os.scandir() instead of os.listdir(). The usage of +os.scandir() reduces the number of calls to os.stat(). Initial patch written +by Ben Hoyt. + +.. + +.. bpo: 23585 +.. date: 7954 +.. nonce: DTIIoI +.. section: Build + +make patchcheck will ensure the interpreter is built. + +.. + +.. bpo: 23583 +.. date: 7953 +.. nonce: bY8AbM +.. section: Tests + +Added tests for standard IO streams in IDLE. + +.. + +.. bpo: 22289 +.. date: 7952 +.. nonce: ybGcC- +.. section: Tests + +Prevent test_urllib2net failures due to ftp connection timeout. + +.. + +.. bpo: 22826 +.. date: 7951 +.. nonce: 3bcoDL +.. section: Tools/Demos + +The result of open() in Tools/freeze/bkfile.py is now better compatible with +regular files (in particular it now supports the context management +protocol). diff --git a/Misc/NEWS.d/3.5.0a4.rst b/Misc/NEWS.d/3.5.0a4.rst new file mode 100644 index 00000000000000..0707cf6a4c8f9b --- /dev/null +++ b/Misc/NEWS.d/3.5.0a4.rst @@ -0,0 +1,665 @@ +.. bpo: 22980 +.. date: 8069 +.. nonce: Lu_y6y +.. release date: 2015-04-19 +.. section: Core and Builtins + +Under Linux, GNU/KFreeBSD and the Hurd, C extensions now include the +architecture triplet in the extension name, to make it easy to test builds +for different ABIs in the same working tree. Under OS X, the extension name +now includes PEP 3149-style information. + +.. + +.. bpo: 22631 +.. date: 8068 +.. nonce: nTx_ZF +.. section: Core and Builtins + +Added Linux-specific socket constant CAN_RAW_FD_FRAMES. Patch courtesy of +Joe Jevnik. + +.. + +.. bpo: 23731 +.. date: 8067 +.. nonce: FOXb37 +.. section: Core and Builtins + +Implement PEP 488: removal of .pyo files. + +.. + +.. bpo: 23726 +.. date: 8066 +.. nonce: ZopTQ0 +.. section: Core and Builtins + +Don't enable GC for user subclasses of non-GC types that don't add any new +fields. Patch by Eugene Toder. + +.. + +.. bpo: 23309 +.. date: 8065 +.. nonce: Wfnsnz +.. section: Core and Builtins + +Avoid a deadlock at shutdown if a daemon thread is aborted while it is +holding a lock to a buffered I/O object, and the main thread tries to use +the same I/O object (typically stdout or stderr). A fatal error is emitted +instead. + +.. + +.. bpo: 22977 +.. date: 8064 +.. nonce: hutEse +.. section: Core and Builtins + +Fixed formatting Windows error messages on Wine. Patch by Martin Panter. + +.. + +.. bpo: 23466 +.. date: 8063 +.. nonce: KhMltK +.. section: Core and Builtins + +%c, %o, %x, and %X in bytes formatting now raise TypeError on non-integer +input. + +.. + +.. bpo: 24044 +.. date: 8062 +.. nonce: H7vb6- +.. section: Core and Builtins + +Fix possible null pointer dereference in list.sort in out of memory +conditions. + +.. + +.. bpo: 21354 +.. date: 8061 +.. nonce: ZZTe1E +.. section: Core and Builtins + +PyCFunction_New function is exposed by python DLL again. + +.. + +.. bpo: 23840 +.. date: 8060 +.. nonce: mtSbqO +.. section: Library + +tokenize.open() now closes the temporary binary file on error to fix a +resource warning. + +.. + +.. bpo: 16914 +.. date: 8059 +.. nonce: GrP2Jr +.. section: Library + +new debuglevel 2 in smtplib adds timestamps to debug output. + +.. + +.. bpo: 7159 +.. date: 8058 +.. nonce: KCgOUm +.. section: Library + +urllib.request now supports sending auth credentials automatically after the +first 401. This enhancement is a superset of the enhancement from issue +#19494 and supersedes that change. + +.. + +.. bpo: 23703 +.. date: 8057 +.. nonce: kYybxm +.. section: Library + +Fix a regression in urljoin() introduced in 901e4e52b20a. Patch by Demian +Brecht. + +.. + +.. bpo: 4254 +.. date: 8056 +.. nonce: eUC_2H +.. section: Library + +Adds _curses.update_lines_cols(). Patch by Arnon Yaari + +.. + +.. bpo: 19933 +.. date: 8055 +.. nonce: Qq8utk +.. section: Library + +Provide default argument for ndigits in round. Patch by Vajrasky Kok. + +.. + +.. bpo: 23193 +.. date: 8054 +.. nonce: n5ahcG +.. section: Library + +Add a numeric_owner parameter to tarfile.TarFile.extract and +tarfile.TarFile.extractall. Patch by Michael Vogt and Eric Smith. + +.. + +.. bpo: 23342 +.. date: 8053 +.. nonce: CbSzYI +.. section: Library + +Add a subprocess.run() function than returns a CalledProcess instance for a +more consistent API than the existing call* functions. + +.. + +.. bpo: 21217 +.. date: 8052 +.. nonce: TkFTlk +.. section: Library + +inspect.getsourcelines() now tries to compute the start and end lines from +the code object, fixing an issue when a lambda function is used as decorator +argument. Patch by Thomas Ballinger and Allison Kaptur. + +.. + +.. bpo: 24521 +.. date: 8051 +.. nonce: bn4U-y +.. section: Library + +Fix possible integer overflows in the pickle module. + +.. + +.. bpo: 22931 +.. date: 8050 +.. nonce: 4CuWYD +.. section: Library + +Allow '[' and ']' in cookie values. + +.. + +.. bpo: 0 +.. date: 8049 +.. nonce: fgX8Qe +.. section: Library + +The keywords attribute of functools.partial is now always a dictionary. + +.. + +.. bpo: 23811 +.. date: 8048 +.. nonce: B6tzf9 +.. section: Library + +Add missing newline to the PyCompileError error message. Patch by Alex +Shkop. + +.. + +.. bpo: 21116 +.. date: 8047 +.. nonce: Orft3K +.. section: Library + +Avoid blowing memory when allocating a multiprocessing shared array that's +larger than 50% of the available RAM. Patch by Médéric Boquien. + +.. + +.. bpo: 22982 +.. date: 8046 +.. nonce: xYmG62 +.. section: Library + +Improve BOM handling when seeking to multiple positions of a writable text +file. + +.. + +.. bpo: 23464 +.. date: 8045 +.. nonce: _XGkBk +.. section: Library + +Removed deprecated asyncio JoinableQueue. + +.. + +.. bpo: 23529 +.. date: 8044 +.. nonce: Hr7AHH +.. section: Library + +Limit the size of decompressed data when reading from GzipFile, BZ2File or +LZMAFile. This defeats denial of service attacks using compressed bombs +(i.e. compressed payloads which decompress to a huge size). Patch by Martin +Panter and Nikolaus Rath. + +.. + +.. bpo: 21859 +.. date: 8043 +.. nonce: GYrUNP +.. section: Library + +Added Python implementation of io.FileIO. + +.. + +.. bpo: 23865 +.. date: 8042 +.. nonce: PtSLgU +.. section: Library + +close() methods in multiple modules now are idempotent and more robust at +shutdown. If they need to release multiple resources, all are released even +if errors occur. + +.. + +.. bpo: 23400 +.. date: 8041 +.. nonce: JSh9Z3 +.. section: Library + +Raise same exception on both Python 2 and 3 if sem_open is not available. +Patch by Davin Potts. + +.. + +.. bpo: 10838 +.. date: 8040 +.. nonce: p9tSPC +.. section: Library + +The subprocess now module includes SubprocessError and TimeoutError in its +list of exported names for the users wild enough to use ``from subprocess +import *``. + +.. + +.. bpo: 23411 +.. date: 8039 +.. nonce: 0im3Qw +.. section: Library + +Added DefragResult, ParseResult, SplitResult, DefragResultBytes, +ParseResultBytes, and SplitResultBytes to urllib.parse.__all__. Patch by +Martin Panter. + +.. + +.. bpo: 23881 +.. date: 8038 +.. nonce: yZjl4b +.. section: Library + +urllib.request.ftpwrapper constructor now closes the socket if the FTP +connection failed to fix a ResourceWarning. + +.. + +.. bpo: 23853 +.. date: 8037 +.. nonce: mNY1eI +.. section: Library + +:meth:`socket.socket.sendall` does no more reset the socket timeout each +time data is sent successfully. The socket timeout is now the maximum total +duration to send all data. + +.. + +.. bpo: 22721 +.. date: 8036 +.. nonce: MVfBL9 +.. section: Library + +An order of multiline pprint output of set or dict containing orderable and +non-orderable elements no longer depends on iteration order of set or dict. + +.. + +.. bpo: 15133 +.. date: 8035 +.. nonce: C0QfV8 +.. section: Library + +_tkinter.tkapp.getboolean() now supports Tcl_Obj and always returns bool. +tkinter.BooleanVar now validates input values (accepted bool, int, str, and +Tcl_Obj). tkinter.BooleanVar.get() now always returns bool. + +.. + +.. bpo: 10590 +.. date: 8034 +.. nonce: nkxXfU +.. section: Library + +xml.sax.parseString() now supports string argument. + +.. + +.. bpo: 23338 +.. date: 8033 +.. nonce: ZYMGN1 +.. section: Library + +Fixed formatting ctypes error messages on Cygwin. Patch by Makoto Kato. + +.. + +.. bpo: 15582 +.. date: 8032 +.. nonce: 26wJNk +.. section: Library + +inspect.getdoc() now follows inheritance chains. + +.. + +.. bpo: 2175 +.. date: 8031 +.. nonce: cHiVOp +.. section: Library + +SAX parsers now support a character stream of InputSource object. + +.. + +.. bpo: 16840 +.. date: 8030 +.. nonce: kKIhPm +.. section: Library + +Tkinter now supports 64-bit integers added in Tcl 8.4 and arbitrary +precision integers added in Tcl 8.5. + +.. + +.. bpo: 23834 +.. date: 8029 +.. nonce: fX3TF4 +.. section: Library + +Fix socket.sendto(), use the C Py_ssize_t type to store the result of +sendto() instead of the C int type. + +.. + +.. bpo: 23618 +.. date: 8028 +.. nonce: Of_q5t +.. section: Library + +:meth:`socket.socket.connect` now waits until the connection completes +instead of raising :exc:`InterruptedError` if the connection is interrupted +by signals, signal handlers don't raise an exception and the socket is +blocking or has a timeout. :meth:`socket.socket.connect` still raise +:exc:`InterruptedError` for non-blocking sockets. + +.. + +.. bpo: 21526 +.. date: 8027 +.. nonce: QQEXrR +.. section: Library + +Tkinter now supports new boolean type in Tcl 8.5. + +.. + +.. bpo: 23836 +.. date: 8026 +.. nonce: zrEmlR +.. section: Library + +Fix the faulthandler module to handle reentrant calls to its signal +handlers. + +.. + +.. bpo: 23838 +.. date: 8025 +.. nonce: IX6FPX +.. section: Library + +linecache now clears the cache and returns an empty result on MemoryError. + +.. + +.. bpo: 10395 +.. date: 8024 +.. nonce: fi_lZp +.. section: Library + +Added os.path.commonpath(). Implemented in posixpath and ntpath. Based on +patch by Rafik Draoui. + +.. + +.. bpo: 23611 +.. date: 8023 +.. nonce: QkBJVB +.. section: Library + +Serializing more "lookupable" objects (such as unbound methods or nested +classes) now are supported with pickle protocols < 4. + +.. + +.. bpo: 13583 +.. date: 8022 +.. nonce: -MPBjZ +.. section: Library + +sqlite3.Row now supports slice indexing. + +.. + +.. bpo: 18473 +.. date: 8021 +.. nonce: 89RHm- +.. section: Library + +Fixed 2to3 and 3to2 compatible pickle mappings. Fixed ambigious reverse +mappings. Added many new mappings. Import mapping is no longer applied to +modules already mapped with full name mapping. + +.. + +.. bpo: 23485 +.. date: 8020 +.. nonce: kQWN6L +.. section: Library + +select.select() is now retried automatically with the recomputed timeout +when interrupted by a signal, except if the signal handler raises an +exception. This change is part of the PEP 475. + +.. + +.. bpo: 23752 +.. date: 8019 +.. nonce: 5fbVNb +.. section: Library + +When built from an existing file descriptor, io.FileIO() now only calls +fstat() once. Before fstat() was called twice, which was not necessary. + +.. + +.. bpo: 23704 +.. date: 8018 +.. nonce: Ggjvm8 +.. section: Library + +collections.deque() objects now support __add__, __mul__, and __imul__(). + +.. + +.. bpo: 23171 +.. date: 8017 +.. nonce: b6PBzM +.. section: Library + +csv.Writer.writerow() now supports arbitrary iterables. + +.. + +.. bpo: 23745 +.. date: 8016 +.. nonce: E00Bml +.. section: Library + +The new email header parser now handles duplicate MIME parameter names +without error, similar to how get_param behaves. + +.. + +.. bpo: 22117 +.. date: 8015 +.. nonce: bTO0xx +.. section: Library + +Fix os.utime(), it now rounds the timestamp towards minus infinity (-inf) +instead of rounding towards zero. + +.. + +.. bpo: 23310 +.. date: 8014 +.. nonce: GXmFMR +.. section: Library + +Fix MagicMock's initializer to work with __methods__, just like +configure_mock(). Patch by Kasia Jachim. + +.. + +.. bpo: 23817 +.. date: 8013 +.. nonce: DTmVan +.. section: Build + +FreeBSD now uses "1.0" in the SOVERSION as other operating systems, instead +of just "1". + +.. + +.. bpo: 23501 +.. date: 8012 +.. nonce: iz10e6 +.. section: Build + +Argument Clinic now generates code into separate files by default. + +.. + +.. bpo: 23799 +.. date: 8011 +.. nonce: XU2xDw +.. section: Tests + +Added test.support.start_threads() for running and cleaning up multiple +threads. + +.. + +.. bpo: 22390 +.. date: 8010 +.. nonce: UPVFnq +.. section: Tests + +test.regrtest now emits a warning if temporary files or directories are left +after running a test. + +.. + +.. bpo: 18128 +.. date: 8009 +.. nonce: lx2V5a +.. section: Tools/Demos + +pygettext now uses standard +NNNN format in the POT-Creation-Date header. + +.. + +.. bpo: 23935 +.. date: 8008 +.. nonce: JSYowT +.. section: Tools/Demos + +Argument Clinic's understanding of format units accepting bytes, bytearrays, +and buffers is now consistent with both the documentation and the +implementation. + +.. + +.. bpo: 23944 +.. date: 8007 +.. nonce: Q8ZL2s +.. section: Tools/Demos + +Argument Clinic now wraps long impl prototypes at column 78. + +.. + +.. bpo: 20586 +.. date: 8006 +.. nonce: 7BiEkx +.. section: Tools/Demos + +Argument Clinic now ensures that functions without docstrings have +signatures. + +.. + +.. bpo: 23492 +.. date: 8005 +.. nonce: kjIcQW +.. section: Tools/Demos + +Argument Clinic now generates argument parsing code with PyArg_Parse instead +of PyArg_ParseTuple if possible. + +.. + +.. bpo: 23500 +.. date: 8004 +.. nonce: H6_dX_ +.. section: Tools/Demos + +Argument Clinic is now smarter about generating the "#ifndef" (empty) +definition of the methoddef macro: it's only generated once, even if +Argument Clinic processes the same symbol multiple times, and it's emitted +at the end of all processing rather than immediately after the first use. + +.. + +.. bpo: 23998 +.. date: 8003 +.. nonce: z7mlLW +.. section: C API + +PyImport_ReInitLock() now checks for lock allocation error diff --git a/Misc/NEWS.d/3.5.0b1.rst b/Misc/NEWS.d/3.5.0b1.rst new file mode 100644 index 00000000000000..cdbed923bf4370 --- /dev/null +++ b/Misc/NEWS.d/3.5.0b1.rst @@ -0,0 +1,848 @@ +.. bpo: 24276 +.. date: 8157 +.. nonce: awsxJJ +.. release date: 2015-05-24 +.. section: Core and Builtins + +Fixed optimization of property descriptor getter. + +.. + +.. bpo: 24268 +.. date: 8156 +.. nonce: nS7uea +.. section: Core and Builtins + +PEP 489: Multi-phase extension module initialization. Patch by Petr +Viktorin. + +.. + +.. bpo: 23955 +.. date: 8155 +.. nonce: hBHSaU +.. section: Core and Builtins + +Add pyvenv.cfg option to suppress registry/environment lookup for generating +sys.path on Windows. + +.. + +.. bpo: 24257 +.. date: 8154 +.. nonce: UBxshR +.. section: Core and Builtins + +Fixed system error in the comparison of faked types.SimpleNamespace. + +.. + +.. bpo: 22939 +.. date: 8153 +.. nonce: DWA9ls +.. section: Core and Builtins + +Fixed integer overflow in iterator object. Patch by Clement Rouault. + +.. + +.. bpo: 23985 +.. date: 8152 +.. nonce: eezPxO +.. section: Core and Builtins + +Fix a possible buffer overrun when deleting a slice from the front of a +bytearray and then appending some other bytes data. + +.. + +.. bpo: 24102 +.. date: 8151 +.. nonce: 9T6h3m +.. section: Core and Builtins + +Fixed exception type checking in standard error handlers. + +.. + +.. bpo: 15027 +.. date: 8150 +.. nonce: wi9sCd +.. section: Core and Builtins + +The UTF-32 encoder is now 3x to 7x faster. + +.. + +.. bpo: 23290 +.. date: 8149 +.. nonce: 57aqLU +.. section: Core and Builtins + +Optimize set_merge() for cases where the target is empty. (Contributed by +Serhiy Storchaka.) + +.. + +.. bpo: 2292 +.. date: 8148 +.. nonce: h4sibO +.. section: Core and Builtins + +PEP 448: Additional Unpacking Generalizations. + +.. + +.. bpo: 24096 +.. date: 8147 +.. nonce: a_Rap7 +.. section: Core and Builtins + +Make warnings.warn_explicit more robust against mutation of the +warnings.filters list. + +.. + +.. bpo: 23996 +.. date: 8146 +.. nonce: znqcT8 +.. section: Core and Builtins + +Avoid a crash when a delegated generator raises an unnormalized +StopIteration exception. Patch by Stefan Behnel. + +.. + +.. bpo: 23910 +.. date: 8145 +.. nonce: _gDzaj +.. section: Core and Builtins + +Optimize property() getter calls. Patch by Joe Jevnik. + +.. + +.. bpo: 23911 +.. date: 8144 +.. nonce: 0FnTHk +.. section: Core and Builtins + +Move path-based importlib bootstrap code to a separate frozen module. + +.. + +.. bpo: 24192 +.. date: 8143 +.. nonce: 6ZxJ_R +.. section: Core and Builtins + +Fix namespace package imports. + +.. + +.. bpo: 24022 +.. date: 8142 +.. nonce: 1l8YBm +.. section: Core and Builtins + +Fix tokenizer crash when processing undecodable source code. + +.. + +.. bpo: 9951 +.. date: 8141 +.. nonce: wGztNC +.. section: Core and Builtins + +Added a hex() method to bytes, bytearray, and memoryview. + +.. + +.. bpo: 22906 +.. date: 8140 +.. nonce: WN_kQ6 +.. section: Core and Builtins + +PEP 479: Change StopIteration handling inside generators. + +.. + +.. bpo: 24017 +.. date: 8139 +.. nonce: QJa1SC +.. section: Core and Builtins + +PEP 492: Coroutines with async and await syntax. + +.. + +.. bpo: 14373 +.. date: 8138 +.. nonce: 0sk6kE +.. section: Library + +Added C implementation of functools.lru_cache(). Based on patches by Matt +Joiner and Alexey Kachayev. + +.. + +.. bpo: 24230 +.. date: 8137 +.. nonce: b-kgme +.. section: Library + +The tempfile module now accepts bytes for prefix, suffix and dir parameters +and returns bytes in such situations (matching the os module APIs). + +.. + +.. bpo: 22189 +.. date: 8136 +.. nonce: 8epgat +.. section: Library + +collections.UserString now supports __getnewargs__(), __rmod__(), +casefold(), format_map(), isprintable(), and maketrans(). Patch by Joe +Jevnik. + +.. + +.. bpo: 24244 +.. date: 8135 +.. nonce: OKE_3R +.. section: Library + +Prevents termination when an invalid format string is encountered on Windows +in strftime. + +.. + +.. bpo: 23973 +.. date: 8134 +.. nonce: EK6awi +.. section: Library + +PEP 484: Add the typing module. + +.. + +.. bpo: 23086 +.. date: 8133 +.. nonce: Aix6Nv +.. section: Library + +The collections.abc.Sequence() abstract base class added *start* and *stop* +parameters to the index() mixin. Patch by Devin Jeanpierre. + +.. + +.. bpo: 20035 +.. date: 8132 +.. nonce: UNZzw6 +.. section: Library + +Replaced the ``tkinter._fix`` module used for setting up the Tcl/Tk +environment on Windows with a private function in the ``_tkinter`` module +that makes no permanent changes to the environment. + +.. + +.. bpo: 24257 +.. date: 8131 +.. nonce: L_efq0 +.. section: Library + +Fixed segmentation fault in sqlite3.Row constructor with faked cursor type. + +.. + +.. bpo: 15836 +.. date: 8130 +.. nonce: gU3Rmx +.. section: Library + +assertRaises(), assertRaisesRegex(), assertWarns() and assertWarnsRegex() +assertments now check the type of the first argument to prevent possible +user error. Based on patch by Daniel Wagner-Hall. + +.. + +.. bpo: 9858 +.. date: 8129 +.. nonce: uke9pa +.. section: Library + +Add missing method stubs to _io.RawIOBase. Patch by Laura Rupprecht. + +.. + +.. bpo: 22955 +.. date: 8128 +.. nonce: Jw_B9_ +.. section: Library + +attrgetter, itemgetter and methodcaller objects in the operator module now +support pickling. Added readable and evaluable repr for these objects. +Based on patch by Josh Rosenberg. + +.. + +.. bpo: 22107 +.. date: 8127 +.. nonce: 2F8k4W +.. section: Library + +tempfile.gettempdir() and tempfile.mkdtemp() now try again when a directory +with the chosen name already exists on Windows as well as on Unix. +tempfile.mkstemp() now fails early if parent directory is not valid (not +exists or is a file) on Windows. + +.. + +.. bpo: 23780 +.. date: 8126 +.. nonce: jFPVcN +.. section: Library + +Improved error message in os.path.join() with single argument. + +.. + +.. bpo: 6598 +.. date: 8125 +.. nonce: JdZNDt +.. section: Library + +Increased time precision and random number range in email.utils.make_msgid() +to strengthen the uniqueness of the message ID. + +.. + +.. bpo: 24091 +.. date: 8124 +.. nonce: Jw0-wj +.. section: Library + +Fixed various crashes in corner cases in C implementation of ElementTree. + +.. + +.. bpo: 21931 +.. date: 8123 +.. nonce: t6lGxY +.. section: Library + +msilib.FCICreate() now raises TypeError in the case of a bad argument +instead of a ValueError with a bogus FCI error number. Patch by Jeffrey +Armstrong. + +.. + +.. bpo: 13866 +.. date: 8122 +.. nonce: n5NAj0 +.. section: Library + +*quote_via* argument added to urllib.parse.urlencode. + +.. + +.. bpo: 20098 +.. date: 8121 +.. nonce: Y4otaf +.. section: Library + +New mangle_from policy option for email, default True for compat32, but +False for all other policies. + +.. + +.. bpo: 24211 +.. date: 8120 +.. nonce: j3Afpc +.. section: Library + +The email library now supports RFC 6532: it can generate headers using utf-8 +instead of encoded words. + +.. + +.. bpo: 16314 +.. date: 8119 +.. nonce: Xc4d1O +.. section: Library + +Added support for the LZMA compression in distutils. + +.. + +.. bpo: 21804 +.. date: 8118 +.. nonce: lEhTlc +.. section: Library + +poplib now supports RFC 6856 (UTF8). + +.. + +.. bpo: 18682 +.. date: 8117 +.. nonce: 6Pnfte +.. section: Library + +Optimized pprint functions for builtin scalar types. + +.. + +.. bpo: 22027 +.. date: 8116 +.. nonce: _aeUQS +.. section: Library + +smtplib now supports RFC 6531 (SMTPUTF8). + +.. + +.. bpo: 23488 +.. date: 8115 +.. nonce: 7gs3Cm +.. section: Library + +Random generator objects now consume 2x less memory on 64-bit. + +.. + +.. bpo: 1322 +.. date: 8114 +.. nonce: 495nFL +.. section: Library + +platform.dist() and platform.linux_distribution() functions are now +deprecated. Initial patch by Vajrasky Kok. + +.. + +.. bpo: 22486 +.. date: 8113 +.. nonce: Yxov5m +.. section: Library + +Added the math.gcd() function. The fractions.gcd() function now is +deprecated. Based on patch by Mark Dickinson. + +.. + +.. bpo: 24064 +.. date: 8112 +.. nonce: zXC7OL +.. section: Library + +Property() docstrings are now writeable. (Patch by Berker Peksag.) + +.. + +.. bpo: 22681 +.. date: 8111 +.. nonce: 2rIoA2 +.. section: Library + +Added support for the koi8_t encoding. + +.. + +.. bpo: 22682 +.. date: 8110 +.. nonce: cP4i3L +.. section: Library + +Added support for the kz1048 encoding. + +.. + +.. bpo: 23796 +.. date: 8109 +.. nonce: JJmUnc +.. section: Library + +peek and read1 methods of BufferedReader now raise ValueError if they called +on a closed object. Patch by John Hergenroeder. + +.. + +.. bpo: 21795 +.. date: 8108 +.. nonce: BDLMS4 +.. section: Library + +smtpd now supports the 8BITMIME extension whenever the new *decode_data* +constructor argument is set to False. + +.. + +.. bpo: 24155 +.. date: 8107 +.. nonce: FZx5c2 +.. section: Library + +optimize heapq.heapify() for better cache performance when heapifying large +lists. + +.. + +.. bpo: 21800 +.. date: 8106 +.. nonce: evGSKc +.. section: Library + +imaplib now supports RFC 5161 (enable), RFC 6855 (utf8/internationalized +email) and automatically encodes non-ASCII usernames and passwords to UTF8. + +.. + +.. bpo: 20274 +.. date: 8105 +.. nonce: uVHogg +.. section: Library + +When calling a _sqlite.Connection, it now complains if passed any keyword +arguments. Previously it silently ignored them. + +.. + +.. bpo: 20274 +.. date: 8104 +.. nonce: hBst4M +.. section: Library + +Remove ignored and erroneous "kwargs" parameters from three METH_VARARGS +methods on _sqlite.Connection. + +.. + +.. bpo: 24134 +.. date: 8103 +.. nonce: Ajw0S- +.. section: Library + +assertRaises(), assertRaisesRegex(), assertWarns() and assertWarnsRegex() +checks now emits a deprecation warning when callable is None or keyword +arguments except msg is passed in the context manager mode. + +.. + +.. bpo: 24018 +.. date: 8102 +.. nonce: hk7Rcn +.. section: Library + +Add a collections.abc.Generator abstract base class. Contributed by Stefan +Behnel. + +.. + +.. bpo: 23880 +.. date: 8101 +.. nonce: QtKupC +.. section: Library + +Tkinter's getint() and getdouble() now support Tcl_Obj. Tkinter's +getdouble() now supports any numbers (in particular int). + +.. + +.. bpo: 22619 +.. date: 8100 +.. nonce: 1gJEqV +.. section: Library + +Added negative limit support in the traceback module. Based on patch by +Dmitry Kazakov. + +.. + +.. bpo: 24094 +.. date: 8099 +.. nonce: 7T-u7k +.. section: Library + +Fix possible crash in json.encode with poorly behaved dict subclasses. + +.. + +.. bpo: 9246 +.. date: 8098 +.. nonce: oM-Ikk +.. section: Library + +On POSIX, os.getcwd() now supports paths longer than 1025 bytes. Patch +written by William Orr. + +.. + +.. bpo: 17445 +.. date: 8097 +.. nonce: Z-QYh5 +.. section: Library + +add difflib.diff_bytes() to support comparison of byte strings (fixes a +regression from Python 2). + +.. + +.. bpo: 23917 +.. date: 8096 +.. nonce: uMVPV7 +.. section: Library + +Fall back to sequential compilation when ProcessPoolExecutor doesn't exist. +Patch by Claudiu Popa. + +.. + +.. bpo: 23008 +.. date: 8095 +.. nonce: OZFCd- +.. section: Library + +Fixed resolving attributes with boolean value is False in pydoc. + +.. + +.. bpo: 0 +.. date: 8094 +.. nonce: 6tJNf2 +.. section: Library + +Fix asyncio issue 235: LifoQueue and PriorityQueue's put didn't increment +unfinished tasks (this bug was introduced when JoinableQueue was merged with +Queue). + +.. + +.. bpo: 23908 +.. date: 8093 +.. nonce: ATdNG- +.. section: Library + +os functions now reject paths with embedded null character on Windows +instead of silently truncating them. + +.. + +.. bpo: 23728 +.. date: 8092 +.. nonce: YBmQmV +.. section: Library + +binascii.crc_hqx() could return an integer outside of the range 0-0xffff for +empty data. + +.. + +.. bpo: 23887 +.. date: 8091 +.. nonce: _XpjPN +.. section: Library + +urllib.error.HTTPError now has a proper repr() representation. Patch by +Berker Peksag. + +.. + +.. bpo: 0 +.. date: 8090 +.. nonce: MjNdSC +.. section: Library + +asyncio: New event loop APIs: set_task_factory() and get_task_factory(). + +.. + +.. bpo: 0 +.. date: 8089 +.. nonce: rVcHXp +.. section: Library + +asyncio: async() function is deprecated in favour of ensure_future(). + +.. + +.. bpo: 24178 +.. date: 8088 +.. nonce: -enO4y +.. section: Library + +asyncio.Lock, Condition, Semaphore, and BoundedSemaphore support new 'async +with' syntax. Contributed by Yury Selivanov. + +.. + +.. bpo: 24179 +.. date: 8087 +.. nonce: wDy_WZ +.. section: Library + +Support 'async for' for asyncio.StreamReader. Contributed by Yury Selivanov. + +.. + +.. bpo: 24184 +.. date: 8086 +.. nonce: El74TU +.. section: Library + +Add AsyncIterator and AsyncIterable ABCs to collections.abc. Contributed by +Yury Selivanov. + +.. + +.. bpo: 22547 +.. date: 8085 +.. nonce: _ikCaj +.. section: Library + +Implement informative __repr__ for inspect.BoundArguments. Contributed by +Yury Selivanov. + +.. + +.. bpo: 24190 +.. date: 8084 +.. nonce: 1a3vWW +.. section: Library + +Implement inspect.BoundArgument.apply_defaults() method. Contributed by Yury +Selivanov. + +.. + +.. bpo: 20691 +.. date: 8083 +.. nonce: -raLyf +.. section: Library + +Add 'follow_wrapped' argument to inspect.Signature.from_callable() and +inspect.signature(). Contributed by Yury Selivanov. + +.. + +.. bpo: 24248 +.. date: 8082 +.. nonce: IxWooo +.. section: Library + +Deprecate inspect.Signature.from_function() and +inspect.Signature.from_builtin(). + +.. + +.. bpo: 23898 +.. date: 8081 +.. nonce: OSiZie +.. section: Library + +Fix inspect.classify_class_attrs() to support attributes with overloaded +__eq__ and __bool__. Patch by Mike Bayer. + +.. + +.. bpo: 24298 +.. date: 8080 +.. nonce: u_TaxI +.. section: Library + +Fix inspect.signature() to correctly unwrap wrappers around bound methods. + +.. + +.. bpo: 23184 +.. date: 8079 +.. nonce: G_Cp9v +.. section: IDLE + +remove unused names and imports in idlelib. Initial patch by Al Sweigart. + +.. + +.. bpo: 21520 +.. date: 8078 +.. nonce: FKtvmQ +.. section: Tests + +test_zipfile no longer fails if the word 'bad' appears anywhere in the name +of the current directory. + +.. + +.. bpo: 9517 +.. date: 8077 +.. nonce: W0Ag2V +.. section: Tests + +Move script_helper into the support package. Patch by Christie Wilson. + +.. + +.. bpo: 22155 +.. date: 8076 +.. nonce: 9EbOit +.. section: Documentation + +Add File Handlers subsection with createfilehandler to tkinter doc. Remove +obsolete example from FAQ. Patch by Martin Panter. + +.. + +.. bpo: 24029 +.. date: 8075 +.. nonce: M2Bnks +.. section: Documentation + +Document the name binding behavior for submodule imports. + +.. + +.. bpo: 24077 +.. date: 8074 +.. nonce: 2Og2j- +.. section: Documentation + +Fix typo in man page for -I command option: -s, not -S + +.. + +.. bpo: 24000 +.. date: 8073 +.. nonce: MJyXRr +.. section: Tools/Demos + +Improved Argument Clinic's mapping of converters to legacy "format units". +Updated the documentation to match. + +.. + +.. bpo: 24001 +.. date: 8072 +.. nonce: m74vst +.. section: Tools/Demos + +Argument Clinic converters now use accept={type} instead of types={'type'} +to specify the types the converter accepts. + +.. + +.. bpo: 23330 +.. date: 8071 +.. nonce: LTlKDp +.. section: Tools/Demos + +h2py now supports arbitrary filenames in #include. + +.. + +.. bpo: 24031 +.. date: 8070 +.. nonce: duGo88 +.. section: Tools/Demos + +make patchcheck now supports git checkouts, too. diff --git a/Misc/NEWS.d/3.5.0b2.rst b/Misc/NEWS.d/3.5.0b2.rst new file mode 100644 index 00000000000000..1197b5947f7680 --- /dev/null +++ b/Misc/NEWS.d/3.5.0b2.rst @@ -0,0 +1,104 @@ +.. bpo: 24284 +.. date: 8168 +.. nonce: NvtEnc +.. release date: 2015-05-31 +.. section: Core and Builtins + +The startswith and endswith methods of the str class no longer return True +when finding the empty string and the indexes are completely out of range. + +.. + +.. bpo: 24115 +.. date: 8167 +.. nonce: y9e_MO +.. section: Core and Builtins + +Update uses of PyObject_IsTrue(), PyObject_Not(), PyObject_IsInstance(), +PyObject_RichCompareBool() and _PyDict_Contains() to check for and handle +errors correctly. + +.. + +.. bpo: 24328 +.. date: 8166 +.. nonce: 5gL8or +.. section: Core and Builtins + +Fix importing one character extension modules. + +.. + +.. bpo: 11205 +.. date: 8165 +.. nonce: bikrRP +.. section: Core and Builtins + +In dictionary displays, evaluate the key before the value. + +.. + +.. bpo: 24285 +.. date: 8164 +.. nonce: wvJumr +.. section: Core and Builtins + +Fixed regression that prevented importing extension modules from inside +packages. Patch by Petr Viktorin. + +.. + +.. bpo: 23247 +.. date: 8163 +.. nonce: nN-K74 +.. section: Library + +Fix a crash in the StreamWriter.reset() of CJK codecs. + +.. + +.. bpo: 24270 +.. date: 8162 +.. nonce: M2rJNs +.. section: Library + +Add math.isclose() and cmath.isclose() functions as per PEP 485. Contributed +by Chris Barker and Tal Einat. + +.. + +.. bpo: 5633 +.. date: 8161 +.. nonce: JNzKZq +.. section: Library + +Fixed timeit when the statement is a string and the setup is not. + +.. + +.. bpo: 24326 +.. date: 8160 +.. nonce: 4t_6Gy +.. section: Library + +Fixed audioop.ratecv() with non-default weightB argument. Original patch by +David Moore. + +.. + +.. bpo: 16991 +.. date: 8159 +.. nonce: 19_Zmj +.. section: Library + +Add a C implementation of OrderedDict. + +.. + +.. bpo: 23934 +.. date: 8158 +.. nonce: esb-45 +.. section: Library + +Fix inspect.signature to fail correctly for builtin types lacking signature +information. Initial patch by James Powell. diff --git a/Misc/NEWS.d/3.5.0b3.rst b/Misc/NEWS.d/3.5.0b3.rst new file mode 100644 index 00000000000000..6b72d4b177e7e9 --- /dev/null +++ b/Misc/NEWS.d/3.5.0b3.rst @@ -0,0 +1,273 @@ +.. bpo: 24467 +.. date: 8196 +.. nonce: BAJ80- +.. release date: 2015-07-05 +.. section: Core and Builtins + +Fixed possible buffer over-read in bytearray. The bytearray object now +always allocates place for trailing null byte and it's buffer now is always +null-terminated. + +.. + +.. bpo: 0 +.. date: 8195 +.. nonce: 944IUY +.. section: Core and Builtins + +Upgrade to Unicode 8.0.0. + +.. + +.. bpo: 24345 +.. date: 8194 +.. nonce: fVcTaB +.. section: Core and Builtins + +Add Py_tp_finalize slot for the stable ABI. + +.. + +.. bpo: 24400 +.. date: 8193 +.. nonce: 2mNeD8 +.. section: Core and Builtins + +Introduce a distinct type for PEP 492 coroutines; add types.CoroutineType, +inspect.getcoroutinestate, inspect.getcoroutinelocals; coroutines no longer +use CO_GENERATOR flag; sys.set_coroutine_wrapper works only for 'async def' +coroutines; inspect.iscoroutine no longer uses collections.abc.Coroutine, +it's intended to test for pure 'async def' coroutines only; add new opcode: +GET_YIELD_FROM_ITER; fix generators wrapper used in types.coroutine to be +instance of collections.abc.Generator; collections.abc.Awaitable and +collections.abc.Coroutine can no longer be used to detect generator-based +coroutines--use inspect.isawaitable instead. + +.. + +.. bpo: 24450 +.. date: 8192 +.. nonce: lF0S5c +.. section: Core and Builtins + +Add gi_yieldfrom to generators and cr_await to coroutines. Contributed by +Benno Leslie and Yury Selivanov. + +.. + +.. bpo: 19235 +.. date: 8191 +.. nonce: 0kW4n5 +.. section: Core and Builtins + +Add new RecursionError exception. Patch by Georg Brandl. + +.. + +.. bpo: 21750 +.. date: 8190 +.. nonce: _Ycvgi +.. section: Library + +mock_open.read_data can now be read from each instance, as it could in +Python 3.3. + +.. + +.. bpo: 24552 +.. date: 8189 +.. nonce: VTO6sf +.. section: Library + +Fix use after free in an error case of the _pickle module. + +.. + +.. bpo: 24514 +.. date: 8188 +.. nonce: _xRb2r +.. section: Library + +tarfile now tolerates number fields consisting of only whitespace. + +.. + +.. bpo: 19176 +.. date: 8187 +.. nonce: 8V6nOK +.. section: Library + +Fixed doctype() related bugs in C implementation of ElementTree. A +deprecation warning no longer issued by XMLParser subclass with default +doctype() method. Direct call of doctype() now issues a warning. Parser's +doctype() now is not called if target's doctype() is called. Based on patch +by Martin Panter. + +.. + +.. bpo: 20387 +.. date: 8186 +.. nonce: aAbWbQ +.. section: Library + +Restore semantic round-trip correctness in tokenize/untokenize for tab- +indented blocks. + +.. + +.. bpo: 24456 +.. date: 8185 +.. nonce: yYSd2u +.. section: Library + +Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() functions of +the audioop module. + +.. + +.. bpo: 24336 +.. date: 8184 +.. nonce: 4a5y1m +.. section: Library + +The contextmanager decorator now works with functions with keyword arguments +called "func" and "self". Patch by Martin Panter. + +.. + +.. bpo: 24522 +.. date: 8183 +.. nonce: PkcqCA +.. section: Library + +Fix possible integer overflow in json accelerator module. + +.. + +.. bpo: 24489 +.. date: 8182 +.. nonce: GJnMcW +.. section: Library + +ensure a previously set C errno doesn't disturb cmath.polar(). + +.. + +.. bpo: 24408 +.. date: 8181 +.. nonce: vPb5UK +.. section: Library + +Fixed AttributeError in measure() and metrics() methods of tkinter.Font. + +.. + +.. bpo: 14373 +.. date: 8180 +.. nonce: CTYZ4J +.. section: Library + +C implementation of functools.lru_cache() now can be used with methods. + +.. + +.. bpo: 24347 +.. date: 8179 +.. nonce: CPPDb8 +.. section: Library + +Set KeyError if PyDict_GetItemWithError returns NULL. + +.. + +.. bpo: 24348 +.. date: 8178 +.. nonce: U11rhr +.. section: Library + +Drop superfluous incref/decref. + +.. + +.. bpo: 24359 +.. date: 8177 +.. nonce: -IRNG9 +.. section: Library + +Check for changed OrderedDict size during iteration. + +.. + +.. bpo: 24368 +.. date: 8176 +.. nonce: 550kDT +.. section: Library + +Support keyword arguments in OrderedDict methods. + +.. + +.. bpo: 24362 +.. date: 8175 +.. nonce: cHYce5 +.. section: Library + +Simplify the C OrderedDict fast nodes resize logic. + +.. + +.. bpo: 24377 +.. date: 8174 +.. nonce: Gp1Bqr +.. section: Library + +Fix a ref leak in OrderedDict.__repr__. + +.. + +.. bpo: 24369 +.. date: 8173 +.. nonce: qFl7lZ +.. section: Library + +Defend against key-changes during iteration. + +.. + +.. bpo: 24373 +.. date: 8172 +.. nonce: 6TL2XG +.. section: Tests + +_testmultiphase and xxlimited now use tp_traverse and tp_finalize to avoid +reference leaks encountered when combining tp_dealloc with PyType_FromSpec +(see issue #16690 for details) + +.. + +.. bpo: 24458 +.. date: 8171 +.. nonce: 1egApX +.. section: Documentation + +Update documentation to cover multi-phase initialization for extension +modules (PEP 489). Patch by Petr Viktorin. + +.. + +.. bpo: 24351 +.. date: 8170 +.. nonce: XeSVl5 +.. section: Documentation + +Clarify what is meant by "identifier" in the context of string.Template +instances. + +.. + +.. bpo: 24432 +.. date: 8169 +.. nonce: IvUSiN +.. section: Build + +Update Windows builds and OS X 10.5 installer to use OpenSSL 1.0.2c. diff --git a/Misc/NEWS.d/3.5.0b4.rst b/Misc/NEWS.d/3.5.0b4.rst new file mode 100644 index 00000000000000..6f115483567e22 --- /dev/null +++ b/Misc/NEWS.d/3.5.0b4.rst @@ -0,0 +1,255 @@ +.. bpo: 23573 +.. date: 8223 +.. nonce: HdJPs7 +.. release date: 2015-07-26 +.. section: Core and Builtins + +Restored optimization of bytes.rfind() and bytearray.rfind() for single-byte +argument on Linux. + +.. + +.. bpo: 24569 +.. date: 8222 +.. nonce: bqh6PQ +.. section: Core and Builtins + +Make PEP 448 dictionary evaluation more consistent. + +.. + +.. bpo: 24583 +.. date: 8221 +.. nonce: Ooq0Tn +.. section: Core and Builtins + +Fix crash when set is mutated while being updated. + +.. + +.. bpo: 24407 +.. date: 8220 +.. nonce: GmCBB3 +.. section: Core and Builtins + +Fix crash when dict is mutated while being updated. + +.. + +.. bpo: 24619 +.. date: 8219 +.. nonce: cnfZGo +.. section: Core and Builtins + +New approach for tokenizing async/await. As a consequence, it is now +possible to have one-line 'async def foo(): await ..' functions. + +.. + +.. bpo: 24687 +.. date: 8218 +.. nonce: 0UaXFe +.. section: Core and Builtins + +Plug refleak on SyntaxError in function parameters annotations. + +.. + +.. bpo: 15944 +.. date: 8217 +.. nonce: 4GuwqX +.. section: Core and Builtins + +memoryview: Allow arbitrary formats when casting to bytes. Patch by Martin +Panter. + +.. + +.. bpo: 23441 +.. date: 8216 +.. nonce: JXt2Yt +.. section: Library + +rcompleter now prints a tab character instead of displaying possible +completions for an empty word. Initial patch by Martin Sekera. + +.. + +.. bpo: 24683 +.. date: 8215 +.. nonce: aJdWEv +.. section: Library + +Fixed crashes in _json functions called with arguments of inappropriate +type. + +.. + +.. bpo: 21697 +.. date: 8214 +.. nonce: jpATha +.. section: Library + +shutil.copytree() now correctly handles symbolic links that point to +directories. Patch by Eduardo Seabra and Thomas Kluyver. + +.. + +.. bpo: 14373 +.. date: 8213 +.. nonce: Je0yDg +.. section: Library + +Fixed segmentation fault when gc.collect() is called during constructing +lru_cache (C implementation). + +.. + +.. bpo: 24695 +.. date: 8212 +.. nonce: QjZzFb +.. section: Library + +Fix a regression in traceback.print_exception(). If exc_traceback is None +we shouldn't print a traceback header like described in the documentation. + +.. + +.. bpo: 24620 +.. date: 8211 +.. nonce: rrnxB- +.. section: Library + +Random.setstate() now validates the value of state last element. + +.. + +.. bpo: 22485 +.. date: 8210 +.. nonce: HvJf6T +.. section: Library + +Fixed an issue that caused `inspect.getsource` to return incorrect results +on nested functions. + +.. + +.. bpo: 22153 +.. date: 8209 +.. nonce: 6n6yld +.. section: Library + +Improve unittest docs. Patch from Martin Panter and evilzero. + +.. + +.. bpo: 24580 +.. date: 8208 +.. nonce: AGi4Gm +.. section: Library + +Symbolic group references to open group in re patterns now are explicitly +forbidden as well as numeric group references. + +.. + +.. bpo: 24206 +.. date: 8207 +.. nonce: ffkVHH +.. section: Library + +Fixed __eq__ and __ne__ methods of inspect classes. + +.. + +.. bpo: 24631 +.. date: 8206 +.. nonce: uljPxM +.. section: Library + +Fixed regression in the timeit module with multiline setup. + +.. + +.. bpo: 18622 +.. date: 8205 +.. nonce: i6nCCW +.. section: Library + +unittest.mock.mock_open().reset_mock would recurse infinitely. Patch from +Nicola Palumbo and Laurent De Buyst. + +.. + +.. bpo: 23661 +.. date: 8204 +.. nonce: 5VHJmh +.. section: Library + +unittest.mock side_effects can now be exceptions again. This was a +regression vs Python 3.4. Patch from Ignacio Rossi + +.. + +.. bpo: 24608 +.. date: 8203 +.. nonce: 0TndL0 +.. section: Library + +chunk.Chunk.read() now always returns bytes, not str. + +.. + +.. bpo: 18684 +.. date: 8202 +.. nonce: S2es0F +.. section: Library + +Fixed reading out of the buffer in the re module. + +.. + +.. bpo: 24259 +.. date: 8201 +.. nonce: vMAi1A +.. section: Library + +tarfile now raises a ReadError if an archive is truncated inside a data +segment. + +.. + +.. bpo: 15014 +.. date: 8200 +.. nonce: hwXwCH +.. section: Library + +SMTP.auth() and SMTP.login() now support RFC 4954's optional initial- +response argument to the SMTP AUTH command. + +.. + +.. bpo: 24669 +.. date: 8199 +.. nonce: kFThK0 +.. section: Library + +Fix inspect.getsource() for 'async def' functions. Patch by Kai Groner. + +.. + +.. bpo: 24688 +.. date: 8198 +.. nonce: -yWfcO +.. section: Library + +ast.get_docstring() for 'async def' functions. + +.. + +.. bpo: 24603 +.. date: 8197 +.. nonce: PyHyF5 +.. section: Build + +Update Windows builds and OS X 10.5 installer to use OpenSSL 1.0.2d. diff --git a/Misc/NEWS.d/3.5.0rc1.rst b/Misc/NEWS.d/3.5.0rc1.rst new file mode 100644 index 00000000000000..859e60247a2d82 --- /dev/null +++ b/Misc/NEWS.d/3.5.0rc1.rst @@ -0,0 +1,241 @@ +.. bpo: 24667 +.. date: 8248 +.. nonce: tdwszf +.. release date: 2015-08-09 +.. section: Core and Builtins + +Resize odict in all cases that the underlying dict resizes. + +.. + +.. bpo: 24824 +.. date: 8247 +.. nonce: Eoc4lq +.. section: Library + +Signatures of codecs.encode() and codecs.decode() now are compatible with +pydoc. + +.. + +.. bpo: 24634 +.. date: 8246 +.. nonce: 7bnVgr +.. section: Library + +Importing uuid should not try to load libc on Windows + +.. + +.. bpo: 24798 +.. date: 8245 +.. nonce: zDXL5R +.. section: Library + +_msvccompiler.py doesn't properly support manifests + +.. + +.. bpo: 4395 +.. date: 8244 +.. nonce: JpT0k7 +.. section: Library + +Better testing and documentation of binary operators. Patch by Martin +Panter. + +.. + +.. bpo: 23973 +.. date: 8243 +.. nonce: wT59Vh +.. section: Library + +Update typing.py from GitHub repo. + +.. + +.. bpo: 23004 +.. date: 8242 +.. nonce: xswcPm +.. section: Library + +mock_open() now reads binary data correctly when the type of read_data is +bytes. Initial patch by Aaron Hill. + +.. + +.. bpo: 23888 +.. date: 8241 +.. nonce: 7gw4oO +.. section: Library + +Handle fractional time in cookie expiry. Patch by ssh. + +.. + +.. bpo: 23652 +.. date: 8240 +.. nonce: DKQ_7t +.. section: Library + +Make it possible to compile the select module against the libc headers from +the Linux Standard Base, which do not include some EPOLL macros. Patch by +Matt Frank. + +.. + +.. bpo: 22932 +.. date: 8239 +.. nonce: mPclSJ +.. section: Library + +Fix timezones in email.utils.formatdate. Patch from Dmitry Shachnev. + +.. + +.. bpo: 23779 +.. date: 8238 +.. nonce: ET4JJP +.. section: Library + +imaplib raises TypeError if authenticator tries to abort. Patch from Craig +Holmquist. + +.. + +.. bpo: 23319 +.. date: 8237 +.. nonce: FXyUH- +.. section: Library + +Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by +Matthieu Gautier. + +.. + +.. bpo: 23254 +.. date: 8236 +.. nonce: zNiy1X +.. section: Library + +Document how to close the TCPServer listening socket. Patch from Martin +Panter. + +.. + +.. bpo: 19450 +.. date: 8235 +.. nonce: VG7T-L +.. section: Library + +Update Windows and OS X installer builds to use SQLite 3.8.11. + +.. + +.. bpo: 17527 +.. date: 8234 +.. nonce: ve9fyw +.. section: Library + +Add PATCH to wsgiref.validator. Patch from Luca Sbardella. + +.. + +.. bpo: 24791 +.. date: 8233 +.. nonce: Ok-3nA +.. section: Library + +Fix grammar regression for call syntax: 'g(\*a or b)'. + +.. + +.. bpo: 23672 +.. date: 8232 +.. nonce: 8td2se +.. section: IDLE + +Allow Idle to edit and run files with astral chars in name. Patch by Mohd +Sanad Zaki Rizvi. + +.. + +.. bpo: 24745 +.. date: 8231 +.. nonce: edbziT +.. section: IDLE + +Idle editor default font. Switch from Courier to platform-sensitive +TkFixedFont. This should not affect current customized font selections. If +there is a problem, edit $HOME/.idlerc/config-main.cfg and remove 'fontxxx' +entries from [Editor Window]. Patch by Mark Roseman. + +.. + +.. bpo: 21192 +.. date: 8230 +.. nonce: CdbipH +.. section: IDLE + +Idle editor. When a file is run, put its name in the restart bar. Do not +print false prompts. Original patch by Adnan Umer. + +.. + +.. bpo: 13884 +.. date: 8229 +.. nonce: vVcO1E +.. section: IDLE + +Idle menus. Remove tearoff lines. Patch by Roger Serwy. + +.. + +.. bpo: 24129 +.. date: 8228 +.. nonce: Imr54z +.. section: Documentation + +Clarify the reference documentation for name resolution. This includes +removing the assumption that readers will be familiar with the name +resolution scheme Python used prior to the introduction of lexical scoping +for function namespaces. Patch by Ivan Levkivskyi. + +.. + +.. bpo: 20769 +.. date: 8227 +.. nonce: ZUc9z9 +.. section: Documentation + +Improve reload() docs. Patch by Dorian Pula. + +.. + +.. bpo: 23589 +.. date: 8226 +.. nonce: rjU421 +.. section: Documentation + +Remove duplicate sentence from the FAQ. Patch by Yongzhi Pan. + +.. + +.. bpo: 24729 +.. date: 8225 +.. nonce: PH3A9p +.. section: Documentation + +Correct IO tutorial to match implementation regarding encoding parameter to +open function. + +.. + +.. bpo: 24751 +.. date: 8224 +.. nonce: pL2pbj +.. section: Tests + +When running regrtest with the ``-w`` command line option, a test run is no +longer marked as a failure if all tests succeed when re-run. diff --git a/Misc/NEWS.d/3.5.0rc2.rst b/Misc/NEWS.d/3.5.0rc2.rst new file mode 100644 index 00000000000000..94511092e0449b --- /dev/null +++ b/Misc/NEWS.d/3.5.0rc2.rst @@ -0,0 +1,56 @@ +.. bpo: 24769 +.. date: 8254 +.. nonce: XgRA0n +.. release date: 2015-08-25 +.. section: Core and Builtins + +Interpreter now starts properly when dynamic loading is disabled. Patch by +Petr Viktorin. + +.. + +.. bpo: 21167 +.. date: 8253 +.. nonce: uom-Dq +.. section: Core and Builtins + +NAN operations are now handled correctly when python is compiled with ICC +even if -fp-model strict is not specified. + +.. + +.. bpo: 24492 +.. date: 8252 +.. nonce: LKDAIu +.. section: Core and Builtins + +A "package" lacking a __name__ attribute when trying to perform a ``from .. +import ...`` statement will trigger an ImportError instead of an +AttributeError. + +.. + +.. bpo: 24847 +.. date: 8251 +.. nonce: SHiiO_ +.. section: Library + +Removes vcruntime140.dll dependency from Tcl/Tk. + +.. + +.. bpo: 24839 +.. date: 8250 +.. nonce: 7_iQZl +.. section: Library + +platform._syscmd_ver raises DeprecationWarning + +.. + +.. bpo: 24867 +.. date: 8249 +.. nonce: rxJIl7 +.. section: Library + +Fix Task.get_stack() for 'async def' coroutines diff --git a/Misc/NEWS.d/3.5.0rc3.rst b/Misc/NEWS.d/3.5.0rc3.rst new file mode 100644 index 00000000000000..72e6578c29a41e --- /dev/null +++ b/Misc/NEWS.d/3.5.0rc3.rst @@ -0,0 +1,76 @@ +.. bpo: 24305 +.. date: 8262 +.. nonce: QeF4A8 +.. release date: 2015-09-07 +.. section: Core and Builtins + +Prevent import subsystem stack frames from being counted by the +warnings.warn(stacklevel=) parameter. + +.. + +.. bpo: 24912 +.. date: 8261 +.. nonce: ubSi5J +.. section: Core and Builtins + +Prevent __class__ assignment to immutable built-in objects. + +.. + +.. bpo: 24975 +.. date: 8260 +.. nonce: 2gLdfN +.. section: Core and Builtins + +Fix AST compilation for PEP 448 syntax. + +.. + +.. bpo: 24917 +.. date: 8259 +.. nonce: xaQocz +.. section: Library + +time_strftime() buffer over-read. + +.. + +.. bpo: 24748 +.. date: 8258 +.. nonce: 83NuO8 +.. section: Library + +To resolve a compatibility problem found with py2exe and pywin32, +imp.load_dynamic() once again ignores previously loaded modules to support +Python modules replacing themselves with extension modules. Patch by Petr +Viktorin. + +.. + +.. bpo: 24635 +.. date: 8257 +.. nonce: EiJPPf +.. section: Library + +Fixed a bug in typing.py where isinstance([], typing.Iterable) would return +True once, then False on subsequent calls. + +.. + +.. bpo: 24989 +.. date: 8256 +.. nonce: 9BJLiy +.. section: Library + +Fixed buffer overread in BytesIO.readline() if a position is set beyond +size. Based on patch by John Leitch. + +.. + +.. bpo: 24913 +.. date: 8255 +.. nonce: p2ZAJ4 +.. section: Library + +Fix overrun error in deque.index(). Found by John Leitch and Bryce Darling. diff --git a/Misc/NEWS.d/3.5.0rc4.rst b/Misc/NEWS.d/3.5.0rc4.rst new file mode 100644 index 00000000000000..e62c80d9b6e529 --- /dev/null +++ b/Misc/NEWS.d/3.5.0rc4.rst @@ -0,0 +1,17 @@ +.. bpo: 25029 +.. date: 8264 +.. nonce: Zf97rk +.. release date: 2015-09-09 +.. section: Library + +Fixes MemoryError in test_strptime. + +.. + +.. bpo: 25027 +.. date: 8263 +.. nonce: Zaib78 +.. section: Build + +Reverts partial-static build options and adds vcruntime140.dll to Windows +installation. diff --git a/Misc/NEWS.d/3.5.1.rst b/Misc/NEWS.d/3.5.1.rst new file mode 100644 index 00000000000000..9a09342047ec6a --- /dev/null +++ b/Misc/NEWS.d/3.5.1.rst @@ -0,0 +1,17 @@ +.. bpo: 25709 +.. date: 8412 +.. nonce: OPX2TS +.. release date: 2015-12-06 +.. section: Core and Builtins + +Fixed problem with in-place string concatenation and utf-8 cache. + +.. + +.. bpo: 25715 +.. date: 8411 +.. nonce: 3LLYLj +.. section: Windows + +Python 3.5.1 installer shows wrong upgrade path and incorrect logic for +launcher detection. diff --git a/Misc/NEWS.d/3.5.1rc1.rst b/Misc/NEWS.d/3.5.1rc1.rst new file mode 100644 index 00000000000000..45cf6d93c226c9 --- /dev/null +++ b/Misc/NEWS.d/3.5.1rc1.rst @@ -0,0 +1,1451 @@ +.. bpo: 25630 +.. date: 8410 +.. nonce: ZxzcoY +.. release date: 2015-11-22 +.. section: Core and Builtins + +Fix a possible segfault during argument parsing in functions that accept +filesystem paths. + +.. + +.. bpo: 23564 +.. date: 8409 +.. nonce: XHarGG +.. section: Core and Builtins + +Fixed a partially broken sanity check in the _posixsubprocess internals +regarding how fds_to_pass were passed to the child. The bug had no actual +impact as subprocess.py already avoided it. + +.. + +.. bpo: 25388 +.. date: 8408 +.. nonce: zm3uuQ +.. section: Core and Builtins + +Fixed tokenizer crash when processing undecodable source code with a null +byte. + +.. + +.. bpo: 25462 +.. date: 8407 +.. nonce: eXDzgO +.. section: Core and Builtins + +The hash of the key now is calculated only once in most operations in C +implementation of OrderedDict. + +.. + +.. bpo: 22995 +.. date: 8406 +.. nonce: 90kpuP +.. section: Core and Builtins + +Default implementation of __reduce__ and __reduce_ex__ now rejects builtin +types with not defined __new__. + +.. + +.. bpo: 25555 +.. date: 8405 +.. nonce: MUpG-j +.. section: Core and Builtins + +Fix parser and AST: fill lineno and col_offset of "arg" node when compiling +AST from Python objects. + +.. + +.. bpo: 24802 +.. date: 8404 +.. nonce: Qie066 +.. section: Core and Builtins + +Avoid buffer overreads when int(), float(), compile(), exec() and eval() are +passed bytes-like objects. These objects are not necessarily terminated by +a null byte, but the functions assumed they were. + +.. + +.. bpo: 24726 +.. date: 8403 +.. nonce: AHk4v2 +.. section: Core and Builtins + +Fixed a crash and leaking NULL in repr() of OrderedDict that was mutated by +direct calls of dict methods. + +.. + +.. bpo: 25449 +.. date: 8402 +.. nonce: VqTOFi +.. section: Core and Builtins + +Iterating OrderedDict with keys with unstable hash now raises KeyError in C +implementations as well as in Python implementation. + +.. + +.. bpo: 25395 +.. date: 8401 +.. nonce: htkE3W +.. section: Core and Builtins + +Fixed crash when highly nested OrderedDict structures were garbage +collected. + +.. + +.. bpo: 25274 +.. date: 8400 +.. nonce: QCGvAF +.. section: Core and Builtins + +sys.setrecursionlimit() now raises a RecursionError if the new recursion +limit is too low depending at the current recursion depth. Modify also the +"lower-water mark" formula to make it monotonic. This mark is used to decide +when the overflowed flag of the thread state is reset. + +.. + +.. bpo: 24402 +.. date: 8399 +.. nonce: MAgi3X +.. section: Core and Builtins + +Fix input() to prompt to the redirected stdout when sys.stdout.fileno() +fails. + +.. + +.. bpo: 24806 +.. date: 8398 +.. nonce: Nb0znT +.. section: Core and Builtins + +Prevent builtin types that are not allowed to be subclassed from being +subclassed through multiple inheritance. + +.. + +.. bpo: 24848 +.. date: 8397 +.. nonce: HlUSuy +.. section: Core and Builtins + +Fixed a number of bugs in UTF-7 decoding of misformed data. + +.. + +.. bpo: 25280 +.. date: 8396 +.. nonce: ivTMwd +.. section: Core and Builtins + +Import trace messages emitted in verbose (-v) mode are no longer formatted +twice. + +.. + +.. bpo: 25003 +.. date: 8395 +.. nonce: _ban92 +.. section: Core and Builtins + +On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function +instead of the getentropy() function. The getentropy() function is blocking +to generate very good quality entropy, os.urandom() doesn't need such high- +quality entropy. + +.. + +.. bpo: 25182 +.. date: 8394 +.. nonce: gBDq-T +.. section: Core and Builtins + +The stdprinter (used as sys.stderr before the io module is imported at +startup) now uses the backslashreplace error handler. + +.. + +.. bpo: 25131 +.. date: 8393 +.. nonce: j5hH6a +.. section: Core and Builtins + +Make the line number and column offset of set/dict literals and +comprehensions correspond to the opening brace. + +.. + +.. bpo: 25150 +.. date: 8392 +.. nonce: 0Gh-Ty +.. section: Core and Builtins + +Hide the private _Py_atomic_xxx symbols from the public Python.h header to +fix a compilation error with OpenMP. PyThreadState_GET() becomes an alias to +PyThreadState_Get() to avoid ABI incompatibilies. + +.. + +.. bpo: 25626 +.. date: 8391 +.. nonce: TQ3fvb +.. section: Library + +Change three zlib functions to accept sizes that fit in Py_ssize_t, but +internally cap those sizes to UINT_MAX. This resolves a regression in 3.5 +where GzipFile.read() failed to read chunks larger than 2 or 4 GiB. The +change affects the zlib.Decompress.decompress() max_length parameter, the +zlib.decompress() bufsize parameter, and the zlib.Decompress.flush() length +parameter. + +.. + +.. bpo: 25583 +.. date: 8390 +.. nonce: Gk-cim +.. section: Library + +Avoid incorrect errors raised by os.makedirs(exist_ok=True) when the OS +gives priority to errors such as EACCES over EEXIST. + +.. + +.. bpo: 25593 +.. date: 8389 +.. nonce: 56uegI +.. section: Library + +Change semantics of EventLoop.stop() in asyncio. + +.. + +.. bpo: 6973 +.. date: 8388 +.. nonce: nl5cHt +.. section: Library + +When we know a subprocess.Popen process has died, do not allow the +send_signal(), terminate(), or kill() methods to do anything as they could +potentially signal a different process. + +.. + +.. bpo: 25590 +.. date: 8387 +.. nonce: aCt-yW +.. section: Library + +In the Readline completer, only call getattr() once per attribute. + +.. + +.. bpo: 25498 +.. date: 8386 +.. nonce: AvqEBl +.. section: Library + +Fix a crash when garbage-collecting ctypes objects created by wrapping a +memoryview. This was a regression made in 3.5a1. Based on patch by +Eryksun. + +.. + +.. bpo: 25584 +.. date: 8385 +.. nonce: 124mYw +.. section: Library + +Added "escape" to the __all__ list in the glob module. + +.. + +.. bpo: 25584 +.. date: 8384 +.. nonce: ZeWX0J +.. section: Library + +Fixed recursive glob() with patterns starting with ``**``. + +.. + +.. bpo: 25446 +.. date: 8383 +.. nonce: k1DByx +.. section: Library + +Fix regression in smtplib's AUTH LOGIN support. + +.. + +.. bpo: 18010 +.. date: 8382 +.. nonce: Azyf1C +.. section: Library + +Fix the pydoc web server's module search function to handle exceptions from +importing packages. + +.. + +.. bpo: 25554 +.. date: 8381 +.. nonce: UM9MlR +.. section: Library + +Got rid of circular references in regular expression parsing. + +.. + +.. bpo: 25510 +.. date: 8380 +.. nonce: 79g7LA +.. section: Library + +fileinput.FileInput.readline() now returns b'' instead of '' at the end if +the FileInput was opened with binary mode. Patch by Ryosuke Ito. + +.. + +.. bpo: 25503 +.. date: 8379 +.. nonce: Zea0Y7 +.. section: Library + +Fixed inspect.getdoc() for inherited docstrings of properties. Original +patch by John Mark Vandenberg. + +.. + +.. bpo: 25515 +.. date: 8378 +.. nonce: fQsyYG +.. section: Library + +Always use os.urandom as a source of randomness in uuid.uuid4. + +.. + +.. bpo: 21827 +.. date: 8377 +.. nonce: k2oreR +.. section: Library + +Fixed textwrap.dedent() for the case when largest common whitespace is a +substring of smallest leading whitespace. Based on patch by Robert Li. + +.. + +.. bpo: 25447 +.. date: 8376 +.. nonce: eDYc4t +.. section: Library + +The lru_cache() wrapper objects now can be copied and pickled (by returning +the original object unchanged). + +.. + +.. bpo: 25390 +.. date: 8375 +.. nonce: 6mSgRq +.. section: Library + +typing: Don't crash on Union[str, Pattern]. + +.. + +.. bpo: 25441 +.. date: 8374 +.. nonce: d7zph6 +.. section: Library + +asyncio: Raise error from drain() when socket is closed. + +.. + +.. bpo: 25410 +.. date: 8373 +.. nonce: QAs_3B +.. section: Library + +Cleaned up and fixed minor bugs in C implementation of OrderedDict. + +.. + +.. bpo: 25411 +.. date: 8372 +.. nonce: qsJTCb +.. section: Library + +Improved Unicode support in SMTPHandler through better use of the email +package. Thanks to user simon04 for the patch. + +.. + +.. bpo: 25407 +.. date: 8371 +.. nonce: ukNt1D +.. section: Library + +Remove mentions of the formatter module being removed in Python 3.6. + +.. + +.. bpo: 25406 +.. date: 8370 +.. nonce: 5MZKU_ +.. section: Library + +Fixed a bug in C implementation of OrderedDict.move_to_end() that caused +segmentation fault or hang in iterating after moving several items to the +start of ordered dict. + +.. + +.. bpo: 25364 +.. date: 8369 +.. nonce: u_1Wi6 +.. section: Library + +zipfile now works in threads disabled builds. + +.. + +.. bpo: 25328 +.. date: 8368 +.. nonce: Rja1Xg +.. section: Library + +smtpd's SMTPChannel now correctly raises a ValueError if both decode_data +and enable_SMTPUTF8 are set to true. + +.. + +.. bpo: 25316 +.. date: 8367 +.. nonce: dHQHWI +.. section: Library + +distutils raises OSError instead of DistutilsPlatformError when MSVC is not +installed. + +.. + +.. bpo: 25380 +.. date: 8366 +.. nonce: sKZ6-I +.. section: Library + +Fixed protocol for the STACK_GLOBAL opcode in pickletools.opcodes. + +.. + +.. bpo: 23972 +.. date: 8365 +.. nonce: s2g30g +.. section: Library + +Updates asyncio datagram create method allowing reuseport and reuseaddr +socket options to be set prior to binding the socket. Mirroring the existing +asyncio create_server method the reuseaddr option for datagram sockets +defaults to True if the O/S is 'posix' (except if the platform is Cygwin). +Patch by Chris Laws. + +.. + +.. bpo: 25304 +.. date: 8364 +.. nonce: CsmLyI +.. section: Library + +Add asyncio.run_coroutine_threadsafe(). This lets you submit a coroutine to +a loop from another thread, returning a concurrent.futures.Future. By +Vincent Michel. + +.. + +.. bpo: 25232 +.. date: 8363 +.. nonce: KhKjCE +.. section: Library + +Fix CGIRequestHandler to split the query from the URL at the first question +mark (?) rather than the last. Patch from Xiang Zhang. + +.. + +.. bpo: 24657 +.. date: 8362 +.. nonce: h2Ag7y +.. section: Library + +Prevent CGIRequestHandler from collapsing slashes in the query part of the +URL as if it were a path. Patch from Xiang Zhang. + +.. + +.. bpo: 24483 +.. date: 8361 +.. nonce: WPLGSJ +.. section: Library + +C implementation of functools.lru_cache() now calculates key's hash only +once. + +.. + +.. bpo: 22958 +.. date: 8360 +.. nonce: Ebu7Gl +.. section: Library + +Constructor and update method of weakref.WeakValueDictionary now accept the +self and the dict keyword arguments. + +.. + +.. bpo: 22609 +.. date: 8359 +.. nonce: fV7hdV +.. section: Library + +Constructor of collections.UserDict now accepts the self keyword argument. + +.. + +.. bpo: 25111 +.. date: 8358 +.. nonce: azL4qE +.. section: Library + +Fixed comparison of traceback.FrameSummary. + +.. + +.. bpo: 25262 +.. date: 8357 +.. nonce: pQS5cB +.. section: Library + +Added support for BINBYTES8 opcode in Python implementation of unpickler. +Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8 opcodes no +longer silently ignored on 32-bit platforms in C implementation. + +.. + +.. bpo: 25034 +.. date: 8356 +.. nonce: eGvOIb +.. section: Library + +Fix string.Formatter problem with auto-numbering and nested format_specs. +Patch by Anthon van der Neut. + +.. + +.. bpo: 25233 +.. date: 8355 +.. nonce: EdZV9x +.. section: Library + +Rewrite the guts of asyncio.Queue and asyncio.Semaphore to be more +understandable and correct. + +.. + +.. bpo: 25203 +.. date: 8354 +.. nonce: IgDEbt +.. section: Library + +Failed readline.set_completer_delims() no longer left the module in +inconsistent state. + +.. + +.. bpo: 23600 +.. date: 8353 +.. nonce: 7J_RD5 +.. section: Library + +Default implementation of tzinfo.fromutc() was returning wrong results in +some cases. + +.. + +.. bpo: 23329 +.. date: 8352 +.. nonce: yccJBE +.. section: Library + +Allow the ssl module to be built with older versions of LibreSSL. + +.. + +.. bpo: 0 +.. date: 8351 +.. nonce: ww9QSm +.. section: Library + +Prevent overflow in _Unpickler_Read. + +.. + +.. bpo: 25047 +.. date: 8350 +.. nonce: kc8tqx +.. section: Library + +The XML encoding declaration written by Element Tree now respects the letter +case given by the user. This restores the ability to write encoding names in +uppercase like "UTF-8", which worked in Python 2. + +.. + +.. bpo: 25135 +.. date: 8349 +.. nonce: gVHNy- +.. section: Library + +Make deque_clear() safer by emptying the deque before clearing. This helps +avoid possible reentrancy issues. + +.. + +.. bpo: 19143 +.. date: 8348 +.. nonce: 76SBSO +.. section: Library + +platform module now reads Windows version from kernel32.dll to avoid +compatibility shims. + +.. + +.. bpo: 25092 +.. date: 8347 +.. nonce: fQ37Ac +.. section: Library + +Fix datetime.strftime() failure when errno was already set to EINVAL. + +.. + +.. bpo: 23517 +.. date: 8346 +.. nonce: 3ABmf1 +.. section: Library + +Fix rounding in fromtimestamp() and utcfromtimestamp() methods of +datetime.datetime: microseconds are now rounded to nearest with ties going +to nearest even integer (ROUND_HALF_EVEN), instead of being rounding towards +minus infinity (ROUND_FLOOR). It's important that these methods use the same +rounding mode than datetime.timedelta to keep the property: +(datetime(1970,1,1) + timedelta(seconds=t)) == datetime.utcfromtimestamp(t). +It also the rounding mode used by round(float) for example. + +.. + +.. bpo: 25155 +.. date: 8345 +.. nonce: JiETzD +.. section: Library + +Fix datetime.datetime.now() and datetime.datetime.utcnow() on Windows to +support date after year 2038. It was a regression introduced in Python +3.5.0. + +.. + +.. bpo: 25108 +.. date: 8344 +.. nonce: zGPbgA +.. section: Library + +Omitted internal frames in traceback functions print_stack(), +format_stack(), and extract_stack() called without arguments. + +.. + +.. bpo: 25118 +.. date: 8343 +.. nonce: wGm1u6 +.. section: Library + +Fix a regression of Python 3.5.0 in os.waitpid() on Windows. + +.. + +.. bpo: 24684 +.. date: 8342 +.. nonce: t4T77O +.. section: Library + +socket.socket.getaddrinfo() now calls PyUnicode_AsEncodedString() instead of +calling the encode() method of the host, to handle correctly custom string +with an encode() method which doesn't return a byte string. The encoder of +the IDNA codec is now called directly instead of calling the encode() method +of the string. + +.. + +.. bpo: 25060 +.. date: 8341 +.. nonce: zLdvIk +.. section: Library + +Correctly compute stack usage of the BUILD_MAP opcode. + +.. + +.. bpo: 24857 +.. date: 8340 +.. nonce: PpJWZ9 +.. section: Library + +Comparing call_args to a long sequence now correctly returns a boolean +result instead of raising an exception. Patch by A Kaptur. + +.. + +.. bpo: 23144 +.. date: 8339 +.. nonce: cLf67X +.. section: Library + +Make sure that HTMLParser.feed() returns all the data, even when +convert_charrefs is True. + +.. + +.. bpo: 24982 +.. date: 8338 +.. nonce: sGMMAR +.. section: Library + +shutil.make_archive() with the "zip" format now adds entries for directories +(including empty directories) in ZIP file. + +.. + +.. bpo: 25019 +.. date: 8337 +.. nonce: JQJlOZ +.. section: Library + +Fixed a crash caused by setting non-string key of expat parser. Based on +patch by John Leitch. + +.. + +.. bpo: 16180 +.. date: 8336 +.. nonce: 6IUcNS +.. section: Library + +Exit pdb if file has syntax error, instead of trapping user in an infinite +loop. Patch by Xavier de Gaye. + +.. + +.. bpo: 24891 +.. date: 8335 +.. nonce: ddVmHS +.. section: Library + +Fix a race condition at Python startup if the file descriptor of stdin (0), +stdout (1) or stderr (2) is closed while Python is creating sys.stdin, +sys.stdout and sys.stderr objects. These attributes are now set to None if +the creation of the object failed, instead of raising an OSError exception. +Initial patch written by Marco Paolini. + +.. + +.. bpo: 24992 +.. date: 8334 +.. nonce: 5sqF74 +.. section: Library + +Fix error handling and a race condition (related to garbage collection) in +collections.OrderedDict constructor. + +.. + +.. bpo: 24881 +.. date: 8333 +.. nonce: ZoVZXu +.. section: Library + +Fixed setting binary mode in Python implementation of FileIO on Windows and +Cygwin. Patch from Akira Li. + +.. + +.. bpo: 25578 +.. date: 8332 +.. nonce: G6S-ft +.. section: Library + +Fix (another) memory leak in SSLSocket.getpeercer(). + +.. + +.. bpo: 25530 +.. date: 8331 +.. nonce: hDFkwu +.. section: Library + +Disable the vulnerable SSLv3 protocol by default when creating +ssl.SSLContext. + +.. + +.. bpo: 25569 +.. date: 8330 +.. nonce: CfvQjK +.. section: Library + +Fix memory leak in SSLSocket.getpeercert(). + +.. + +.. bpo: 25471 +.. date: 8329 +.. nonce: T0A02M +.. section: Library + +Sockets returned from accept() shouldn't appear to be nonblocking. + +.. + +.. bpo: 25319 +.. date: 8328 +.. nonce: iyuglv +.. section: Library + +When threading.Event is reinitialized, the underlying condition should use a +regular lock rather than a recursive lock. + +.. + +.. bpo: 21112 +.. date: 8327 +.. nonce: vSFU1r +.. section: Library + +Fix regression in unittest.expectedFailure on subclasses. Patch from Berker +Peksag. + +.. + +.. bpo: 24764 +.. date: 8326 +.. nonce: QwFZ2S +.. section: Library + +cgi.FieldStorage.read_multi() now ignores the Content-Length header in part +headers. Patch written by Peter Landry and reviewed by Pierre Quentel. + +.. + +.. bpo: 24913 +.. date: 8325 +.. nonce: p2ZAJ4 +.. section: Library + +Fix overrun error in deque.index(). Found by John Leitch and Bryce Darling. + +.. + +.. bpo: 24774 +.. date: 8324 +.. nonce: xLbskG +.. section: Library + +Fix docstring in http.server.test. Patch from Chiu-Hsiang Hsu. + +.. + +.. bpo: 21159 +.. date: 8323 +.. nonce: ochL5W +.. section: Library + +Improve message in configparser.InterpolationMissingOptionError. Patch from +Łukasz Langa. + +.. + +.. bpo: 20362 +.. date: 8322 +.. nonce: 5aP_Ri +.. section: Library + +Honour TestCase.longMessage correctly in assertRegex. Patch from Ilia +Kurenkov. + +.. + +.. bpo: 23572 +.. date: 8321 +.. nonce: QhQ9RD +.. section: Library + +Fixed functools.singledispatch on classes with falsy metaclasses. Patch by +Ethan Furman. + +.. + +.. bpo: 0 +.. date: 8320 +.. nonce: DO1sFa +.. section: Library + +asyncio: ensure_future() now accepts awaitable objects. + +.. + +.. bpo: 15348 +.. date: 8319 +.. nonce: d1Fg01 +.. section: IDLE + +Stop the debugger engine (normally in a user process) before closing the +debugger window (running in the IDLE process). This prevents the +RuntimeErrors that were being caught and ignored. + +.. + +.. bpo: 24455 +.. date: 8318 +.. nonce: x6YqtE +.. section: IDLE + +Prevent IDLE from hanging when a) closing the shell while the debugger is +active (15347); b) closing the debugger with the [X] button (15348); and c) +activating the debugger when already active (24455). The patch by Mark +Roseman does this by making two changes. 1. Suspend and resume the +gui.interaction method with the tcl vwait mechanism intended for this +purpose (instead of root.mainloop & .quit). 2. In gui.run, allow any +existing interaction to terminate first. + +.. + +.. bpo: 0 +.. date: 8317 +.. nonce: Yp9LRY +.. section: IDLE + +Change 'The program' to 'Your program' in an IDLE 'kill program?' message to +make it clearer that the program referred to is the currently running user +program, not IDLE itself. + +.. + +.. bpo: 24750 +.. date: 8316 +.. nonce: xgsi-K +.. section: IDLE + +Improve the appearance of the IDLE editor window status bar. Patch by Mark +Roseman. + +.. + +.. bpo: 25313 +.. date: 8315 +.. nonce: xMXHpO +.. section: IDLE + +Change the handling of new built-in text color themes to better address the +compatibility problem introduced by the addition of IDLE Dark. Consistently +use the revised idleConf.CurrentTheme everywhere in idlelib. + +.. + +.. bpo: 24782 +.. date: 8314 +.. nonce: RgIPYE +.. section: IDLE + +Extension configuration is now a tab in the IDLE Preferences dialog rather +than a separate dialog. The former tabs are now a sorted list. Patch by +Mark Roseman. + +.. + +.. bpo: 22726 +.. date: 8313 +.. nonce: x8T0dA +.. section: IDLE + +Re-activate the config dialog help button with some content about the other +buttons and the new IDLE Dark theme. + +.. + +.. bpo: 24820 +.. date: 8312 +.. nonce: TFPJhr +.. section: IDLE + +IDLE now has an 'IDLE Dark' built-in text color theme. It is more or less +IDLE Classic inverted, with a cobalt blue background. Strings, comments, +keywords, ... are still green, red, orange, ... . To use it with IDLEs +released before November 2015, hit the 'Save as New Custom Theme' button and +enter a new name, such as 'Custom Dark'. The custom theme will work with +any IDLE release, and can be modified. + +.. + +.. bpo: 25224 +.. date: 8311 +.. nonce: 5Llwo4 +.. section: IDLE + +README.txt is now an idlelib index for IDLE developers and curious users. +The previous user content is now in the IDLE doc chapter. 'IDLE' now means +'Integrated Development and Learning Environment'. + +.. + +.. bpo: 24820 +.. date: 8310 +.. nonce: ZUz9Fn +.. section: IDLE + +Users can now set breakpoint colors in Settings -> Custom Highlighting. +Original patch by Mark Roseman. + +.. + +.. bpo: 24972 +.. date: 8309 +.. nonce: uc0uNo +.. section: IDLE + +Inactive selection background now matches active selection background, as +configured by users, on all systems. Found items are now always highlighted +on Windows. Initial patch by Mark Roseman. + +.. + +.. bpo: 24570 +.. date: 8308 +.. nonce: s3EkNn +.. section: IDLE + +Idle: make calltip and completion boxes appear on Macs affected by a tk +regression. Initial patch by Mark Roseman. + +.. + +.. bpo: 24988 +.. date: 8307 +.. nonce: tXqq4T +.. section: IDLE + +Idle ScrolledList context menus (used in debugger) now work on Mac Aqua. +Patch by Mark Roseman. + +.. + +.. bpo: 24801 +.. date: 8306 +.. nonce: -bj_Ou +.. section: IDLE + +Make right-click for context menu work on Mac Aqua. Patch by Mark Roseman. + +.. + +.. bpo: 25173 +.. date: 8305 +.. nonce: EZzrPg +.. section: IDLE + +Associate tkinter messageboxes with a specific widget. For Mac OSX, make +them a 'sheet'. Patch by Mark Roseman. + +.. + +.. bpo: 25198 +.. date: 8304 +.. nonce: -j_BV7 +.. section: IDLE + +Enhance the initial html viewer now used for Idle Help. * Properly indent +fixed-pitch text (patch by Mark Roseman). * Give code snippet a very Sphinx- +like light blueish-gray background. * Re-use initial width and height set by +users for shell and editor. * When the Table of Contents (TOC) menu is used, +put the section header at the top of the screen. + +.. + +.. bpo: 25225 +.. date: 8303 +.. nonce: 9pvdq6 +.. section: IDLE + +Condense and rewrite Idle doc section on text colors. + +.. + +.. bpo: 21995 +.. date: 8302 +.. nonce: C5Rmzx +.. section: IDLE + +Explain some differences between IDLE and console Python. + +.. + +.. bpo: 22820 +.. date: 8301 +.. nonce: hix_8X +.. section: IDLE + +Explain need for *print* when running file from Idle editor. + +.. + +.. bpo: 25224 +.. date: 8300 +.. nonce: UVMYQq +.. section: IDLE + +Doc: augment Idle feature list and no-subprocess section. + +.. + +.. bpo: 25219 +.. date: 8299 +.. nonce: 8_9DYg +.. section: IDLE + +Update doc for Idle command line options. Some were missing and notes were +not correct. + +.. + +.. bpo: 24861 +.. date: 8298 +.. nonce: Ecg2yT +.. section: IDLE + +Most of idlelib is private and subject to change. Use idleib.idle.* to start +Idle. See idlelib.__init__.__doc__. + +.. + +.. bpo: 25199 +.. date: 8297 +.. nonce: ih7yY3 +.. section: IDLE + +Idle: add synchronization comments for future maintainers. + +.. + +.. bpo: 16893 +.. date: 8296 +.. nonce: bZtPgJ +.. section: IDLE + +Replace help.txt with help.html for Idle doc display. The new +idlelib/help.html is rstripped Doc/build/html/library/idle.html. It looks +better than help.txt and will better document Idle as released. The tkinter +html viewer that works for this file was written by Mark Roseman. The now +unused EditorWindow.HelpDialog class and helt.txt file are deprecated. + +.. + +.. bpo: 24199 +.. date: 8295 +.. nonce: VKnZEv +.. section: IDLE + +Deprecate unused idlelib.idlever with possible removal in 3.6. + +.. + +.. bpo: 24790 +.. date: 8294 +.. nonce: hD1hlj +.. section: IDLE + +Remove extraneous code (which also create 2 & 3 conflicts). + +.. + +.. bpo: 22558 +.. date: 8293 +.. nonce: Pk02YC +.. section: Documentation + +Add remaining doc links to source code for Python-coded modules. Patch by +Yoni Lavi. + +.. + +.. bpo: 12067 +.. date: 8292 +.. nonce: nLD2M- +.. section: Documentation + +Rewrite Comparisons section in the Expressions chapter of the language +reference. Some of the details of comparing mixed types were incorrect or +ambiguous. NotImplemented is only relevant at a lower level than the +Expressions chapter. Added details of comparing range() objects, and default +behaviour and consistency suggestions for user-defined classes. Patch from +Andy Maier. + +.. + +.. bpo: 24952 +.. date: 8291 +.. nonce: RHhFPE +.. section: Documentation + +Clarify the default size argument of stack_size() in the "threading" and +"_thread" modules. Patch from Mattip. + +.. + +.. bpo: 23725 +.. date: 8290 +.. nonce: 49TZ5f +.. section: Documentation + +Overhaul tempfile docs. Note deprecated status of mktemp. Patch from +Zbigniew Jędrzejewski-Szmek. + +.. + +.. bpo: 24808 +.. date: 8289 +.. nonce: MGjc3F +.. section: Documentation + +Update the types of some PyTypeObject fields. Patch by Joseph Weston. + +.. + +.. bpo: 22812 +.. date: 8288 +.. nonce: kLCF0G +.. section: Documentation + +Fix unittest discovery examples. Patch from Pam McA'Nulty. + +.. + +.. bpo: 25449 +.. date: 8287 +.. nonce: MP6KNs +.. section: Tests + +Added tests for OrderedDict subclasses. + +.. + +.. bpo: 25099 +.. date: 8286 +.. nonce: tJQOWx +.. section: Tests + +Make test_compileall not fail when an entry on sys.path cannot be written to +(commonly seen in administrative installs on Windows). + +.. + +.. bpo: 23919 +.. date: 8285 +.. nonce: vJnjaq +.. section: Tests + +Prevents assert dialogs appearing in the test suite. + +.. + +.. bpo: 0 +.. date: 8284 +.. nonce: X-Bk5l +.. section: Tests + +``PCbuild\rt.bat`` now accepts an unlimited number of arguments to pass +along to regrtest.py. Previously there was a limit of 9. + +.. + +.. bpo: 24915 +.. date: 8283 +.. nonce: PgD3Cx +.. section: Build + +Add LLVM support for PGO builds and use the test suite to generate the +profile data. Initial patch by Alecsandru Patrascu of Intel. + +.. + +.. bpo: 24910 +.. date: 8282 +.. nonce: ZZdfl0 +.. section: Build + +Windows MSIs now have unique display names. + +.. + +.. bpo: 24986 +.. date: 8281 +.. nonce: 1WyXeU +.. section: Build + +It is now possible to build Python on Windows without errors when external +libraries are not available. + +.. + +.. bpo: 25450 +.. date: 8280 +.. nonce: X4xlWf +.. section: Windows + +Updates shortcuts to start Python in installation directory. + +.. + +.. bpo: 25164 +.. date: 8279 +.. nonce: FHVOOA +.. section: Windows + +Changes default all-users install directory to match per-user directory. + +.. + +.. bpo: 25143 +.. date: 8278 +.. nonce: hmxsia +.. section: Windows + +Improves installer error messages for unsupported platforms. + +.. + +.. bpo: 25163 +.. date: 8277 +.. nonce: uCRe8H +.. section: Windows + +Display correct directory in installer when using non-default settings. + +.. + +.. bpo: 25361 +.. date: 8276 +.. nonce: GETaSY +.. section: Windows + +Disables use of SSE2 instructions in Windows 32-bit build + +.. + +.. bpo: 25089 +.. date: 8275 +.. nonce: n_YJgw +.. section: Windows + +Adds logging to installer for case where launcher is not selected on +upgrade. + +.. + +.. bpo: 25165 +.. date: 8274 +.. nonce: aUTN1e +.. section: Windows + +Windows uninstallation should not remove launcher if other versions remain + +.. + +.. bpo: 25112 +.. date: 8273 +.. nonce: frdKij +.. section: Windows + +py.exe launcher is missing icons + +.. + +.. bpo: 25102 +.. date: 8272 +.. nonce: 6y6Akl +.. section: Windows + +Windows installer does not precompile for -O or -OO. + +.. + +.. bpo: 25081 +.. date: 8271 +.. nonce: dcRCTO +.. section: Windows + +Makes Back button in installer go back to upgrade page when upgrading. + +.. + +.. bpo: 25091 +.. date: 8270 +.. nonce: 1u-VKy +.. section: Windows + +Increases font size of the installer. + +.. + +.. bpo: 25126 +.. date: 8269 +.. nonce: ANx3DW +.. section: Windows + +Clarifies that the non-web installer will download some components. + +.. + +.. bpo: 25213 +.. date: 8268 +.. nonce: KGmXoe +.. section: Windows + +Restores requestedExecutionLevel to manifest to disable UAC virtualization. + +.. + +.. bpo: 25022 +.. date: 8267 +.. nonce: vAt_zr +.. section: Windows + +Removed very outdated PC/example_nt/ directory. + +.. + +.. bpo: 25440 +.. date: 8266 +.. nonce: 5xhyGr +.. section: Tools/Demos + +Fix output of python-config --extension-suffix. diff --git a/Misc/NEWS.d/3.5.2.rst b/Misc/NEWS.d/3.5.2.rst new file mode 100644 index 00000000000000..f08657cd13dcc7 --- /dev/null +++ b/Misc/NEWS.d/3.5.2.rst @@ -0,0 +1,25 @@ +.. bpo: 26930 +.. date: 8636 +.. nonce: 9JUeSD +.. release date: 2016-06-26 +.. section: Core and Builtins + +Update Windows builds to use OpenSSL 1.0.2h. + +.. + +.. bpo: 26867 +.. date: 8635 +.. nonce: QPSyP5 +.. section: Tests + +Ubuntu's openssl OP_NO_SSLv3 is forced on by default; fix test. + +.. + +.. bpo: 27365 +.. date: 8634 +.. nonce: ipkJ_M +.. section: IDLE + +Allow non-ascii in idlelib/NEWS.txt - minimal part for 3.5.2. diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst new file mode 100644 index 00000000000000..2cd4580d5889b5 --- /dev/null +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -0,0 +1,2204 @@ +.. bpo: 27066 +.. date: 8633 +.. nonce: SNExZi +.. release date: 2016-06-12 +.. section: Core and Builtins + +Fixed SystemError if a custom opener (for open()) returns a negative number +without setting an exception. + +.. + +.. bpo: 20041 +.. date: 8632 +.. nonce: TypyGp +.. section: Core and Builtins + +Fixed TypeError when frame.f_trace is set to None. Patch by Xavier de Gaye. + +.. + +.. bpo: 26168 +.. date: 8631 +.. nonce: -nPBL6 +.. section: Core and Builtins + +Fixed possible refleaks in failing Py_BuildValue() with the "N" format unit. + +.. + +.. bpo: 26991 +.. date: 8630 +.. nonce: yWGNhz +.. section: Core and Builtins + +Fix possible refleak when creating a function with annotations. + +.. + +.. bpo: 27039 +.. date: 8629 +.. nonce: Zj7tV7 +.. section: Core and Builtins + +Fixed bytearray.remove() for values greater than 127. Patch by Joe Jevnik. + +.. + +.. bpo: 23640 +.. date: 8628 +.. nonce: kvNC4c +.. section: Core and Builtins + +int.from_bytes() no longer bypasses constructors for subclasses. + +.. + +.. bpo: 26811 +.. date: 8627 +.. nonce: oNzUWt +.. section: Core and Builtins + +gc.get_objects() no longer contains a broken tuple with NULL pointer. + +.. + +.. bpo: 20120 +.. date: 8626 +.. nonce: c-FZZc +.. section: Core and Builtins + +Use RawConfigParser for .pypirc parsing, removing support for interpolation +unintentionally added with move to Python 3. Behavior no longer does any +interpolation in .pypirc files, matching behavior in Python 2.7 and +Setuptools 19.0. + +.. + +.. bpo: 26659 +.. date: 8625 +.. nonce: 5PRa83 +.. section: Core and Builtins + +Make the builtin slice type support cycle collection. + +.. + +.. bpo: 26718 +.. date: 8624 +.. nonce: K5PQ8j +.. section: Core and Builtins + +super.__init__ no longer leaks memory if called multiple times. NOTE: A +direct call of super.__init__ is not endorsed! + +.. + +.. bpo: 25339 +.. date: 8623 +.. nonce: ZcaC2E +.. section: Core and Builtins + +PYTHONIOENCODING now has priority over locale in setting the error handler +for stdin and stdout. + +.. + +.. bpo: 26494 +.. date: 8622 +.. nonce: G6eXIi +.. section: Core and Builtins + +Fixed crash on iterating exhausting iterators. Affected classes are generic +sequence iterators, iterators of str, bytes, bytearray, list, tuple, set, +frozenset, dict, OrderedDict, corresponding views and os.scandir() iterator. + +.. + +.. bpo: 26581 +.. date: 8621 +.. nonce: yNA7nm +.. section: Core and Builtins + +If coding cookie is specified multiple times on a line in Python source code +file, only the first one is taken to account. + +.. + +.. bpo: 26464 +.. date: 8620 +.. nonce: 7BreGz +.. section: Core and Builtins + +Fix str.translate() when string is ASCII and first replacements removes +character, but next replacement uses a non-ASCII character or a string +longer than 1 character. Regression introduced in Python 3.5.0. + +.. + +.. bpo: 22836 +.. date: 8619 +.. nonce: cimt1y +.. section: Core and Builtins + +Ensure exception reports from PyErr_Display() and PyErr_WriteUnraisable() +are sensible even when formatting them produces secondary errors. This +affects the reports produced by sys.__excepthook__() and when __del__() +raises an exception. + +.. + +.. bpo: 26302 +.. date: 8618 +.. nonce: UD9XQt +.. section: Core and Builtins + +Correct behavior to reject comma as a legal character for cookie names. + +.. + +.. bpo: 4806 +.. date: 8617 +.. nonce: i9m3hj +.. section: Core and Builtins + +Avoid masking the original TypeError exception when using star (``*``) +unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel +Urban. + +.. + +.. bpo: 27138 +.. date: 8616 +.. nonce: ifYEro +.. section: Core and Builtins + +Fix the doc comment for FileFinder.find_spec(). + +.. + +.. bpo: 26154 +.. date: 8615 +.. nonce: MtnRAH +.. section: Core and Builtins + +Add a new private _PyThreadState_UncheckedGet() function to get the current +Python thread state, but don't issue a fatal error if it is NULL. This new +function must be used instead of accessing directly the +_PyThreadState_Current variable. The variable is no more exposed since +Python 3.5.1 to hide the exact implementation of atomic C types, to avoid +compiler issues. + +.. + +.. bpo: 26194 +.. date: 8614 +.. nonce: j9zand +.. section: Core and Builtins + +Deque.insert() gave odd results for bounded deques that had reached their +maximum size. Now an IndexError will be raised when attempting to insert +into a full deque. + +.. + +.. bpo: 25843 +.. date: 8613 +.. nonce: t2kGug +.. section: Core and Builtins + +When compiling code, don't merge constants if they are equal but have a +different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` is now +correctly compiled to two different functions: ``f1()`` returns ``1`` +(``int``) and ``f2()`` returns ``1.0`` (``int``), even if ``1`` and ``1.0`` +are equal. + +.. + +.. bpo: 22995 +.. date: 8612 +.. nonce: KYNKvs +.. section: Core and Builtins + +[UPDATE] Comment out the one of the pickleability tests in +_PyObject_GetState() due to regressions observed in Cython-based projects. + +.. + +.. bpo: 25961 +.. date: 8611 +.. nonce: Hdjjw0 +.. section: Core and Builtins + +Disallowed null characters in the type name. + +.. + +.. bpo: 25973 +.. date: 8610 +.. nonce: Ud__ZP +.. section: Core and Builtins + +Fix segfault when an invalid nonlocal statement binds a name starting with +two underscores. + +.. + +.. bpo: 22995 +.. date: 8609 +.. nonce: Wq0E86 +.. section: Core and Builtins + +Instances of extension types with a state that aren't subclasses of list or +dict and haven't implemented any pickle-related methods (__reduce__, +__reduce_ex__, __getnewargs__, __getnewargs_ex__, or __getstate__), can no +longer be pickled. Including memoryview. + +.. + +.. bpo: 20440 +.. date: 8608 +.. nonce: GCwOfH +.. section: Core and Builtins + +Massive replacing unsafe attribute setting code with special macro +Py_SETREF. + +.. + +.. bpo: 25766 +.. date: 8607 +.. nonce: jn93Yu +.. section: Core and Builtins + +Special method __bytes__() now works in str subclasses. + +.. + +.. bpo: 25421 +.. date: 8606 +.. nonce: c47YEL +.. section: Core and Builtins + +__sizeof__ methods of builtin types now use dynamic basic size. This allows +sys.getsize() to work correctly with their subclasses with __slots__ +defined. + +.. + +.. bpo: 25709 +.. date: 8605 +.. nonce: WwGm2k +.. section: Core and Builtins + +Fixed problem with in-place string concatenation and utf-8 cache. + +.. + +.. bpo: 27147 +.. date: 8604 +.. nonce: tCCgmH +.. section: Core and Builtins + +Mention PEP 420 in the importlib docs. + +.. + +.. bpo: 24097 +.. date: 8603 +.. nonce: Vt4E-i +.. section: Core and Builtins + +Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. + +.. + +.. bpo: 24731 +.. date: 8602 +.. nonce: h9-hnz +.. section: Core and Builtins + +Fixed crash on converting objects with special methods __bytes__, __trunc__, +and __float__ returning instances of subclasses of bytes, int, and float to +subclasses of bytes, int, and float correspondingly. + +.. + +.. bpo: 26478 +.. date: 8601 +.. nonce: n0dB8e +.. section: Core and Builtins + +Fix semantic bugs when using binary operators with dictionary views and +tuples. + +.. + +.. bpo: 26171 +.. date: 8600 +.. nonce: 8SaQEa +.. section: Core and Builtins + +Fix possible integer overflow and heap corruption in zipimporter.get_data(). + +.. + +.. bpo: 25660 +.. date: 8599 +.. nonce: 93DzBo +.. section: Core and Builtins + +Fix TAB key behaviour in REPL with readline. + +.. + +.. bpo: 25887 +.. date: 8598 +.. nonce: PtVIX7 +.. section: Core and Builtins + +Raise a RuntimeError when a coroutine object is awaited more than once. + +.. + +.. bpo: 27243 +.. date: 8597 +.. nonce: U36M4E +.. section: Core and Builtins + +Update the __aiter__ protocol: instead of returning an awaitable that +resolves to an asynchronous iterator, the asynchronous iterator should be +returned directly. Doing the former will trigger a +PendingDeprecationWarning. + +.. + +.. bpo: 26556 +.. date: 8596 +.. nonce: v5j2uL +.. original section: Library +.. section: Security + +Update expat to 2.1.1, fixes CVE-2015-1283. + +.. + +.. bpo: 0 +.. date: 8595 +.. nonce: E4ochz +.. original section: Library +.. section: Security + +Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team +Oststrom + +.. + +.. bpo: 21386 +.. date: 8594 +.. nonce: DjV72U +.. section: Library + +Implement missing IPv4Address.is_global property. It was documented since +07a5610bae9d. Initial patch by Roger Luethi. + +.. + +.. bpo: 20900 +.. date: 8593 +.. nonce: H5YQPR +.. section: Library + +distutils register command now decodes HTTP responses correctly. Initial +patch by ingrid. + +.. + +.. bpo: 0 +.. date: 8592 +.. nonce: iYIeng +.. section: Library + +A new version of typing.py provides several new classes and features: +@overload outside stubs, Reversible, DefaultDict, Text, ContextManager, +Type[], NewType(), TYPE_CHECKING, and numerous bug fixes (note that some of +the new features are not yet implemented in mypy or other static analyzers). +Also classes for PEP 492 (Awaitable, AsyncIterable, AsyncIterator) have been +added (in fact they made it into 3.5.1 but were never mentioned). + +.. + +.. bpo: 25738 +.. date: 8591 +.. nonce: mED9w4 +.. section: Library + +Stop http.server.BaseHTTPRequestHandler.send_error() from sending a message +body for 205 Reset Content. Also, don't send Content header fields in +responses that don't have a body. Patch by Susumu Koshiba. + +.. + +.. bpo: 21313 +.. date: 8590 +.. nonce: W30MBr +.. section: Library + +Fix the "platform" module to tolerate when sys.version contains truncated +build information. + +.. + +.. bpo: 26839 +.. date: 8589 +.. nonce: yVvy7R +.. original section: Library +.. section: Security + +On Linux, :func:`os.urandom` now calls ``getrandom()`` with +``GRND_NONBLOCK`` to fall back on reading ``/dev/urandom`` if the urandom +entropy pool is not initialized yet. Patch written by Colm Buckley. + +.. + +.. bpo: 27164 +.. date: 8588 +.. nonce: 6wmjx2 +.. section: Library + +In the zlib module, allow decompressing raw Deflate streams with a +predefined zdict. Based on patch by Xiang Zhang. + +.. + +.. bpo: 24291 +.. date: 8587 +.. nonce: Ac6HvL +.. section: Library + +Fix wsgiref.simple_server.WSGIRequestHandler to completely write data to the +client. Previously it could do partial writes and truncate data. Also, +wsgiref.handler.ServerHandler can now handle stdout doing partial writes, +but this is deprecated. + +.. + +.. bpo: 26809 +.. date: 8586 +.. nonce: ya7JMb +.. section: Library + +Add ``__all__`` to :mod:`string`. Patch by Emanuel Barry. + +.. + +.. bpo: 26373 +.. date: 8585 +.. nonce: P6qz6o +.. section: Library + +subprocess.Popen.communicate now correctly ignores BrokenPipeError when the +child process dies before .communicate() is called in more/all +circumstances. + +.. + +.. bpo: 21776 +.. date: 8584 +.. nonce: 04eQfa +.. section: Library + +distutils.upload now correctly handles HTTPError. Initial patch by Claudiu +Popa. + +.. + +.. bpo: 27114 +.. date: 8583 +.. nonce: bGCuAM +.. section: Library + +Fix SSLContext._load_windows_store_certs fails with PermissionError + +.. + +.. bpo: 18383 +.. date: 8582 +.. nonce: jr-b0l +.. section: Library + +Avoid creating duplicate filters when using filterwarnings and simplefilter. +Based on patch by Alex Shkop. + +.. + +.. bpo: 27057 +.. date: 8581 +.. nonce: YzTA_Q +.. section: Library + +Fix os.set_inheritable() on Android, ioctl() is blocked by SELinux and fails +with EACCESS. The function now falls back to fcntl(). Patch written by +Michał Bednarski. + +.. + +.. bpo: 27014 +.. date: 8580 +.. nonce: ui7Khn +.. section: Library + +Fix infinite recursion using typing.py. Thanks to Kalle Tuure! + +.. + +.. bpo: 14132 +.. date: 8579 +.. nonce: 5wR9MN +.. section: Library + +Fix urllib.request redirect handling when the target only has a query +string. Original fix by Ján Janech. + +.. + +.. bpo: 17214 +.. date: 8578 +.. nonce: lUbZOV +.. section: Library + +The "urllib.request" module now percent-encodes non-ASCII bytes found in +redirect target URLs. Some servers send Location header fields with non- +ASCII bytes, but "http.client" requires the request target to be ASCII- +encodable, otherwise a UnicodeEncodeError is raised. Based on patch by +Christian Heimes. + +.. + +.. bpo: 26892 +.. date: 8577 +.. nonce: XIXb0h +.. section: Library + +Honor debuglevel flag in urllib.request.HTTPHandler. Patch contributed by +Chi Hsuan Yen. + +.. + +.. bpo: 22274 +.. date: 8576 +.. nonce: 0RHDMN +.. section: Library + +In the subprocess module, allow stderr to be redirected to stdout even when +stdout is not redirected. Patch by Akira Li. + +.. + +.. bpo: 26807 +.. date: 8575 +.. nonce: LXSPP6 +.. section: Library + +mock_open 'files' no longer error on readline at end of file. Patch from +Yolanda Robla. + +.. + +.. bpo: 25745 +.. date: 8574 +.. nonce: -n8acU +.. section: Library + +Fixed leaking a userptr in curses panel destructor. + +.. + +.. bpo: 26977 +.. date: 8573 +.. nonce: 5G4HtL +.. section: Library + +Removed unnecessary, and ignored, call to sum of squares helper in +statistics.pvariance. + +.. + +.. bpo: 26881 +.. date: 8572 +.. nonce: mdiq_L +.. section: Library + +The modulefinder module now supports extended opcode arguments. + +.. + +.. bpo: 23815 +.. date: 8571 +.. nonce: _krNe8 +.. section: Library + +Fixed crashes related to directly created instances of types in _tkinter and +curses.panel modules. + +.. + +.. bpo: 17765 +.. date: 8570 +.. nonce: hiSVS1 +.. section: Library + +weakref.ref() no longer silently ignores keyword arguments. Patch by Georg +Brandl. + +.. + +.. bpo: 26873 +.. date: 8569 +.. nonce: cYXRcH +.. section: Library + +xmlrpc now raises ResponseError on unsupported type tags instead of silently +return incorrect result. + +.. + +.. bpo: 26711 +.. date: 8568 +.. nonce: Eu85Qw +.. section: Library + +Fixed the comparison of plistlib.Data with other types. + +.. + +.. bpo: 24114 +.. date: 8567 +.. nonce: RMRMtM +.. section: Library + +Fix an uninitialized variable in `ctypes.util`. + +The bug only occurs on SunOS when the ctypes implementation searches for the +`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. + +.. + +.. bpo: 26864 +.. date: 8566 +.. nonce: 1KgGds +.. section: Library + +In urllib.request, change the proxy bypass host checking against no_proxy to +be case-insensitive, and to not match unrelated host names that happen to +have a bypassed hostname as a suffix. Patch by Xiang Zhang. + +.. + +.. bpo: 26634 +.. date: 8565 +.. nonce: FZvsSb +.. section: Library + +recursive_repr() now sets __qualname__ of wrapper. Patch by Xiang Zhang. + +.. + +.. bpo: 26804 +.. date: 8564 +.. nonce: 9Orp-G +.. section: Library + +urllib.request will prefer lower_case proxy environment variables over +UPPER_CASE or Mixed_Case ones. Patch contributed by Hans-Peter Jansen. + +.. + +.. bpo: 26837 +.. date: 8563 +.. nonce: 2FXGsD +.. section: Library + +assertSequenceEqual() now correctly outputs non-stringified differing items +(like bytes in the -b mode). This affects assertListEqual() and +assertTupleEqual(). + +.. + +.. bpo: 26041 +.. date: 8562 +.. nonce: bVem-p +.. section: Library + +Remove "will be removed in Python 3.7" from deprecation messages of +platform.dist() and platform.linux_distribution(). Patch by Kumaripaba +Miyurusara Athukorala. + +.. + +.. bpo: 26822 +.. date: 8561 +.. nonce: rYSL4W +.. section: Library + +itemgetter, attrgetter and methodcaller objects no longer silently ignore +keyword arguments. + +.. + +.. bpo: 26733 +.. date: 8560 +.. nonce: YxaJmL +.. section: Library + +Disassembling a class now disassembles class and static methods. Patch by +Xiang Zhang. + +.. + +.. bpo: 26801 +.. date: 8559 +.. nonce: TQGY-7 +.. section: Library + +Fix error handling in :func:`shutil.get_terminal_size`, catch +:exc:`AttributeError` instead of :exc:`NameError`. Patch written by Emanuel +Barry. + +.. + +.. bpo: 24838 +.. date: 8558 +.. nonce: 3Pfx8T +.. section: Library + +tarfile's ustar and gnu formats now correctly calculate name and link field +limits for multibyte character encodings like utf-8. + +.. + +.. bpo: 26657 +.. date: 8557 +.. nonce: C_-XFg +.. original section: Library +.. section: Security + +Fix directory traversal vulnerability with http.server on Windows. This +fixes a regression that was introduced in 3.3.4rc1 and 3.4.0rc1. Based on +patch by Philipp Hagemeister. + +.. + +.. bpo: 26717 +.. date: 8556 +.. nonce: jngTdu +.. section: Library + +Stop encoding Latin-1-ized WSGI paths with UTF-8. Patch by Anthony Sottile. + +.. + +.. bpo: 26735 +.. date: 8555 +.. nonce: riSl3b +.. section: Library + +Fix :func:`os.urandom` on Solaris 11.3 and newer when reading more than +1,024 bytes: call ``getrandom()`` multiple times with a limit of 1024 bytes +per call. + +.. + +.. bpo: 16329 +.. date: 8554 +.. nonce: nuXD8W +.. section: Library + +Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. + +.. + +.. bpo: 13952 +.. date: 8553 +.. nonce: SOoTVE +.. section: Library + +Add .csv to mimetypes.types_map. Patch by Geoff Wilson. + +.. + +.. bpo: 26709 +.. date: 8552 +.. nonce: luOPbP +.. section: Library + +Fixed Y2038 problem in loading binary PLists. + +.. + +.. bpo: 23735 +.. date: 8551 +.. nonce: Y5oQ9r +.. section: Library + +Handle terminal resizing with Readline 6.3+ by installing our own SIGWINCH +handler. Patch by Eric Price. + +.. + +.. bpo: 26586 +.. date: 8550 +.. nonce: V5pZNa +.. section: Library + +In http.server, respond with "413 Request header fields too large" if there +are too many header fields to parse, rather than killing the connection and +raising an unhandled exception. Patch by Xiang Zhang. + +.. + +.. bpo: 22854 +.. date: 8549 +.. nonce: K3rMEH +.. section: Library + +Change BufferedReader.writable() and BufferedWriter.readable() to always +return False. + +.. + +.. bpo: 25195 +.. date: 8548 +.. nonce: EOc4Po +.. section: Library + +Fix a regression in mock.MagicMock. _Call is a subclass of tuple (changeset +3603bae63c13 only works for classes) so we need to implement __ne__ +ourselves. Patch by Andrew Plummer. + +.. + +.. bpo: 26644 +.. date: 8547 +.. nonce: 7tt1tk +.. section: Library + +Raise ValueError rather than SystemError when a negative length is passed to +SSLSocket.recv() or read(). + +.. + +.. bpo: 23804 +.. date: 8546 +.. nonce: PP63Ff +.. section: Library + +Fix SSL recv(0) and read(0) methods to return zero bytes instead of up to +1024. + +.. + +.. bpo: 26616 +.. date: 8545 +.. nonce: v3QwdD +.. section: Library + +Fixed a bug in datetime.astimezone() method. + +.. + +.. bpo: 21925 +.. date: 8544 +.. nonce: _fr69L +.. section: Library + +:func:`warnings.formatwarning` now catches exceptions on +``linecache.getline(...)`` to be able to log :exc:`ResourceWarning` emitted +late during the Python shutdown process. + +.. + +.. bpo: 24266 +.. date: 8543 +.. nonce: YZgVyM +.. section: Library + +Ctrl+C during Readline history search now cancels the search mode when +compiled with Readline 7. + +.. + +.. bpo: 26560 +.. date: 8542 +.. nonce: A4WXNz +.. section: Library + +Avoid potential ValueError in BaseHandler.start_response. Initial patch by +Peter Inglesby. + +.. + +.. bpo: 26313 +.. date: 8541 +.. nonce: LjZAjy +.. original section: Library +.. section: Security + +ssl.py _load_windows_store_certs fails if windows cert store is empty. Patch +by Baji. + +.. + +.. bpo: 26569 +.. date: 8540 +.. nonce: EX8vF1 +.. section: Library + +Fix :func:`pyclbr.readmodule` and :func:`pyclbr.readmodule_ex` to support +importing packages. + +.. + +.. bpo: 26499 +.. date: 8539 +.. nonce: NP08PI +.. section: Library + +Account for remaining Content-Length in HTTPResponse.readline() and read1(). +Based on patch by Silent Ghost. Also document that HTTPResponse now supports +these methods. + +.. + +.. bpo: 25320 +.. date: 8538 +.. nonce: V96LIy +.. section: Library + +Handle sockets in directories unittest discovery is scanning. Patch from +Victor van den Elzen. + +.. + +.. bpo: 16181 +.. date: 8537 +.. nonce: P7lLvo +.. section: Library + +cookiejar.http2time() now returns None if year is higher than +datetime.MAXYEAR. + +.. + +.. bpo: 26513 +.. date: 8536 +.. nonce: HoPepy +.. section: Library + +Fixes platform module detection of Windows Server + +.. + +.. bpo: 23718 +.. date: 8535 +.. nonce: AMPC0o +.. section: Library + +Fixed parsing time in week 0 before Jan 1. Original patch by Tamás Bence +Gedai. + +.. + +.. bpo: 20589 +.. date: 8534 +.. nonce: NsQ_I1 +.. section: Library + +Invoking Path.owner() and Path.group() on Windows now raise +NotImplementedError instead of ImportError. + +.. + +.. bpo: 26177 +.. date: 8533 +.. nonce: HlSWer +.. section: Library + +Fixed the keys() method for Canvas and Scrollbar widgets. + +.. + +.. bpo: 15068 +.. date: 8532 +.. nonce: bcHtiw +.. section: Library + +Got rid of excessive buffering in the fileinput module. The bufsize +parameter is no longer used. + +.. + +.. bpo: 2202 +.. date: 8531 +.. nonce: dk9sd0 +.. section: Library + +Fix UnboundLocalError in AbstractDigestAuthHandler.get_algorithm_impls. +Initial patch by Mathieu Dupuy. + +.. + +.. bpo: 25718 +.. date: 8530 +.. nonce: 4EjZyv +.. section: Library + +Fixed pickling and copying the accumulate() iterator with total is None. + +.. + +.. bpo: 26475 +.. date: 8529 +.. nonce: JXVccY +.. section: Library + +Fixed debugging output for regular expressions with the (?x) flag. + +.. + +.. bpo: 26457 +.. date: 8528 +.. nonce: Xe6Clh +.. section: Library + +Fixed the subnets() methods in IP network classes for the case when +resulting prefix length is equal to maximal prefix length. Based on patch by +Xiang Zhang. + +.. + +.. bpo: 26385 +.. date: 8527 +.. nonce: 50bDXm +.. section: Library + +Remove the file if the internal open() call in NamedTemporaryFile() fails. +Patch by Silent Ghost. + +.. + +.. bpo: 26402 +.. date: 8526 +.. nonce: k7DVuU +.. section: Library + +Fix XML-RPC client to retry when the server shuts down a persistent +connection. This was a regression related to the new +http.client.RemoteDisconnected exception in 3.5.0a4. + +.. + +.. bpo: 25913 +.. date: 8525 +.. nonce: 5flb95 +.. section: Library + +Leading ``<~`` is optional now in base64.a85decode() with adobe=True. Patch +by Swati Jaiswal. + +.. + +.. bpo: 26186 +.. date: 8524 +.. nonce: R9rfiL +.. section: Library + +Remove an invalid type check in importlib.util.LazyLoader. + +.. + +.. bpo: 26367 +.. date: 8523 +.. nonce: ckpNeU +.. section: Library + +importlib.__import__() raises SystemError like builtins.__import__() when +``level`` is specified but without an accompanying package specified. + +.. + +.. bpo: 26309 +.. date: 8522 +.. nonce: ubEeiz +.. section: Library + +In the "socketserver" module, shut down the request (closing the connected +socket) when verify_request() returns false. Patch by Aviv Palivoda. + +.. + +.. bpo: 25939 +.. date: 8521 +.. nonce: X49Fqd +.. original section: Library +.. section: Security + +On Windows open the cert store readonly in ssl.enum_certificates. + +.. + +.. bpo: 25995 +.. date: 8520 +.. nonce: NfcimP +.. section: Library + +os.walk() no longer uses FDs proportional to the tree depth. + +.. + +.. bpo: 26117 +.. date: 8519 +.. nonce: ne6p11 +.. section: Library + +The os.scandir() iterator now closes file descriptor not only when the +iteration is finished, but when it was failed with error. + +.. + +.. bpo: 25911 +.. date: 8518 +.. nonce: d4Zadh +.. section: Library + +Restored support of bytes paths in os.walk() on Windows. + +.. + +.. bpo: 26045 +.. date: 8517 +.. nonce: WmzUrX +.. section: Library + +Add UTF-8 suggestion to error message when posting a non-Latin-1 string with +http.client. + +.. + +.. bpo: 12923 +.. date: 8516 +.. nonce: HPAu-B +.. section: Library + +Reset FancyURLopener's redirect counter even if there is an exception. +Based on patches by Brian Brazil and Daniel Rocco. + +.. + +.. bpo: 25945 +.. date: 8515 +.. nonce: guNgNM +.. section: Library + +Fixed a crash when unpickle the functools.partial object with wrong state. +Fixed a leak in failed functools.partial constructor. "args" and "keywords" +attributes of functools.partial have now always types tuple and dict +correspondingly. + +.. + +.. bpo: 26202 +.. date: 8514 +.. nonce: LPIXLg +.. section: Library + +copy.deepcopy() now correctly copies range() objects with non-atomic +attributes. + +.. + +.. bpo: 23076 +.. date: 8513 +.. nonce: 8rphoP +.. section: Library + +Path.glob() now raises a ValueError if it's called with an invalid pattern. +Patch by Thomas Nyberg. + +.. + +.. bpo: 19883 +.. date: 8512 +.. nonce: z9TsO6 +.. section: Library + +Fixed possible integer overflows in zipimport. + +.. + +.. bpo: 26227 +.. date: 8511 +.. nonce: Fe6oiB +.. section: Library + +On Windows, getnameinfo(), gethostbyaddr() and gethostbyname_ex() functions +of the socket module now decode the hostname from the ANSI code page rather +than UTF-8. + +.. + +.. bpo: 26147 +.. date: 8510 +.. nonce: i-Jc01 +.. section: Library + +xmlrpc now works with strings not encodable with used non-UTF-8 encoding. + +.. + +.. bpo: 25935 +.. date: 8509 +.. nonce: cyni91 +.. section: Library + +Garbage collector now breaks reference loops with OrderedDict. + +.. + +.. bpo: 16620 +.. date: 8508 +.. nonce: rxpn_Y +.. section: Library + +Fixed AttributeError in msilib.Directory.glob(). + +.. + +.. bpo: 26013 +.. date: 8507 +.. nonce: 93RKNz +.. section: Library + +Added compatibility with broken protocol 2 pickles created in old Python 3 +versions (3.4.3 and lower). + +.. + +.. bpo: 25850 +.. date: 8506 +.. nonce: jwFPxj +.. section: Library + +Use cross-compilation by default for 64-bit Windows. + +.. + +.. bpo: 17633 +.. date: 8505 +.. nonce: 9mpbUO +.. section: Library + +Improve zipimport's support for namespace packages. + +.. + +.. bpo: 24705 +.. date: 8504 +.. nonce: IZYwjR +.. section: Library + +Fix sysconfig._parse_makefile not expanding ${} vars appearing before $() +vars. + +.. + +.. bpo: 22138 +.. date: 8503 +.. nonce: nRNYkc +.. section: Library + +Fix mock.patch behavior when patching descriptors. Restore original values +after patching. Patch contributed by Sean McCully. + +.. + +.. bpo: 25672 +.. date: 8502 +.. nonce: fw9RJP +.. section: Library + +In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is +safe to do so. + +.. + +.. bpo: 26012 +.. date: 8501 +.. nonce: IFSXNm +.. section: Library + +Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). + +.. + +.. bpo: 24120 +.. date: 8500 +.. nonce: Yiwa0h +.. section: Library + +Ignore PermissionError when traversing a tree with pathlib.Path.[r]glob(). +Patch by Ulrich Petri. + +.. + +.. bpo: 25447 +.. date: 8499 +.. nonce: -4m4xO +.. section: Library + +fileinput now uses sys.stdin as-is if it does not have a buffer attribute +(restores backward compatibility). + +.. + +.. bpo: 25447 +.. date: 8498 +.. nonce: AtHkWA +.. section: Library + +Copying the lru_cache() wrapper object now always works, independedly from +the type of the wrapped object (by returning the original object unchanged). + +.. + +.. bpo: 24103 +.. date: 8497 +.. nonce: WufqrQ +.. section: Library + +Fixed possible use after free in ElementTree.XMLPullParser. + +.. + +.. bpo: 25860 +.. date: 8496 +.. nonce: 0hActb +.. section: Library + +os.fwalk() no longer skips remaining directories when error occurs. +Original patch by Samson Lee. + +.. + +.. bpo: 25914 +.. date: 8495 +.. nonce: h0V61F +.. section: Library + +Fixed and simplified OrderedDict.__sizeof__. + +.. + +.. bpo: 25902 +.. date: 8494 +.. nonce: 6t2FmH +.. section: Library + +Fixed various refcount issues in ElementTree iteration. + +.. + +.. bpo: 25717 +.. date: 8493 +.. nonce: 0_xjaK +.. section: Library + +Restore the previous behaviour of tolerating most fstat() errors when +opening files. This was a regression in 3.5a1, and stopped anonymous +temporary files from working in special cases. + +.. + +.. bpo: 24903 +.. date: 8492 +.. nonce: 3LBdzb +.. section: Library + +Fix regression in number of arguments compileall accepts when '-d' is +specified. The check on the number of arguments has been dropped completely +as it never worked correctly anyway. + +.. + +.. bpo: 25764 +.. date: 8491 +.. nonce: 7WWG07 +.. section: Library + +In the subprocess module, preserve any exception caused by fork() failure +when preexec_fn is used. + +.. + +.. bpo: 6478 +.. date: 8490 +.. nonce: -Bi9Hb +.. section: Library + +_strptime's regexp cache now is reset after changing timezone with +time.tzset(). + +.. + +.. bpo: 14285 +.. date: 8489 +.. nonce: UyG8Hj +.. section: Library + +When executing a package with the "python -m package" option, and package +initialization fails, a proper traceback is now reported. The "runpy" +module now lets exceptions from package initialization pass back to the +caller, rather than raising ImportError. + +.. + +.. bpo: 19771 +.. date: 8488 +.. nonce: 5NG-bg +.. section: Library + +Also in runpy and the "-m" option, omit the irrelevant message ". . . is a +package and cannot be directly executed" if the package could not even be +initialized (e.g. due to a bad ``*.pyc`` file). + +.. + +.. bpo: 25177 +.. date: 8487 +.. nonce: aNR4Ha +.. section: Library + +Fixed problem with the mean of very small and very large numbers. As a side +effect, statistics.mean and statistics.variance should be significantly +faster. + +.. + +.. bpo: 25718 +.. date: 8486 +.. nonce: D9mHZF +.. section: Library + +Fixed copying object with state with boolean value is false. + +.. + +.. bpo: 10131 +.. date: 8485 +.. nonce: a7tptz +.. section: Library + +Fixed deep copying of minidom documents. Based on patch by Marian Ganisin. + +.. + +.. bpo: 25725 +.. date: 8484 +.. nonce: XIKv3R +.. section: Library + +Fixed a reference leak in pickle.loads() when unpickling invalid data +including tuple instructions. + +.. + +.. bpo: 25663 +.. date: 8483 +.. nonce: Ofwfqa +.. section: Library + +In the Readline completer, avoid listing duplicate global names, and search +the global namespace before searching builtins. + +.. + +.. bpo: 25688 +.. date: 8482 +.. nonce: 8P1HOv +.. section: Library + +Fixed file leak in ElementTree.iterparse() raising an error. + +.. + +.. bpo: 23914 +.. date: 8481 +.. nonce: 1sEz4J +.. section: Library + +Fixed SystemError raised by unpickler on broken pickle data. + +.. + +.. bpo: 25691 +.. date: 8480 +.. nonce: ZEaapY +.. section: Library + +Fixed crash on deleting ElementTree.Element attributes. + +.. + +.. bpo: 25624 +.. date: 8479 +.. nonce: ed-fM0 +.. section: Library + +ZipFile now always writes a ZIP_STORED header for directory entries. Patch +by Dingyuan Wang. + +.. + +.. bpo: 0 +.. date: 8478 +.. nonce: rtZyid +.. section: Library + +Skip getaddrinfo if host is already resolved. Patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26050 +.. date: 8477 +.. nonce: sclyvk +.. section: Library + +Add asyncio.StreamReader.readuntil() method. Patch by Марк Коренберг. + +.. + +.. bpo: 25924 +.. date: 8476 +.. nonce: Uxr2vt +.. section: Library + +Avoid unnecessary serialization of getaddrinfo(3) calls on OS X versions +10.5 or higher. Original patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26406 +.. date: 8475 +.. nonce: ihvhF4 +.. section: Library + +Avoid unnecessary serialization of getaddrinfo(3) calls on current versions +of OpenBSD and NetBSD. Patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26848 +.. date: 8474 +.. nonce: ChBOpQ +.. section: Library + +Fix asyncio/subprocess.communicate() to handle empty input. Patch by Jack +O'Connor. + +.. + +.. bpo: 27040 +.. date: 8473 +.. nonce: UASyCC +.. section: Library + +Add loop.get_exception_handler method + +.. + +.. bpo: 27041 +.. date: 8472 +.. nonce: p3893U +.. section: Library + +asyncio: Add loop.create_future method + +.. + +.. bpo: 27223 +.. date: 8471 +.. nonce: PRf4I6 +.. section: Library + +asyncio: Fix _read_ready and _write_ready to respect _conn_lost. Patch by +Łukasz Langa. + +.. + +.. bpo: 22970 +.. date: 8470 +.. nonce: WhdhyM +.. section: Library + +asyncio: Fix inconsistency cancelling Condition.wait. Patch by David Coles. + +.. + +.. bpo: 5124 +.. date: 8469 +.. nonce: 4kwBvM +.. section: IDLE + +Paste with text selected now replaces the selection on X11. This matches how +paste works on Windows, Mac, most modern Linux apps, and ttk widgets. +Original patch by Serhiy Storchaka. + +.. + +.. bpo: 24759 +.. date: 8468 +.. nonce: ccmySu +.. section: IDLE + +Make clear in idlelib.idle_test.__init__ that the directory is a private +implementation of test.test_idle and tool for maintainers. + +.. + +.. bpo: 27196 +.. date: 8467 +.. nonce: 3yp8TF +.. section: IDLE + +Stop 'ThemeChanged' warnings when running IDLE tests. These persisted after +other warnings were suppressed in #20567. Apply Serhiy Storchaka's +update_idletasks solution to four test files. Record this additional advice +in idle_test/README.txt + +.. + +.. bpo: 20567 +.. date: 8466 +.. nonce: hhT32b +.. section: IDLE + +Revise idle_test/README.txt with advice about avoiding tk warning messages +from tests. Apply advice to several IDLE tests. + +.. + +.. bpo: 27117 +.. date: 8465 +.. nonce: YrLPf4 +.. section: IDLE + +Make colorizer htest and turtledemo work with dark themes. Move code for +configuring text widget colors to a new function. + +.. + +.. bpo: 26673 +.. date: 8464 +.. nonce: dh0_Ij +.. section: IDLE + +When tk reports font size as 0, change to size 10. Such fonts on Linux +prevented the configuration dialog from opening. + +.. + +.. bpo: 21939 +.. date: 8463 +.. nonce: pWz-OK +.. section: IDLE + +Add test for IDLE's percolator. Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 21676 +.. date: 8462 +.. nonce: hqy6Qh +.. section: IDLE + +Add test for IDLE's replace dialog. Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 18410 +.. date: 8461 +.. nonce: DLSPZo +.. section: IDLE + +Add test for IDLE's search dialog. Original patch by Westley Martínez. + +.. + +.. bpo: 21703 +.. date: 8460 +.. nonce: BAZfDM +.. section: IDLE + +Add test for IDLE's undo delegator. Original patch by Saimadhav Heblikar . + +.. + +.. bpo: 27044 +.. date: 8459 +.. nonce: 4y7tyM +.. section: IDLE + +Add ConfigDialog.remove_var_callbacks to stop memory leaks. + +.. + +.. bpo: 23977 +.. date: 8458 +.. nonce: miDjj8 +.. section: IDLE + +Add more asserts to test_delegator. + +.. + +.. bpo: 20640 +.. date: 8457 +.. nonce: PmI-G8 +.. section: IDLE + +Add tests for idlelib.configHelpSourceEdit. Patch by Saimadhav Heblikar. + +.. + +.. bpo: 0 +.. date: 8456 +.. nonce: _YJfG7 +.. section: IDLE + +In the 'IDLE-console differences' section of the IDLE doc, clarify how +running with IDLE affects sys.modules and the standard streams. + +.. + +.. bpo: 25507 +.. date: 8455 +.. nonce: i8bNpk +.. section: IDLE + +fix incorrect change in IOBinding that prevented printing. Augment IOBinding +htest to include all major IOBinding functions. + +.. + +.. bpo: 25905 +.. date: 8454 +.. nonce: FzNb3B +.. section: IDLE + +Revert unwanted conversion of ' to ’ RIGHT SINGLE QUOTATION MARK in +README.txt and open this and NEWS.txt with 'ascii'. Re-encode CREDITS.txt to +utf-8 and open it with 'utf-8'. + +.. + +.. bpo: 19489 +.. date: 8453 +.. nonce: jvzuO7 +.. section: Documentation + +Moved the search box from the sidebar to the header and footer of each page. +Patch by Ammar Askar. + +.. + +.. bpo: 24136 +.. date: 8452 +.. nonce: MUK0zK +.. section: Documentation + +Document the new PEP 448 unpacking syntax of 3.5. + +.. + +.. bpo: 26736 +.. date: 8451 +.. nonce: U_Hyqo +.. section: Documentation + +Used HTTPS for external links in the documentation if possible. + +.. + +.. bpo: 6953 +.. date: 8450 +.. nonce: Zk6rno +.. section: Documentation + +Rework the Readline module documentation to group related functions +together, and add more details such as what underlying Readline functions +and variables are accessed. + +.. + +.. bpo: 23606 +.. date: 8449 +.. nonce: 9MhIso +.. section: Documentation + +Adds note to ctypes documentation regarding cdll.msvcrt. + +.. + +.. bpo: 25500 +.. date: 8448 +.. nonce: AV47eF +.. section: Documentation + +Fix documentation to not claim that __import__ is searched for in the global +scope. + +.. + +.. bpo: 26014 +.. date: 8447 +.. nonce: ptdZ_I +.. section: Documentation + +Update 3.x packaging documentation: * "See also" links to the new docs are +now provided in the legacy pages * links to setuptools documentation have +been updated + +.. + +.. bpo: 21916 +.. date: 8446 +.. nonce: muwCyp +.. section: Tests + +Added tests for the turtle module. Patch by ingrid, Gregory Loyse and Jelle +Zijlstra. + +.. + +.. bpo: 26523 +.. date: 8445 +.. nonce: em_Uzt +.. section: Tests + +The multiprocessing thread pool (multiprocessing.dummy.Pool) was untested. + +.. + +.. bpo: 26015 +.. date: 8444 +.. nonce: p3oWK3 +.. section: Tests + +Added new tests for pickling iterators of mutable sequences. + +.. + +.. bpo: 26325 +.. date: 8443 +.. nonce: KOUc82 +.. section: Tests + +Added test.support.check_no_resource_warning() to check that no +ResourceWarning is emitted. + +.. + +.. bpo: 25940 +.. date: 8442 +.. nonce: PgiLVN +.. section: Tests + +Changed test_ssl to use self-signed.pythontest.net. This avoids relying on +svn.python.org, which recently changed root certificate. + +.. + +.. bpo: 25616 +.. date: 8441 +.. nonce: Qr-60p +.. section: Tests + +Tests for OrderedDict are extracted from test_collections into separate file +test_ordered_dict. + +.. + +.. bpo: 26583 +.. date: 8440 +.. nonce: Up7hTl +.. section: Tests + +Skip test_timestamp_overflow in test_import if bytecode files cannot be +written. + +.. + +.. bpo: 26884 +.. date: 8439 +.. nonce: O8-azL +.. section: Build + +Fix linking extension modules for cross builds. Patch by Xavier de Gaye. + +.. + +.. bpo: 22359 +.. date: 8438 +.. nonce: HDjM4s +.. section: Build + +Disable the rules for running _freeze_importlib and pgen when cross- +compiling. The output of these programs is normally saved with the source +code anyway, and is still regenerated when doing a native build. Patch by +Xavier de Gaye. + +.. + +.. bpo: 27229 +.. date: 8437 +.. nonce: C2NDch +.. section: Build + +Fix the cross-compiling pgen rule for in-tree builds. Patch by Xavier de +Gaye. + +.. + +.. bpo: 21668 +.. date: 8436 +.. nonce: 4sMAa1 +.. section: Build + +Link audioop, _datetime, _ctypes_test modules to libm, except on Mac OS X. +Patch written by Xavier de Gaye. + +.. + +.. bpo: 25702 +.. date: 8435 +.. nonce: ipxyJs +.. section: Build + +A --with-lto configure option has been added that will enable link time +optimizations at build time during a make profile-opt. Some compilers and +toolchains are known to not produce stable code when using LTO, be sure to +test things thoroughly before relying on it. It can provide a few % speed up +over profile-opt alone. + +.. + +.. bpo: 26624 +.. date: 8434 +.. nonce: 4fGrTl +.. section: Build + +Adds validation of ucrtbase[d].dll version with warning for old versions. + +.. + +.. bpo: 17603 +.. date: 8433 +.. nonce: 102DA- +.. section: Build + +Avoid error about nonexistant fileblocks.o file by using a lower-level check +for st_blocks in struct stat. + +.. + +.. bpo: 26079 +.. date: 8432 +.. nonce: mEzW0O +.. section: Build + +Fixing the build output folder for tix-8.4.3.6. Patch by Bjoern Thiel. + +.. + +.. bpo: 26465 +.. date: 8431 +.. nonce: _YR608 +.. section: Build + +Update Windows builds to use OpenSSL 1.0.2g. + +.. + +.. bpo: 24421 +.. date: 8430 +.. nonce: 2zY7vM +.. section: Build + +Compile Modules/_math.c once, before building extensions. Previously it +could fail to compile properly if the math and cmath builds were concurrent. + +.. + +.. bpo: 25348 +.. date: 8429 +.. nonce: u6_BaQ +.. section: Build + +Added ``--pgo`` and ``--pgo-job`` arguments to ``PCbuild\build.bat`` for +building with Profile-Guided Optimization. The old +``PCbuild\build_pgo.bat`` script is now deprecated, and simply calls +``PCbuild\build.bat --pgo %*``. + +.. + +.. bpo: 25827 +.. date: 8428 +.. nonce: yg3DMM +.. section: Build + +Add support for building with ICC to ``configure``, including a new +``--with-icc`` flag. + +.. + +.. bpo: 25696 +.. date: 8427 +.. nonce: 2R_wIv +.. section: Build + +Fix installation of Python on UNIX with make -j9. + +.. + +.. bpo: 26930 +.. date: 8426 +.. nonce: Sqz2O3 +.. section: Build + +Update OS X 10.5+ 32-bit-only installer to build and link with OpenSSL +1.0.2h. + +.. + +.. bpo: 26268 +.. date: 8425 +.. nonce: I3-YLh +.. section: Build + +Update Windows builds to use OpenSSL 1.0.2f. + +.. + +.. bpo: 25136 +.. date: 8424 +.. nonce: Vi-fmO +.. section: Build + +Support Apple Xcode 7's new textual SDK stub libraries. + +.. + +.. bpo: 24324 +.. date: 8423 +.. nonce: m6DZMx +.. section: Build + +Do not enable unreachable code warnings when using gcc as the option does +not work correctly in older versions of gcc and has been silently removed as +of gcc-4.5. + +.. + +.. bpo: 27053 +.. date: 8422 +.. nonce: 1IRbae +.. section: Windows + +Updates make_zip.py to correctly generate library ZIP file. + +.. + +.. bpo: 26268 +.. date: 8421 +.. nonce: Z-lJEh +.. section: Windows + +Update the prepare_ssl.py script to handle OpenSSL releases that don't +include the contents of the include directory (that is, 1.0.2e and later). + +.. + +.. bpo: 26071 +.. date: 8420 +.. nonce: wLxL2l +.. section: Windows + +bdist_wininst created binaries fail to start and find 32bit Python + +.. + +.. bpo: 26073 +.. date: 8419 +.. nonce: XwWgHp +.. section: Windows + +Update the list of magic numbers in launcher + +.. + +.. bpo: 26065 +.. date: 8418 +.. nonce: SkVLJp +.. section: Windows + +Excludes venv from library when generating embeddable distro. + +.. + +.. bpo: 26799 +.. date: 8417 +.. nonce: gK2VXX +.. section: Tools/Demos + +Fix python-gdb.py: don't get C types once when the Python code is loaded, +but get C types on demand. The C types can change if python-gdb.py is loaded +before the Python executable. Patch written by Thomas Ilsche. + +.. + +.. bpo: 26271 +.. date: 8416 +.. nonce: wg-rzr +.. section: Tools/Demos + +Fix the Freeze tool to properly use flags passed through configure. Patch by +Daniel Shaulov. + +.. + +.. bpo: 26489 +.. date: 8415 +.. nonce: rJ_U5S +.. section: Tools/Demos + +Add dictionary unpacking support to Tools/parser/unparse.py. Patch by Guo Ci +Teo. + +.. + +.. bpo: 26316 +.. date: 8414 +.. nonce: QJvVOi +.. section: Tools/Demos + +Fix variable name typo in Argument Clinic. + +.. + +.. bpo: 17500 +.. date: 8413 +.. nonce: QTZbRV +.. section: Windows + +Remove unused and outdated icons. (See also: +https://github.com/python/pythondotorg/issues/945) diff --git a/Misc/NEWS.d/3.5.3.rst b/Misc/NEWS.d/3.5.3.rst new file mode 100644 index 00000000000000..716e156403c9fd --- /dev/null +++ b/Misc/NEWS.d/3.5.3.rst @@ -0,0 +1,7 @@ +.. bpo: 0 +.. date: 8859 +.. no changes: True +.. nonce: zYPqUK +.. release date: 2017-01-17 + +There were no code changes between 3.5.3rc1 and 3.5.3 final. diff --git a/Misc/NEWS.d/3.5.3rc1.rst b/Misc/NEWS.d/3.5.3rc1.rst new file mode 100644 index 00000000000000..caba4d403be5ef --- /dev/null +++ b/Misc/NEWS.d/3.5.3rc1.rst @@ -0,0 +1,2164 @@ +.. bpo: 29073 +.. date: 8858 +.. nonce: EFpHQ7 +.. release date: 2017-01-02 +.. section: Core and Builtins + +bytearray formatting no longer truncates on first null byte. + +.. + +.. bpo: 28932 +.. date: 8857 +.. nonce: QnLx8A +.. section: Core and Builtins + +Do not include if it does not exist. + +.. + +.. bpo: 28147 +.. date: 8856 +.. nonce: EV4bm6 +.. section: Core and Builtins + +Fix a memory leak in split-table dictionaries: setattr() must not convert +combined table into split table. + +.. + +.. bpo: 25677 +.. date: 8855 +.. nonce: RWhZrb +.. section: Core and Builtins + +Correct the positioning of the syntax error caret for indented blocks. +Based on patch by Michael Layzell. + +.. + +.. bpo: 29000 +.. date: 8854 +.. nonce: K6wQ-3 +.. section: Core and Builtins + +Fixed bytes formatting of octals with zero padding in alternate form. + +.. + +.. bpo: 28512 +.. date: 8853 +.. nonce: i-pv6d +.. section: Core and Builtins + +Fixed setting the offset attribute of SyntaxError by +PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +.. + +.. bpo: 28991 +.. date: 8852 +.. nonce: -qOTxS +.. section: Core and Builtins + +functools.lru_cache() was susceptible to an obscure reentrancy bug caused by +a monkey-patched len() function. + +.. + +.. bpo: 28648 +.. date: 8851 +.. nonce: z7B52W +.. section: Core and Builtins + +Fixed crash in Py_DecodeLocale() in debug build on Mac OS X when decode +astral characters. Patch by Xiang Zhang. + +.. + +.. bpo: 19398 +.. date: 8850 +.. nonce: RYbEGH +.. section: Core and Builtins + +Extra slash no longer added to sys.path components in case of empty compile- +time PYTHONPATH components. + +.. + +.. bpo: 28426 +.. date: 8849 +.. nonce: E_quyK +.. section: Core and Builtins + +Fixed potential crash in PyUnicode_AsDecodedObject() in debug build. + +.. + +.. bpo: 23782 +.. date: 8848 +.. nonce: lonDzj +.. section: Core and Builtins + +Fixed possible memory leak in _PyTraceback_Add() and exception loss in +PyTraceBack_Here(). + +.. + +.. bpo: 28379 +.. date: 8847 +.. nonce: DuXlco +.. section: Core and Builtins + +Added sanity checks and tests for PyUnicode_CopyCharacters(). Patch by Xiang +Zhang. + +.. + +.. bpo: 28376 +.. date: 8846 +.. nonce: oPD-5D +.. section: Core and Builtins + +The type of long range iterator is now registered as Iterator. Patch by Oren +Milman. + +.. + +.. bpo: 28376 +.. date: 8845 +.. nonce: fLeHM2 +.. section: Core and Builtins + +The constructor of range_iterator now checks that step is not 0. Patch by +Oren Milman. + +.. + +.. bpo: 26906 +.. date: 8844 +.. nonce: YBjcwI +.. section: Core and Builtins + +Resolving special methods of uninitialized type now causes implicit +initialization of the type instead of a fail. + +.. + +.. bpo: 18287 +.. date: 8843 +.. nonce: k6jffS +.. section: Core and Builtins + +PyType_Ready() now checks that tp_name is not NULL. Original patch by Niklas +Koep. + +.. + +.. bpo: 24098 +.. date: 8842 +.. nonce: XqlP_1 +.. section: Core and Builtins + +Fixed possible crash when AST is changed in process of compiling it. + +.. + +.. bpo: 28350 +.. date: 8841 +.. nonce: 8M5Eg9 +.. section: Core and Builtins + +String constants with null character no longer interned. + +.. + +.. bpo: 26617 +.. date: 8840 +.. nonce: Gh5LvN +.. section: Core and Builtins + +Fix crash when GC runs during weakref callbacks. + +.. + +.. bpo: 27942 +.. date: 8839 +.. nonce: ZGuhns +.. section: Core and Builtins + +String constants now interned recursively in tuples and frozensets. + +.. + +.. bpo: 21578 +.. date: 8838 +.. nonce: GI1bhj +.. section: Core and Builtins + +Fixed misleading error message when ImportError called with invalid keyword +args. + +.. + +.. bpo: 28203 +.. date: 8837 +.. nonce: kOgvtp +.. section: Core and Builtins + +Fix incorrect type in error message from ``complex(1.0, {2:3})``. Patch by +Soumya Sharma. + +.. + +.. bpo: 27955 +.. date: 8836 +.. nonce: HC4pZ4 +.. section: Core and Builtins + +Fallback on reading /dev/urandom device when the getrandom() syscall fails +with EPERM, for example when blocked by SECCOMP. + +.. + +.. bpo: 28131 +.. date: 8835 +.. nonce: owq0wW +.. section: Core and Builtins + +Fix a regression in zipimport's compile_source(). zipimport should use the +same optimization level as the interpreter. + +.. + +.. bpo: 25221 +.. date: 8834 +.. nonce: Zvkz9i +.. section: Core and Builtins + +Fix corrupted result from PyLong_FromLong(0) when Python is compiled with +NSMALLPOSINTS = 0. + +.. + +.. bpo: 25758 +.. date: 8833 +.. nonce: yR-YTD +.. section: Core and Builtins + +Prevents zipimport from unnecessarily encoding a filename (patch by Eryk +Sun) + +.. + +.. bpo: 28189 +.. date: 8832 +.. nonce: c_nbR_ +.. section: Core and Builtins + +dictitems_contains no longer swallows compare errors. (Patch by Xiang Zhang) + +.. + +.. bpo: 27812 +.. date: 8831 +.. nonce: sidcs8 +.. section: Core and Builtins + +Properly clear out a generator's frame's backreference to the generator to +prevent crashes in frame.clear(). + +.. + +.. bpo: 27811 +.. date: 8830 +.. nonce: T4AuBo +.. section: Core and Builtins + +Fix a crash when a coroutine that has not been awaited is finalized with +warnings-as-errors enabled. + +.. + +.. bpo: 27587 +.. date: 8829 +.. nonce: mbavY2 +.. section: Core and Builtins + +Fix another issue found by PVS-Studio: Null pointer check after use of 'def' +in _PyState_AddModule(). Initial patch by Christian Heimes. + +.. + +.. bpo: 26020 +.. date: 8828 +.. nonce: niLbLa +.. section: Core and Builtins + +set literal evaluation order did not match documented behaviour. + +.. + +.. bpo: 27782 +.. date: 8827 +.. nonce: C8OBQD +.. section: Core and Builtins + +Multi-phase extension module import now correctly allows the ``m_methods`` +field to be used to add module level functions to instances of non-module +types returned from ``Py_create_mod``. Patch by Xiang Zhang. + +.. + +.. bpo: 27936 +.. date: 8826 +.. nonce: AdOann +.. section: Core and Builtins + +The round() function accepted a second None argument for some types but not +for others. Fixed the inconsistency by accepting None for all numeric +types. + +.. + +.. bpo: 27487 +.. date: 8825 +.. nonce: jeTQNr +.. section: Core and Builtins + +Warn if a submodule argument to "python -m" or runpy.run_module() is found +in sys.modules after parent packages are imported, but before the submodule +is executed. + +.. + +.. bpo: 27558 +.. date: 8824 +.. nonce: VmltMh +.. section: Core and Builtins + +Fix a SystemError in the implementation of "raise" statement. In a brand new +thread, raise a RuntimeError since there is no active exception to reraise. +Patch written by Xiang Zhang. + +.. + +.. bpo: 27419 +.. date: 8823 +.. nonce: JZ94ju +.. section: Core and Builtins + +Standard __import__() no longer look up "__import__" in globals or builtins +for importing submodules or "from import". Fixed handling an error of non- +string package name. + +.. + +.. bpo: 27083 +.. date: 8822 +.. nonce: F4ZT1C +.. section: Core and Builtins + +Respect the PYTHONCASEOK environment variable under Windows. + +.. + +.. bpo: 27514 +.. date: 8821 +.. nonce: NLbwPG +.. section: Core and Builtins + +Make having too many statically nested blocks a SyntaxError instead of +SystemError. + +.. + +.. bpo: 27473 +.. date: 8820 +.. nonce: _nOtTA +.. section: Core and Builtins + +Fixed possible integer overflow in bytes and bytearray concatenations. +Patch by Xiang Zhang. + +.. + +.. bpo: 27507 +.. date: 8819 +.. nonce: 3pX0Be +.. section: Core and Builtins + +Add integer overflow check in bytearray.extend(). Patch by Xiang Zhang. + +.. + +.. bpo: 27581 +.. date: 8818 +.. nonce: KezjNt +.. section: Core and Builtins + +Don't rely on wrapping for overflow check in PySequence_Tuple(). Patch by +Xiang Zhang. + +.. + +.. bpo: 27443 +.. date: 8817 +.. nonce: 87ZwZ1 +.. section: Core and Builtins + +__length_hint__() of bytearray iterators no longer return a negative integer +for a resized bytearray. + +.. + +.. bpo: 27942 +.. date: 8816 +.. nonce: wCAkW5 +.. section: Core and Builtins + +Fix memory leak in codeobject.c + +.. + +.. bpo: 15812 +.. date: 8815 +.. nonce: R1U-Ec +.. section: Library + +inspect.getframeinfo() now correctly shows the first line of a context. +Patch by Sam Breese. + +.. + +.. bpo: 29094 +.. date: 8814 +.. nonce: 460ZQo +.. section: Library + +Offsets in a ZIP file created with extern file object and modes "w" and "x" +now are relative to the start of the file. + +.. + +.. bpo: 13051 +.. date: 8813 +.. nonce: YzC1Te +.. section: Library + +Fixed recursion errors in large or resized curses.textpad.Textbox. Based on +patch by Tycho Andersen. + +.. + +.. bpo: 29119 +.. date: 8812 +.. nonce: Ov69fr +.. section: Library + +Fix weakrefs in the pure python version of collections.OrderedDict +move_to_end() method. Contributed by Andra Bogildea. + +.. + +.. bpo: 9770 +.. date: 8811 +.. nonce: WJJnwP +.. section: Library + +curses.ascii predicates now work correctly with negative integers. + +.. + +.. bpo: 28427 +.. date: 8810 +.. nonce: vUd-va +.. section: Library + +old keys should not remove new values from WeakValueDictionary when +collecting from another thread. + +.. + +.. bpo: 28923 +.. date: 8809 +.. nonce: naVULD +.. section: Library + +Remove editor artifacts from Tix.py. + +.. + +.. bpo: 28871 +.. date: 8808 +.. nonce: cPMXCJ +.. section: Library + +Fixed a crash when deallocate deep ElementTree. + +.. + +.. bpo: 19542 +.. date: 8807 +.. nonce: 5tCkaK +.. section: Library + +Fix bugs in WeakValueDictionary.setdefault() and WeakValueDictionary.pop() +when a GC collection happens in another thread. + +.. + +.. bpo: 20191 +.. date: 8806 +.. nonce: P_EZ7c +.. section: Library + +Fixed a crash in resource.prlimit() when pass a sequence that doesn't own +its elements as limits. + +.. + +.. bpo: 28779 +.. date: 8805 +.. nonce: t-mjED +.. section: Library + +multiprocessing.set_forkserver_preload() would crash the forkserver process +if a preloaded module instantiated some multiprocessing objects such as +locks. + +.. + +.. bpo: 28847 +.. date: 8804 +.. nonce: J7d3nG +.. section: Library + +dbm.dumb now supports reading read-only files and no longer writes the index +file when it is not changed. + +.. + +.. bpo: 25659 +.. date: 8803 +.. nonce: lE2IlT +.. section: Library + +In ctypes, prevent a crash calling the from_buffer() and from_buffer_copy() +methods on abstract classes like Array. + +.. + +.. bpo: 28732 +.. date: 8802 +.. nonce: xkG8k7 +.. section: Library + +Fix crash in os.spawnv() with no elements in args + +.. + +.. bpo: 28485 +.. date: 8801 +.. nonce: WuKqKh +.. section: Library + +Always raise ValueError for negative compileall.compile_dir(workers=...) +parameter, even when multithreading is unavailable. + +.. + +.. bpo: 28387 +.. date: 8800 +.. nonce: 1clJu7 +.. section: Library + +Fixed possible crash in _io.TextIOWrapper deallocator when the garbage +collector is invoked in other thread. Based on patch by Sebastian Cufre. + +.. + +.. bpo: 27517 +.. date: 8799 +.. nonce: 1CYM8A +.. section: Library + +LZMA compressor and decompressor no longer raise exceptions if given empty +data twice. Patch by Benjamin Fogle. + +.. + +.. bpo: 28549 +.. date: 8798 +.. nonce: ShnM2y +.. section: Library + +Fixed segfault in curses's addch() with ncurses6. + +.. + +.. bpo: 28449 +.. date: 8797 +.. nonce: 5JK6ES +.. section: Library + +tarfile.open() with mode "r" or "r:" now tries to open a tar file with +compression before trying to open it without compression. Otherwise it had +50% chance failed with ignore_zeros=True. + +.. + +.. bpo: 23262 +.. date: 8796 +.. nonce: 6EVB7N +.. section: Library + +The webbrowser module now supports Firefox 36+ and derived browsers. Based +on patch by Oleg Broytman. + +.. + +.. bpo: 27939 +.. date: 8795 +.. nonce: mTfADV +.. section: Library + +Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused by +representing the scale as float value internally in Tk. tkinter.IntVar now +works if float value is set to underlying Tk variable. + +.. + +.. bpo: 28255 +.. date: 8794 +.. nonce: _ZH4wm +.. section: Library + +calendar.TextCalendar().prmonth() no longer prints a space at the start of +new line after printing a month's calendar. Patch by Xiang Zhang. + +.. + +.. bpo: 20491 +.. date: 8793 +.. nonce: ObgnQ2 +.. section: Library + +The textwrap.TextWrapper class now honors non-breaking spaces. Based on +patch by Kaarle Ritvanen. + +.. + +.. bpo: 28353 +.. date: 8792 +.. nonce: sKGbLL +.. section: Library + +os.fwalk() no longer fails on broken links. + +.. + +.. bpo: 25464 +.. date: 8791 +.. nonce: HDUTCu +.. section: Library + +Fixed HList.header_exists() in tkinter.tix module by addin a workaround to +Tix library bug. + +.. + +.. bpo: 28488 +.. date: 8790 +.. nonce: NlkjBM +.. section: Library + +shutil.make_archive() no longer add entry "./" to ZIP archive. + +.. + +.. bpo: 24452 +.. date: 8789 +.. nonce: m9Kyg3 +.. section: Library + +Make webbrowser support Chrome on Mac OS X. + +.. + +.. bpo: 20766 +.. date: 8788 +.. nonce: 4kvCzx +.. section: Library + +Fix references leaked by pdb in the handling of SIGINT handlers. + +.. + +.. bpo: 26293 +.. date: 8787 +.. nonce: 2mjvwX +.. section: Library + +Fixed writing ZIP files that starts not from the start of the file. Offsets +in ZIP file now are relative to the start of the archive in conforming to +the specification. + +.. + +.. bpo: 28321 +.. date: 8786 +.. nonce: bQ-IIX +.. section: Library + +Fixed writing non-BMP characters with binary format in plistlib. + +.. + +.. bpo: 28322 +.. date: 8785 +.. nonce: l9hzap +.. section: Library + +Fixed possible crashes when unpickle itertools objects from incorrect pickle +data. Based on patch by John Leitch. + +.. + +.. bpo: 0 +.. date: 8784 +.. nonce: 81jNns +.. section: Library + +Fix possible integer overflows and crashes in the mmap module with unusual +usage patterns. + +.. + +.. bpo: 1703178 +.. date: 8783 +.. nonce: meb49K +.. section: Library + +Fix the ability to pass the --link-objects option to the distutils build_ext +command. + +.. + +.. bpo: 28253 +.. date: 8782 +.. nonce: aLfmhe +.. section: Library + +Fixed calendar functions for extreme months: 0001-01 and 9999-12. + +Methods itermonthdays() and itermonthdays2() are reimplemented so that they +don't call itermonthdates() which can cause datetime.date under/overflow. + +.. + +.. bpo: 28275 +.. date: 8781 +.. nonce: EhWIsz +.. section: Library + +Fixed possible use after free in the decompress() methods of the +LZMADecompressor and BZ2Decompressor classes. Original patch by John Leitch. + +.. + +.. bpo: 27897 +.. date: 8780 +.. nonce: I0Ppmx +.. section: Library + +Fixed possible crash in sqlite3.Connection.create_collation() if pass +invalid string-like object as a name. Patch by Xiang Zhang. + +.. + +.. bpo: 18893 +.. date: 8779 +.. nonce: osiX5c +.. section: Library + +Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. Patch by +Madison May. + +.. + +.. bpo: 27611 +.. date: 8778 +.. nonce: qL-UVQ +.. section: Library + +Fixed support of default root window in the tkinter.tix module. + +.. + +.. bpo: 27348 +.. date: 8777 +.. nonce: tDx7Vw +.. section: Library + +In the traceback module, restore the formatting of exception messages like +"Exception: None". This fixes a regression introduced in 3.5a2. + +.. + +.. bpo: 25651 +.. date: 8776 +.. nonce: 3UhyPo +.. section: Library + +Allow falsy values to be used for msg parameter of subTest(). + +.. + +.. bpo: 27932 +.. date: 8775 +.. nonce: mtgl-6 +.. section: Library + +Prevent memory leak in win32_ver(). + +.. + +.. bpo: 0 +.. date: 8774 +.. nonce: iPpjqX +.. section: Library + +Fix UnboundLocalError in socket._sendfile_use_sendfile. + +.. + +.. bpo: 28075 +.. date: 8773 +.. nonce: aLiUs9 +.. section: Library + +Check for ERROR_ACCESS_DENIED in Windows implementation of os.stat(). Patch +by Eryk Sun. + +.. + +.. bpo: 25270 +.. date: 8772 +.. nonce: jrZruM +.. section: Library + +Prevent codecs.escape_encode() from raising SystemError when an empty +bytestring is passed. + +.. + +.. bpo: 28181 +.. date: 8771 +.. nonce: NGc4Yv +.. section: Library + +Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +.. + +.. bpo: 25895 +.. date: 8770 +.. nonce: j92qoQ +.. section: Library + +Enable WebSocket URL schemes in urllib.parse.urljoin. Patch by Gergely Imreh +and Markus Holtermann. + +.. + +.. bpo: 27599 +.. date: 8769 +.. nonce: itvm8T +.. section: Library + +Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +.. + +.. bpo: 19003 +.. date: 8768 +.. nonce: UUcK_F +.. section: Library + +m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per +the RFC, instead of all unicode line endings. + +.. + +.. bpo: 28019 +.. date: 8767 +.. nonce: KUhBaS +.. section: Library + +itertools.count() no longer rounds non-integer step in range between 1.0 and +2.0 to 1. + +.. + +.. bpo: 25969 +.. date: 8766 +.. nonce: qSPkl- +.. section: Library + +Update the lib2to3 grammar to handle the unpacking generalizations added in +3.5. + +.. + +.. bpo: 14977 +.. date: 8765 +.. nonce: 4MvALg +.. section: Library + +mailcap now respects the order of the lines in the mailcap files ("first +match"), as required by RFC 1542. Patch by Michael Lazar. + +.. + +.. bpo: 24594 +.. date: 8764 +.. nonce: 9CnFVS +.. section: Library + +Validates persist parameter when opening MSI database + +.. + +.. bpo: 17582 +.. date: 8763 +.. nonce: MXEHxQ +.. section: Library + +xml.etree.ElementTree nows preserves whitespaces in attributes (Patch by +Duane Griffin. Reviewed and approved by Stefan Behnel.) + +.. + +.. bpo: 28047 +.. date: 8762 +.. nonce: pDu3Fm +.. section: Library + +Fixed calculation of line length used for the base64 CTE in the new email +policies. + +.. + +.. bpo: 27445 +.. date: 8761 +.. nonce: wOG0C0 +.. section: Library + +Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. + +.. + +.. bpo: 22450 +.. date: 8760 +.. nonce: T3Sn_J +.. section: Library + +urllib now includes an ``Accept: */*`` header among the default headers. +This makes the results of REST API requests more consistent and predictable +especially when proxy servers are involved. + +.. + +.. bpo: 0 +.. date: 8759 +.. nonce: PVZStR +.. section: Library + +lib2to3.pgen3.driver.load_grammar() now creates a stable cache file between +runs given the same Grammar.txt input regardless of the hash randomization +setting. + +.. + +.. bpo: 27570 +.. date: 8758 +.. nonce: pU0Zie +.. section: Library + +Avoid zero-length memcpy() etc calls with null source pointers in the +"ctypes" and "array" modules. + +.. + +.. bpo: 22233 +.. date: 8757 +.. nonce: uXSN0R +.. section: Library + +Break email header lines *only* on the RFC specified CR and LF characters, +not on arbitrary unicode line breaks. This also fixes a bug in HTTP header +parsing. + +.. + +.. bpo: 27988 +.. date: 8756 +.. nonce: VfMzZH +.. section: Library + +Fix email iter_attachments incorrect mutation of payload list. + +.. + +.. bpo: 27691 +.. date: 8755 +.. nonce: TMYF5_ +.. section: Library + +Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 +certs. + +.. + +.. bpo: 27850 +.. date: 8754 +.. nonce: kIVQ0m +.. section: Library + +Remove 3DES from ssl module's default cipher list to counter measure sweet32 +attack (CVE-2016-2183). + +.. + +.. bpo: 27766 +.. date: 8753 +.. nonce: WI70Tc +.. section: Library + +Add ChaCha20 Poly1305 to ssl module's default ciper list. (Required OpenSSL +1.1.0 or LibreSSL). + +.. + +.. bpo: 26470 +.. date: 8752 +.. nonce: QGu_wo +.. section: Library + +Port ssl and hashlib module to OpenSSL 1.1.0. + +.. + +.. bpo: 0 +.. date: 8751 +.. nonce: 6TjEgz +.. section: Library + +Remove support for passing a file descriptor to os.access. It never worked +but previously didn't raise. + +.. + +.. bpo: 12885 +.. date: 8750 +.. nonce: r-IV1g +.. section: Library + +Fix error when distutils encounters symlink. + +.. + +.. bpo: 27881 +.. date: 8749 +.. nonce: fkETd9 +.. section: Library + +Fixed possible bugs when setting sqlite3.Connection.isolation_level. Based +on patch by Xiang Zhang. + +.. + +.. bpo: 27861 +.. date: 8748 +.. nonce: DBYuo9 +.. section: Library + +Fixed a crash in sqlite3.Connection.cursor() when a factory creates not a +cursor. Patch by Xiang Zhang. + +.. + +.. bpo: 19884 +.. date: 8747 +.. nonce: MO8AWH +.. section: Library + +Avoid spurious output on OS X with Gnu Readline. + +.. + +.. bpo: 27706 +.. date: 8746 +.. nonce: ZY67yu +.. section: Library + +Restore deterministic behavior of random.Random().seed() for string seeds +using seeding version 1. Allows sequences of calls to random() to exactly +match those obtained in Python 2. Patch by Nofar Schnider. + +.. + +.. bpo: 10513 +.. date: 8745 +.. nonce: tQIQD_ +.. section: Library + +Fix a regression in Connection.commit(). Statements should not be reset +after a commit. + +.. + +.. bpo: 0 +.. date: 8744 +.. nonce: cYraeH +.. section: Library + +A new version of typing.py from https://github.com/python/typing: - +Collection (only for 3.6) (Issue #27598) - Add FrozenSet to __all__ +(upstream #261) - fix crash in _get_type_vars() (upstream #259) - Remove the +dict constraint in ForwardRef._eval_type (upstream #252) + +.. + +.. bpo: 27539 +.. date: 8743 +.. nonce: S4L1cq +.. section: Library + +Fix unnormalised ``Fraction.__pow__`` result in the case of negative +exponent and negative base. + +.. + +.. bpo: 21718 +.. date: 8742 +.. nonce: FUJd-7 +.. section: Library + +cursor.description is now available for queries using CTEs. + +.. + +.. bpo: 2466 +.. date: 8741 +.. nonce: VRNlkg +.. section: Library + +posixpath.ismount now correctly recognizes mount points which the user does +not have permission to access. + +.. + +.. bpo: 27773 +.. date: 8740 +.. nonce: hMSSeX +.. section: Library + +Correct some memory management errors server_hostname in _ssl.wrap_socket(). + +.. + +.. bpo: 26750 +.. date: 8739 +.. nonce: rv76vt +.. section: Library + +unittest.mock.create_autospec() now works properly for subclasses of +property() and other data descriptors. + +.. + +.. bpo: 0 +.. date: 8738 +.. nonce: Ny9oPv +.. section: Library + +In the curses module, raise an error if window.getstr() or window.instr() is +passed a negative value. + +.. + +.. bpo: 27783 +.. date: 8737 +.. nonce: cR1jXH +.. section: Library + +Fix possible usage of uninitialized memory in operator.methodcaller. + +.. + +.. bpo: 27774 +.. date: 8736 +.. nonce: FDcik1 +.. section: Library + +Fix possible Py_DECREF on unowned object in _sre. + +.. + +.. bpo: 27760 +.. date: 8735 +.. nonce: gxMjp4 +.. section: Library + +Fix possible integer overflow in binascii.b2a_qp. + +.. + +.. bpo: 27758 +.. date: 8734 +.. nonce: 0NRV03 +.. section: Library + +Fix possible integer overflow in the _csv module for large record lengths. + +.. + +.. bpo: 27568 +.. date: 8733 +.. nonce: OnuO9s +.. section: Library + +Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +when REQUEST_METHOD environment is set, which indicates that the script is +in CGI mode. + +.. + +.. bpo: 27656 +.. date: 8732 +.. nonce: joTscM +.. section: Library + +Do not assume sched.h defines any SCHED_* constants. + +.. + +.. bpo: 27130 +.. date: 8731 +.. nonce: SUxwXZ +.. section: Library + +In the "zlib" module, fix handling of large buffers (typically 4 GiB) when +compressing and decompressing. Previously, inputs were limited to 4 GiB, +and compression and decompression operations did not properly handle results +of 4 GiB. + +.. + +.. bpo: 27533 +.. date: 8730 +.. nonce: iDmKzV +.. section: Library + +Release GIL in nt._isdir + +.. + +.. bpo: 17711 +.. date: 8729 +.. nonce: 47AILJ +.. section: Library + +Fixed unpickling by the persistent ID with protocol 0. Original patch by +Alexandre Vassalotti. + +.. + +.. bpo: 27522 +.. date: 8728 +.. nonce: 8vVz_t +.. section: Library + +Avoid an unintentional reference cycle in email.feedparser. + +.. + +.. bpo: 26844 +.. date: 8727 +.. nonce: I0wdnY +.. section: Library + +Fix error message for imp.find_module() to refer to 'path' instead of +'name'. Patch by Lev Maximov. + +.. + +.. bpo: 23804 +.. date: 8726 +.. nonce: ipFvxc +.. section: Library + +Fix SSL zero-length recv() calls to not block and not raise an error about +unclean EOF. + +.. + +.. bpo: 27466 +.. date: 8725 +.. nonce: C_3a8E +.. section: Library + +Change time format returned by http.cookie.time2netscape, confirming the +netscape cookie format and making it consistent with documentation. + +.. + +.. bpo: 26664 +.. date: 8724 +.. nonce: OzsSzf +.. section: Library + +Fix activate.fish by removing mis-use of ``$``. + +.. + +.. bpo: 22115 +.. date: 8723 +.. nonce: apoFQ9 +.. section: Library + +Fixed tracing Tkinter variables: trace_vdelete() with wrong mode no longer +break tracing, trace_vinfo() now always returns a list of pairs of strings, +tracing in the "u" mode now works. + +.. + +.. bpo: 0 +.. date: 8722 +.. nonce: oZOeFE +.. section: Library + +Fix a scoping issue in importlib.util.LazyLoader which triggered an +UnboundLocalError when lazy-loading a module that was already put into +sys.modules. + +.. + +.. bpo: 27079 +.. date: 8721 +.. nonce: c7d0Ym +.. section: Library + +Fixed curses.ascii functions isblank(), iscntrl() and ispunct(). + +.. + +.. bpo: 26754 +.. date: 8720 +.. nonce: J3n0QW +.. section: Library + +Some functions (compile() etc) accepted a filename argument encoded as an +iterable of integers. Now only strings and byte-like objects are accepted. + +.. + +.. bpo: 27048 +.. date: 8719 +.. nonce: EVe-Bk +.. section: Library + +Prevents distutils failing on Windows when environment variables contain +non-ASCII characters + +.. + +.. bpo: 27330 +.. date: 8718 +.. nonce: GJaFCV +.. section: Library + +Fixed possible leaks in the ctypes module. + +.. + +.. bpo: 27238 +.. date: 8717 +.. nonce: Q6v6Qv +.. section: Library + +Got rid of bare excepts in the turtle module. Original patch by Jelle +Zijlstra. + +.. + +.. bpo: 27122 +.. date: 8716 +.. nonce: 06t7zN +.. section: Library + +When an exception is raised within the context being managed by a +contextlib.ExitStack() and one of the exit stack generators catches and +raises it in a chain, do not re-raise the original exception when exiting, +let the new chained one through. This avoids the PEP 479 bug described in +issue25782. + +.. + +.. bpo: 27278 +.. date: 8715 +.. nonce: y_HkGw +.. original section: Library +.. section: Security + +Fix os.urandom() implementation using getrandom() on Linux. Truncate size +to INT_MAX and loop until we collected enough random bytes, instead of +casting a directly Py_ssize_t to int. + +.. + +.. bpo: 26386 +.. date: 8714 +.. nonce: 9L3Ut4 +.. section: Library + +Fixed ttk.TreeView selection operations with item id's containing spaces. + +.. + +.. bpo: 22636 +.. date: 8713 +.. nonce: 3fQW_g +.. original section: Library +.. section: Security + +Avoid shell injection problems with ctypes.util.find_library(). + +.. + +.. bpo: 16182 +.. date: 8712 +.. nonce: RgFXyr +.. section: Library + +Fix various functions in the "readline" module to use the locale encoding, +and fix get_begidx() and get_endidx() to return code point indexes. + +.. + +.. bpo: 27392 +.. date: 8711 +.. nonce: obfni7 +.. section: Library + +Add loop.connect_accepted_socket(). Patch by Jim Fulton. + +.. + +.. bpo: 27930 +.. date: 8710 +.. nonce: BkOfSi +.. section: Library + +Improved behaviour of logging.handlers.QueueListener. Thanks to Paulo +Andrade and Petr Viktorin for the analysis and patch. + +.. + +.. bpo: 21201 +.. date: 8709 +.. nonce: wLCKiA +.. section: Library + +Improves readability of multiprocessing error message. Thanks to Wojciech +Walczak for patch. + +.. + +.. bpo: 27456 +.. date: 8708 +.. nonce: lI_IE7 +.. section: Library + +asyncio: Set TCP_NODELAY by default. + +.. + +.. bpo: 27906 +.. date: 8707 +.. nonce: TBBXrv +.. section: Library + +Fix socket accept exhaustion during high TCP traffic. Patch by Kevin Conway. + +.. + +.. bpo: 28174 +.. date: 8706 +.. nonce: CV1UdI +.. section: Library + +Handle when SO_REUSEPORT isn't properly supported. Patch by Seth Michael +Larson. + +.. + +.. bpo: 26654 +.. date: 8705 +.. nonce: XtzTE9 +.. section: Library + +Inspect functools.partial in asyncio.Handle.__repr__. Patch by iceboy. + +.. + +.. bpo: 26909 +.. date: 8704 +.. nonce: ASiakT +.. section: Library + +Fix slow pipes IO in asyncio. Patch by INADA Naoki. + +.. + +.. bpo: 28176 +.. date: 8703 +.. nonce: sU8R6L +.. section: Library + +Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +.. + +.. bpo: 27759 +.. date: 8702 +.. nonce: qpMDGq +.. section: Library + +Fix selectors incorrectly retain invalid file descriptors. Patch by Mark +Williams. + +.. + +.. bpo: 28368 +.. date: 8701 +.. nonce: n594X4 +.. section: Library + +Refuse monitoring processes if the child watcher has no loop attached. Patch +by Vincent Michel. + +.. + +.. bpo: 28369 +.. date: 8700 +.. nonce: 8DTANe +.. section: Library + +Raise RuntimeError when transport's FD is used with add_reader, add_writer, +etc. + +.. + +.. bpo: 28370 +.. date: 8699 +.. nonce: 18jBuZ +.. section: Library + +Speedup asyncio.StreamReader.readexactly. Patch by Коренберг Марк. + +.. + +.. bpo: 28371 +.. date: 8698 +.. nonce: U9Zqdk +.. section: Library + +Deprecate passing asyncio.Handles to run_in_executor. + +.. + +.. bpo: 28372 +.. date: 8697 +.. nonce: njcIPk +.. section: Library + +Fix asyncio to support formatting of non-python coroutines. + +.. + +.. bpo: 28399 +.. date: 8696 +.. nonce: QKIqRX +.. section: Library + +Remove UNIX socket from FS before binding. Patch by Коренберг Марк. + +.. + +.. bpo: 27972 +.. date: 8695 +.. nonce: ZK-GFm +.. section: Library + +Prohibit Tasks to await on themselves. + +.. + +.. bpo: 26923 +.. date: 8694 +.. nonce: 8dh3AV +.. section: Library + +Fix asyncio.Gather to refuse being cancelled once all children are done. +Patch by Johannes Ebke. + +.. + +.. bpo: 26796 +.. date: 8693 +.. nonce: TZyAfJ +.. section: Library + +Don't configure the number of workers for default threadpool executor. +Initial patch by Hans Lawrenz. + +.. + +.. bpo: 28600 +.. date: 8692 +.. nonce: 2ThUQV +.. section: Library + +Optimize loop.call_soon(). + +.. + +.. bpo: 28613 +.. date: 8691 +.. nonce: sqUPrv +.. section: Library + +Fix get_event_loop() return the current loop if called from +coroutines/callbacks. + +.. + +.. bpo: 28639 +.. date: 8690 +.. nonce: WUPo1o +.. section: Library + +Fix inspect.isawaitable to always return bool Patch by Justin Mayfield. + +.. + +.. bpo: 28652 +.. date: 8689 +.. nonce: f5M8FG +.. section: Library + +Make loop methods reject socket kinds they do not support. + +.. + +.. bpo: 28653 +.. date: 8688 +.. nonce: S5bA9i +.. section: Library + +Fix a refleak in functools.lru_cache. + +.. + +.. bpo: 28703 +.. date: 8687 +.. nonce: CRLTJc +.. section: Library + +Fix asyncio.iscoroutinefunction to handle Mock objects. + +.. + +.. bpo: 24142 +.. date: 8686 +.. nonce: _BgogI +.. section: Library + +Reading a corrupt config file left the parser in an invalid state. Original +patch by Florian Höch. + +.. + +.. bpo: 28990 +.. date: 8685 +.. nonce: W8tuYZ +.. section: Library + +Fix SSL hanging if connection is closed before handshake completed. (Patch +by HoHo-Ho) + +.. + +.. bpo: 15308 +.. date: 8684 +.. nonce: zZxn8m +.. section: IDLE + +Add 'interrupt execution' (^C) to Shell menu. Patch by Roger Serwy, updated +by Bayard Randel. + +.. + +.. bpo: 27922 +.. date: 8683 +.. nonce: UEtEv9 +.. section: IDLE + +Stop IDLE tests from 'flashing' gui widgets on the screen. + +.. + +.. bpo: 0 +.. date: 8682 +.. nonce: zWZs6o +.. section: IDLE + +Add version to title of IDLE help window. + +.. + +.. bpo: 25564 +.. date: 8681 +.. nonce: GN0p14 +.. section: IDLE + +In section on IDLE -- console differences, mention that using exec means +that __builtins__ is defined for each statement. + +.. + +.. bpo: 27714 +.. date: 8680 +.. nonce: bUEDsI +.. section: IDLE + +text_textview and test_autocomplete now pass when re-run in the same +process. This occurs when test_idle fails when run with the -w option but +without -jn. Fix warning from test_config. + +.. + +.. bpo: 25507 +.. date: 8679 +.. nonce: lxf68d +.. section: IDLE + +IDLE no longer runs buggy code because of its tkinter imports. Users must +include the same imports required to run directly in Python. + +.. + +.. bpo: 27452 +.. date: 8678 +.. nonce: RtWnyR +.. section: IDLE + +add line counter and crc to IDLE configHandler test dump. + +.. + +.. bpo: 27365 +.. date: 8677 +.. nonce: y7ys_A +.. section: IDLE + +Allow non-ascii chars in IDLE NEWS.txt, for contributor names. + +.. + +.. bpo: 27245 +.. date: 8676 +.. nonce: u9aKO1 +.. section: IDLE + +IDLE: Cleanly delete custom themes and key bindings. Previously, when IDLE +was started from a console or by import, a cascade of warnings was emitted. +Patch by Serhiy Storchaka. + +.. + +.. bpo: 28808 +.. date: 8675 +.. nonce: A03X6r +.. section: C API + +PyUnicode_CompareWithASCIIString() now never raises exceptions. + +.. + +.. bpo: 26754 +.. date: 8674 +.. nonce: j2czHF +.. section: C API + +PyUnicode_FSDecoder() accepted a filename argument encoded as an iterable of +integers. Now only strings and bytes-like objects are accepted. + +.. + +.. bpo: 28513 +.. date: 8673 +.. nonce: L3joAz +.. section: Documentation + +Documented command-line interface of zipfile. + +.. + +.. bpo: 28950 +.. date: 8672 +.. nonce: 9_vY6R +.. section: Tests + +Disallow -j0 to be combined with -T/-l/-M in regrtest command line +arguments. + +.. + +.. bpo: 28666 +.. date: 8671 +.. nonce: RtTk-4 +.. section: Tests + +Now test.support.rmtree is able to remove unwritable or unreadable +directories. + +.. + +.. bpo: 23839 +.. date: 8670 +.. nonce: zsT_L9 +.. section: Tests + +Various caches now are cleared before running every test file. + +.. + +.. bpo: 28409 +.. date: 8669 +.. nonce: Q2IlxJ +.. section: Tests + +regrtest: fix the parser of command line arguments. + +.. + +.. bpo: 27787 +.. date: 8668 +.. nonce: kf0YAt +.. section: Tests + +Call gc.collect() before checking each test for "dangling threads", since +the dangling threads are weak references. + +.. + +.. bpo: 27369 +.. date: 8667 +.. nonce: LG7U2D +.. section: Tests + +In test_pyexpat, avoid testing an error message detail that changed in Expat +2.2.0. + +.. + +.. bpo: 27952 +.. date: 8666 +.. nonce: WX9Ufc +.. section: Tools/Demos + +Get Tools/scripts/fixcid.py working with Python 3 and the current "re" +module, avoid invalid Python backslash escapes, and fix a bug parsing +escaped C quote signs. + +.. + +.. bpo: 27332 +.. date: 8665 +.. nonce: OuRZp9 +.. section: Tools/Demos + +Fixed the type of the first argument of module-level functions generated by +Argument Clinic. Patch by Petr Viktorin. + +.. + +.. bpo: 27418 +.. date: 8664 +.. nonce: W2m_8I +.. section: Tools/Demos + +Fixed Tools/importbench/importbench.py. + +.. + +.. bpo: 28251 +.. date: 8663 +.. nonce: tR_AFs +.. section: Windows + +Improvements to help manuals on Windows. + +.. + +.. bpo: 28110 +.. date: 8662 +.. nonce: cnkP5F +.. section: Windows + +launcher.msi has different product codes between 32-bit and 64-bit + +.. + +.. bpo: 25144 +.. date: 8661 +.. nonce: iUha52 +.. section: Windows + +Ensures TargetDir is set before continuing with custom install. + +.. + +.. bpo: 27469 +.. date: 8660 +.. nonce: 0GwDkX +.. section: Windows + +Adds a shell extension to the launcher so that drag and drop works +correctly. + +.. + +.. bpo: 27309 +.. date: 8659 +.. nonce: 4DPjhF +.. section: Windows + +Enabled proper Windows styles in python[w].exe manifest. + +.. + +.. bpo: 29080 +.. date: 8658 +.. nonce: b3qLQT +.. section: Build + +Removes hard dependency on hg.exe from PCBuild/build.bat + +.. + +.. bpo: 23903 +.. date: 8657 +.. nonce: JXJ889 +.. section: Build + +Added missed names to PC/python3.def. + +.. + +.. bpo: 10656 +.. date: 8656 +.. nonce: pR8FFU +.. section: Build + +Fix out-of-tree building on AIX. Patch by Tristan Carel and Michael +Haubenwallner. + +.. + +.. bpo: 26359 +.. date: 8655 +.. nonce: CLz6qy +.. section: Build + +Rename --with-optimiations to --enable-optimizations. + +.. + +.. bpo: 28444 +.. date: 8654 +.. nonce: zkc9nT +.. section: Build + +Fix missing extensions modules when cross compiling. + +.. + +.. bpo: 28248 +.. date: 8653 +.. nonce: KY_-en +.. section: Build + +Update Windows build and OS X installers to use OpenSSL 1.0.2j. + +.. + +.. bpo: 28258 +.. date: 8652 +.. nonce: iKtAHd +.. section: Build + +Fixed build with Estonian locale (python-config and distclean targets in +Makefile). Patch by Arfrever Frehtes Taifersar Arahesis. + +.. + +.. bpo: 26661 +.. date: 8651 +.. nonce: Z_HNbs +.. section: Build + +setup.py now detects system libffi with multiarch wrapper. + +.. + +.. bpo: 28066 +.. date: 8650 +.. nonce: _3xImV +.. section: Build + +Fix the logic that searches build directories for generated include files +when building outside the source tree. + +.. + +.. bpo: 15819 +.. date: 8649 +.. nonce: QVDr3E +.. section: Build + +Remove redundant include search directory option for building outside the +source tree. + +.. + +.. bpo: 27566 +.. date: 8648 +.. nonce: xDWjEb +.. section: Build + +Fix clean target in freeze makefile (patch by Lisa Roach) + +.. + +.. bpo: 27705 +.. date: 8647 +.. nonce: 8C2Ms3 +.. section: Build + +Update message in validate_ucrtbase.py + +.. + +.. bpo: 27983 +.. date: 8646 +.. nonce: jL_1n8 +.. section: Build + +Cause lack of llvm-profdata tool when using clang as required for PGO +linking to be a configure time error rather than make time when --with- +optimizations is enabled. Also improve our ability to find the llvm- +profdata tool on MacOS and some Linuxes. + +.. + +.. bpo: 26307 +.. date: 8645 +.. nonce: Puk2rd +.. section: Build + +The profile-opt build now applies PGO to the built-in modules. + +.. + +.. bpo: 26359 +.. date: 8644 +.. nonce: WXBL-Y +.. section: Build + +Add the --with-optimizations configure flag. + +.. + +.. bpo: 27713 +.. date: 8643 +.. nonce: _3DgXG +.. section: Build + +Suppress spurious build warnings when updating importlib's bootstrap files. +Patch by Xiang Zhang + +.. + +.. bpo: 25825 +.. date: 8642 +.. nonce: JD8aRp +.. section: Build + +Correct the references to Modules/python.exp and ld_so_aix, which are +required on AIX. This updates references to an installation path that was +changed in 3.2a4, and undoes changed references to the build tree that were +made in 3.5.0a1. + +.. + +.. bpo: 27453 +.. date: 8641 +.. nonce: Pb5DBi +.. section: Build + +CPP invocation in configure must use CPPFLAGS. Patch by Chi Hsuan Yen. + +.. + +.. bpo: 27641 +.. date: 8640 +.. nonce: eGzgCk +.. section: Build + +The configure script now inserts comments into the makefile to prevent the +pgen and _freeze_importlib executables from being cross- compiled. + +.. + +.. bpo: 26662 +.. date: 8639 +.. nonce: XkwRxM +.. section: Build + +Set PYTHON_FOR_GEN in configure as the Python program to be used for file +generation during the build. + +.. + +.. bpo: 10910 +.. date: 8638 +.. nonce: ZdRayb +.. section: Build + +Avoid C++ compilation errors on FreeBSD and OS X. Also update FreedBSD +version checks for the original ctype UTF-8 workaround. + +.. + +.. bpo: 28676 +.. date: 8637 +.. nonce: Wxf6Ds +.. section: Build + +Prevent missing 'getentropy' declaration warning on macOS. Patch by Gareth +Rees. diff --git a/Misc/NEWS.d/3.6.0.rst b/Misc/NEWS.d/3.6.0.rst new file mode 100644 index 00000000000000..f9805cab28615e --- /dev/null +++ b/Misc/NEWS.d/3.6.0.rst @@ -0,0 +1,7 @@ +.. bpo: 0 +.. date: 9796 +.. no changes: True +.. nonce: F9ENBV +.. release date: 2016-12-23 + +No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst new file mode 100644 index 00000000000000..e9745fb3033093 --- /dev/null +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -0,0 +1,3941 @@ +.. bpo: 20041 +.. date: 9253 +.. nonce: TypyGp +.. release date: 2016-05-16 +.. section: Core and Builtins + +Fixed TypeError when frame.f_trace is set to None. Patch by Xavier de Gaye. + +.. + +.. bpo: 26168 +.. date: 9252 +.. nonce: -nPBL6 +.. section: Core and Builtins + +Fixed possible refleaks in failing Py_BuildValue() with the "N" format unit. + +.. + +.. bpo: 26991 +.. date: 9251 +.. nonce: yWGNhz +.. section: Core and Builtins + +Fix possible refleak when creating a function with annotations. + +.. + +.. bpo: 27039 +.. date: 9250 +.. nonce: oO-wLV +.. section: Core and Builtins + +Fixed bytearray.remove() for values greater than 127. Based on patch by Joe +Jevnik. + +.. + +.. bpo: 23640 +.. date: 9249 +.. nonce: kvNC4c +.. section: Core and Builtins + +int.from_bytes() no longer bypasses constructors for subclasses. + +.. + +.. bpo: 27005 +.. date: 9248 +.. nonce: ZtcJf- +.. section: Core and Builtins + +Optimized the float.fromhex() class method for exact float. It is now 2 +times faster. + +.. + +.. bpo: 18531 +.. date: 9247 +.. nonce: PkXgtO +.. section: Core and Builtins + +Single var-keyword argument of dict subtype was passed unscathed to the +C-defined function. Now it is converted to exact dict. + +.. + +.. bpo: 26811 +.. date: 9246 +.. nonce: oNzUWt +.. section: Core and Builtins + +gc.get_objects() no longer contains a broken tuple with NULL pointer. + +.. + +.. bpo: 20120 +.. date: 9245 +.. nonce: c-FZZc +.. section: Core and Builtins + +Use RawConfigParser for .pypirc parsing, removing support for interpolation +unintentionally added with move to Python 3. Behavior no longer does any +interpolation in .pypirc files, matching behavior in Python 2.7 and +Setuptools 19.0. + +.. + +.. bpo: 26249 +.. date: 9244 +.. nonce: ZbpWF3 +.. section: Core and Builtins + +Memory functions of the :c:func:`PyMem_Malloc` domain +(:c:data:`PYMEM_DOMAIN_MEM`) now use the :ref:`pymalloc allocator +` rather than system :c:func:`malloc`. Applications calling +:c:func:`PyMem_Malloc` without holding the GIL can now crash: use +``PYTHONMALLOC=debug`` environment variable to validate the usage of memory +allocators in your application. + +.. + +.. bpo: 26802 +.. date: 9243 +.. nonce: hWpU4v +.. section: Core and Builtins + +Optimize function calls only using unpacking like ``func(*tuple)`` (no other +positional argument, no keyword): avoid copying the tuple. Patch written by +Joe Jevnik. + +.. + +.. bpo: 26659 +.. date: 9242 +.. nonce: 5PRa83 +.. section: Core and Builtins + +Make the builtin slice type support cycle collection. + +.. + +.. bpo: 26718 +.. date: 9241 +.. nonce: K5PQ8j +.. section: Core and Builtins + +super.__init__ no longer leaks memory if called multiple times. NOTE: A +direct call of super.__init__ is not endorsed! + +.. + +.. bpo: 27138 +.. date: 9240 +.. nonce: ifYEro +.. section: Core and Builtins + +Fix the doc comment for FileFinder.find_spec(). + +.. + +.. bpo: 27147 +.. date: 9239 +.. nonce: tCCgmH +.. section: Core and Builtins + +Mention PEP 420 in the importlib docs. + +.. + +.. bpo: 25339 +.. date: 9238 +.. nonce: ZcaC2E +.. section: Core and Builtins + +PYTHONIOENCODING now has priority over locale in setting the error handler +for stdin and stdout. + +.. + +.. bpo: 26494 +.. date: 9237 +.. nonce: G6eXIi +.. section: Core and Builtins + +Fixed crash on iterating exhausting iterators. Affected classes are generic +sequence iterators, iterators of str, bytes, bytearray, list, tuple, set, +frozenset, dict, OrderedDict, corresponding views and os.scandir() iterator. + +.. + +.. bpo: 26574 +.. date: 9236 +.. nonce: D2YL_w +.. section: Core and Builtins + +Optimize ``bytes.replace(b'', b'.')`` and ``bytearray.replace(b'', b'.')``. +Patch written by Josh Snider. + +.. + +.. bpo: 26581 +.. date: 9235 +.. nonce: yNA7nm +.. section: Core and Builtins + +If coding cookie is specified multiple times on a line in Python source code +file, only the first one is taken to account. + +.. + +.. bpo: 19711 +.. date: 9234 +.. nonce: gDDPJE +.. section: Core and Builtins + +Add tests for reloading namespace packages. + +.. + +.. bpo: 21099 +.. date: 9233 +.. nonce: CuMWZJ +.. section: Core and Builtins + +Switch applicable importlib tests to use PEP 451 API. + +.. + +.. bpo: 26563 +.. date: 9232 +.. nonce: lyrB2Q +.. section: Core and Builtins + +Debug hooks on Python memory allocators now raise a fatal error if functions +of the :c:func:`PyMem_Malloc` family are called without holding the GIL. + +.. + +.. bpo: 26564 +.. date: 9231 +.. nonce: xeRXaz +.. section: Core and Builtins + +On error, the debug hooks on Python memory allocators now use the +:mod:`tracemalloc` module to get the traceback where a memory block was +allocated. + +.. + +.. bpo: 26558 +.. date: 9230 +.. nonce: s05jz7 +.. section: Core and Builtins + +The debug hooks on Python memory allocator :c:func:`PyObject_Malloc` now +detect when functions are called without holding the GIL. + +.. + +.. bpo: 26516 +.. date: 9229 +.. nonce: OjekqZ +.. section: Core and Builtins + +Add :envvar:`PYTHONMALLOC` environment variable to set the Python memory +allocators and/or install debug hooks. + +.. + +.. bpo: 26516 +.. date: 9228 +.. nonce: chNJuF +.. section: Core and Builtins + +The :c:func:`PyMem_SetupDebugHooks` function can now also be used on Python +compiled in release mode. + +.. + +.. bpo: 26516 +.. date: 9227 +.. nonce: q7fu1f +.. section: Core and Builtins + +The :envvar:`PYTHONMALLOCSTATS` environment variable can now also be used on +Python compiled in release mode. It now has no effect if set to an empty +string. + +.. + +.. bpo: 26516 +.. date: 9226 +.. nonce: 2k9k6R +.. section: Core and Builtins + +In debug mode, debug hooks are now also installed on Python memory +allocators when Python is configured without pymalloc. + +.. + +.. bpo: 26464 +.. date: 9225 +.. nonce: 7BreGz +.. section: Core and Builtins + +Fix str.translate() when string is ASCII and first replacements removes +character, but next replacement uses a non-ASCII character or a string +longer than 1 character. Regression introduced in Python 3.5.0. + +.. + +.. bpo: 22836 +.. date: 9224 +.. nonce: cimt1y +.. section: Core and Builtins + +Ensure exception reports from PyErr_Display() and PyErr_WriteUnraisable() +are sensible even when formatting them produces secondary errors. This +affects the reports produced by sys.__excepthook__() and when __del__() +raises an exception. + +.. + +.. bpo: 26302 +.. date: 9223 +.. nonce: UD9XQt +.. section: Core and Builtins + +Correct behavior to reject comma as a legal character for cookie names. + +.. + +.. bpo: 26136 +.. date: 9222 +.. nonce: eZ0t1K +.. section: Core and Builtins + +Upgrade the warning when a generator raises StopIteration from +PendingDeprecationWarning to DeprecationWarning. Patch by Anish Shah. + +.. + +.. bpo: 26204 +.. date: 9221 +.. nonce: x3Zp8E +.. section: Core and Builtins + +The compiler now ignores all constant statements: bytes, str, int, float, +complex, name constants (None, False, True), Ellipsis and ast.Constant; not +only str and int. For example, ``1.0`` is now ignored in ``def f(): 1.0``. + +.. + +.. bpo: 4806 +.. date: 9220 +.. nonce: i9m3hj +.. section: Core and Builtins + +Avoid masking the original TypeError exception when using star (``*``) +unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel +Urban. + +.. + +.. bpo: 26146 +.. date: 9219 +.. nonce: HKrUth +.. section: Core and Builtins + +Add a new kind of AST node: ``ast.Constant``. It can be used by external AST +optimizers, but the compiler does not emit directly such node. + +.. + +.. bpo: 23601 +.. date: 9218 +.. nonce: 2E4seG +.. section: Core and Builtins + +Sped-up allocation of dict key objects by using Python's small object +allocator. (Contributed by Julian Taylor.) + +.. + +.. bpo: 18018 +.. date: 9217 +.. nonce: XKKap3 +.. section: Core and Builtins + +Import raises ImportError instead of SystemError if a relative import is +attempted without a known parent package. + +.. + +.. bpo: 25843 +.. date: 9216 +.. nonce: NtJZie +.. section: Core and Builtins + +When compiling code, don't merge constants if they are equal but have a +different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` is now +correctly compiled to two different functions: ``f1()`` returns ``1`` +(``int``) and ``f2()`` returns ``1.0`` (``float``), even if ``1`` and +``1.0`` are equal. + +.. + +.. bpo: 26107 +.. date: 9215 +.. nonce: q0LBMY +.. section: Core and Builtins + +The format of the ``co_lnotab`` attribute of code objects changes to support +negative line number delta. + +.. + +.. bpo: 26154 +.. date: 9214 +.. nonce: MtnRAH +.. section: Core and Builtins + +Add a new private _PyThreadState_UncheckedGet() function to get the current +Python thread state, but don't issue a fatal error if it is NULL. This new +function must be used instead of accessing directly the +_PyThreadState_Current variable. The variable is no more exposed since +Python 3.5.1 to hide the exact implementation of atomic C types, to avoid +compiler issues. + +.. + +.. bpo: 25791 +.. date: 9213 +.. nonce: gllzPw +.. section: Core and Builtins + +If __package__ != __spec__.parent or if neither __package__ or __spec__ are +defined then ImportWarning is raised. + +.. + +.. bpo: 22995 +.. date: 9212 +.. nonce: KYNKvs +.. section: Core and Builtins + +[UPDATE] Comment out the one of the pickleability tests in +_PyObject_GetState() due to regressions observed in Cython-based projects. + +.. + +.. bpo: 25961 +.. date: 9211 +.. nonce: Hdjjw0 +.. section: Core and Builtins + +Disallowed null characters in the type name. + +.. + +.. bpo: 25973 +.. date: 9210 +.. nonce: Ud__ZP +.. section: Core and Builtins + +Fix segfault when an invalid nonlocal statement binds a name starting with +two underscores. + +.. + +.. bpo: 22995 +.. date: 9209 +.. nonce: Wq0E86 +.. section: Core and Builtins + +Instances of extension types with a state that aren't subclasses of list or +dict and haven't implemented any pickle-related methods (__reduce__, +__reduce_ex__, __getnewargs__, __getnewargs_ex__, or __getstate__), can no +longer be pickled. Including memoryview. + +.. + +.. bpo: 20440 +.. date: 9208 +.. nonce: GCwOfH +.. section: Core and Builtins + +Massive replacing unsafe attribute setting code with special macro +Py_SETREF. + +.. + +.. bpo: 25766 +.. date: 9207 +.. nonce: jn93Yu +.. section: Core and Builtins + +Special method __bytes__() now works in str subclasses. + +.. + +.. bpo: 25421 +.. date: 9206 +.. nonce: c47YEL +.. section: Core and Builtins + +__sizeof__ methods of builtin types now use dynamic basic size. This allows +sys.getsize() to work correctly with their subclasses with __slots__ +defined. + +.. + +.. bpo: 25709 +.. date: 9205 +.. nonce: WwGm2k +.. section: Core and Builtins + +Fixed problem with in-place string concatenation and utf-8 cache. + +.. + +.. bpo: 5319 +.. date: 9204 +.. nonce: HxlGwI +.. section: Core and Builtins + +New Py_FinalizeEx() API allowing Python to set an exit status of 120 on +failure to flush buffered streams. + +.. + +.. bpo: 25485 +.. date: 9203 +.. nonce: 9qnaPt +.. section: Core and Builtins + +telnetlib.Telnet is now a context manager. + +.. + +.. bpo: 24097 +.. date: 9202 +.. nonce: Vt4E-i +.. section: Core and Builtins + +Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. + +.. + +.. bpo: 24731 +.. date: 9201 +.. nonce: h9-hnz +.. section: Core and Builtins + +Fixed crash on converting objects with special methods __bytes__, __trunc__, +and __float__ returning instances of subclasses of bytes, int, and float to +subclasses of bytes, int, and float correspondingly. + +.. + +.. bpo: 25630 +.. date: 9200 +.. nonce: ZxzcoY +.. section: Core and Builtins + +Fix a possible segfault during argument parsing in functions that accept +filesystem paths. + +.. + +.. bpo: 23564 +.. date: 9199 +.. nonce: XHarGG +.. section: Core and Builtins + +Fixed a partially broken sanity check in the _posixsubprocess internals +regarding how fds_to_pass were passed to the child. The bug had no actual +impact as subprocess.py already avoided it. + +.. + +.. bpo: 25388 +.. date: 9198 +.. nonce: zm3uuQ +.. section: Core and Builtins + +Fixed tokenizer crash when processing undecodable source code with a null +byte. + +.. + +.. bpo: 25462 +.. date: 9197 +.. nonce: eXDzgO +.. section: Core and Builtins + +The hash of the key now is calculated only once in most operations in C +implementation of OrderedDict. + +.. + +.. bpo: 22995 +.. date: 9196 +.. nonce: 90kpuP +.. section: Core and Builtins + +Default implementation of __reduce__ and __reduce_ex__ now rejects builtin +types with not defined __new__. + +.. + +.. bpo: 24802 +.. date: 9195 +.. nonce: Qie066 +.. section: Core and Builtins + +Avoid buffer overreads when int(), float(), compile(), exec() and eval() are +passed bytes-like objects. These objects are not necessarily terminated by +a null byte, but the functions assumed they were. + +.. + +.. bpo: 25555 +.. date: 9194 +.. nonce: MUpG-j +.. section: Core and Builtins + +Fix parser and AST: fill lineno and col_offset of "arg" node when compiling +AST from Python objects. + +.. + +.. bpo: 24726 +.. date: 9193 +.. nonce: AHk4v2 +.. section: Core and Builtins + +Fixed a crash and leaking NULL in repr() of OrderedDict that was mutated by +direct calls of dict methods. + +.. + +.. bpo: 25449 +.. date: 9192 +.. nonce: VqTOFi +.. section: Core and Builtins + +Iterating OrderedDict with keys with unstable hash now raises KeyError in C +implementations as well as in Python implementation. + +.. + +.. bpo: 25395 +.. date: 9191 +.. nonce: htkE3W +.. section: Core and Builtins + +Fixed crash when highly nested OrderedDict structures were garbage +collected. + +.. + +.. bpo: 25401 +.. date: 9190 +.. nonce: ofrAtd +.. section: Core and Builtins + +Optimize bytes.fromhex() and bytearray.fromhex(): they are now between 2x +and 3.5x faster. + +.. + +.. bpo: 25399 +.. date: 9189 +.. nonce: dNKIhY +.. section: Core and Builtins + +Optimize bytearray % args using the new private _PyBytesWriter API. +Formatting is now between 2.5 and 5 times faster. + +.. + +.. bpo: 25274 +.. date: 9188 +.. nonce: QCGvAF +.. section: Core and Builtins + +sys.setrecursionlimit() now raises a RecursionError if the new recursion +limit is too low depending at the current recursion depth. Modify also the +"lower-water mark" formula to make it monotonic. This mark is used to decide +when the overflowed flag of the thread state is reset. + +.. + +.. bpo: 24402 +.. date: 9187 +.. nonce: MAgi3X +.. section: Core and Builtins + +Fix input() to prompt to the redirected stdout when sys.stdout.fileno() +fails. + +.. + +.. bpo: 25349 +.. date: 9186 +.. nonce: 7lBgJ8 +.. section: Core and Builtins + +Optimize bytes % args using the new private _PyBytesWriter API. Formatting +is now up to 2 times faster. + +.. + +.. bpo: 24806 +.. date: 9185 +.. nonce: Nb0znT +.. section: Core and Builtins + +Prevent builtin types that are not allowed to be subclassed from being +subclassed through multiple inheritance. + +.. + +.. bpo: 25301 +.. date: 9184 +.. nonce: hUTCfr +.. section: Core and Builtins + +The UTF-8 decoder is now up to 15 times as fast for error handlers: +``ignore``, ``replace`` and ``surrogateescape``. + +.. + +.. bpo: 24848 +.. date: 9183 +.. nonce: HlUSuy +.. section: Core and Builtins + +Fixed a number of bugs in UTF-7 decoding of misformed data. + +.. + +.. bpo: 25267 +.. date: 9182 +.. nonce: SW8Gs6 +.. section: Core and Builtins + +The UTF-8 encoder is now up to 75 times as fast for error handlers: +``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. Patch co- +written with Serhiy Storchaka. + +.. + +.. bpo: 25280 +.. date: 9181 +.. nonce: ivTMwd +.. section: Core and Builtins + +Import trace messages emitted in verbose (-v) mode are no longer formatted +twice. + +.. + +.. bpo: 25227 +.. date: 9180 +.. nonce: 19v5rp +.. section: Core and Builtins + +Optimize ASCII and latin1 encoders with the ``surrogateescape`` error +handler: the encoders are now up to 3 times as fast. Initial patch written +by Serhiy Storchaka. + +.. + +.. bpo: 25003 +.. date: 9179 +.. nonce: _ban92 +.. section: Core and Builtins + +On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function +instead of the getentropy() function. The getentropy() function is blocking +to generate very good quality entropy, os.urandom() doesn't need such high- +quality entropy. + +.. + +.. bpo: 9232 +.. date: 9178 +.. nonce: pjsmWw +.. section: Core and Builtins + +Modify Python's grammar to allow trailing commas in the argument list of a +function declaration. For example, "def f(\*, a = 3,): pass" is now legal. +Patch from Mark Dickinson. + +.. + +.. bpo: 24965 +.. date: 9177 +.. nonce: wfyxbB +.. section: Core and Builtins + +Implement PEP 498 "Literal String Interpolation". This allows you to embed +expressions inside f-strings, which are converted to normal strings at run +time. Given x=3, then f'value={x}' == 'value=3'. Patch by Eric V. Smith. + +.. + +.. bpo: 26478 +.. date: 9176 +.. nonce: n0dB8e +.. section: Core and Builtins + +Fix semantic bugs when using binary operators with dictionary views and +tuples. + +.. + +.. bpo: 26171 +.. date: 9175 +.. nonce: 8SaQEa +.. section: Core and Builtins + +Fix possible integer overflow and heap corruption in zipimporter.get_data(). + +.. + +.. bpo: 25660 +.. date: 9174 +.. nonce: 93DzBo +.. section: Core and Builtins + +Fix TAB key behaviour in REPL with readline. + +.. + +.. bpo: 26288 +.. date: 9173 +.. nonce: f67RLk +.. section: Core and Builtins + +Optimize PyLong_AsDouble. + +.. + +.. bpo: 26289 +.. date: 9172 +.. nonce: uG9ozG +.. section: Core and Builtins + +Optimize floor and modulo division for single-digit longs. Microbenchmarks +show 2-2.5x improvement. Built-in 'divmod' function is now also ~10% +faster. (See also: bpo-26315) + +.. + +.. bpo: 25887 +.. date: 9171 +.. nonce: PtVIX7 +.. section: Core and Builtins + +Raise a RuntimeError when a coroutine object is awaited more than once. + +.. + +.. bpo: 27057 +.. date: 9170 +.. nonce: YzTA_Q +.. section: Library + +Fix os.set_inheritable() on Android, ioctl() is blocked by SELinux and fails +with EACCESS. The function now falls back to fcntl(). Patch written by +Michał Bednarski. + +.. + +.. bpo: 27014 +.. date: 9169 +.. nonce: ui7Khn +.. section: Library + +Fix infinite recursion using typing.py. Thanks to Kalle Tuure! + +.. + +.. bpo: 27031 +.. date: 9168 +.. nonce: FtvDPs +.. section: Library + +Removed dummy methods in Tkinter widget classes: tk_menuBar() and +tk_bindForTraversal(). + +.. + +.. bpo: 14132 +.. date: 9167 +.. nonce: 5wR9MN +.. section: Library + +Fix urllib.request redirect handling when the target only has a query +string. Original fix by Ján Janech. + +.. + +.. bpo: 17214 +.. date: 9166 +.. nonce: lUbZOV +.. section: Library + +The "urllib.request" module now percent-encodes non-ASCII bytes found in +redirect target URLs. Some servers send Location header fields with non- +ASCII bytes, but "http.client" requires the request target to be ASCII- +encodable, otherwise a UnicodeEncodeError is raised. Based on patch by +Christian Heimes. + +.. + +.. bpo: 27033 +.. date: 9165 +.. nonce: o4XIPr +.. section: Library + +The default value of the decode_data parameter for smtpd.SMTPChannel and +smtpd.SMTPServer constructors is changed to False. + +.. + +.. bpo: 27034 +.. date: 9164 +.. nonce: ptzz_S +.. section: Library + +Removed deprecated class asynchat.fifo. + +.. + +.. bpo: 26870 +.. date: 9163 +.. nonce: 5tCUlp +.. section: Library + +Added readline.set_auto_history(), which can stop entries being +automatically added to the history list. Based on patch by Tyler Crompton. + +.. + +.. bpo: 26039 +.. date: 9162 +.. nonce: JnXjiE +.. section: Library + +zipfile.ZipFile.open() can now be used to write data into a ZIP file, as +well as for extracting data. Patch by Thomas Kluyver. + +.. + +.. bpo: 26892 +.. date: 9161 +.. nonce: XIXb0h +.. section: Library + +Honor debuglevel flag in urllib.request.HTTPHandler. Patch contributed by +Chi Hsuan Yen. + +.. + +.. bpo: 22274 +.. date: 9160 +.. nonce: 0RHDMN +.. section: Library + +In the subprocess module, allow stderr to be redirected to stdout even when +stdout is not redirected. Patch by Akira Li. + +.. + +.. bpo: 26807 +.. date: 9159 +.. nonce: LXSPP6 +.. section: Library + +mock_open 'files' no longer error on readline at end of file. Patch from +Yolanda Robla. + +.. + +.. bpo: 25745 +.. date: 9158 +.. nonce: -n8acU +.. section: Library + +Fixed leaking a userptr in curses panel destructor. + +.. + +.. bpo: 26977 +.. date: 9157 +.. nonce: 5G4HtL +.. section: Library + +Removed unnecessary, and ignored, call to sum of squares helper in +statistics.pvariance. + +.. + +.. bpo: 26002 +.. date: 9156 +.. nonce: bVD4pW +.. section: Library + +Use bisect in statistics.median instead of a linear search. Patch by Upendra +Kuma. + +.. + +.. bpo: 25974 +.. date: 9155 +.. nonce: cpOy5R +.. section: Library + +Make use of new Decimal.as_integer_ratio() method in statistics module. +Patch by Stefan Krah. + +.. + +.. bpo: 26996 +.. date: 9154 +.. nonce: LR__VD +.. section: Library + +Add secrets module as described in PEP 506. + +.. + +.. bpo: 26881 +.. date: 9153 +.. nonce: mdiq_L +.. section: Library + +The modulefinder module now supports extended opcode arguments. + +.. + +.. bpo: 23815 +.. date: 9152 +.. nonce: _krNe8 +.. section: Library + +Fixed crashes related to directly created instances of types in _tkinter and +curses.panel modules. + +.. + +.. bpo: 17765 +.. date: 9151 +.. nonce: hiSVS1 +.. section: Library + +weakref.ref() no longer silently ignores keyword arguments. Patch by Georg +Brandl. + +.. + +.. bpo: 26873 +.. date: 9150 +.. nonce: cYXRcH +.. section: Library + +xmlrpc now raises ResponseError on unsupported type tags instead of silently +return incorrect result. + +.. + +.. bpo: 26915 +.. date: 9149 +.. nonce: GoQKUL +.. section: Library + +The __contains__ methods in the collections ABCs now check for identity +before checking equality. This better matches the behavior of the concrete +classes, allows sensible handling of NaNs, and makes it easier to reason +about container invariants. + +.. + +.. bpo: 26711 +.. date: 9148 +.. nonce: Eu85Qw +.. section: Library + +Fixed the comparison of plistlib.Data with other types. + +.. + +.. bpo: 24114 +.. date: 9147 +.. nonce: RMRMtM +.. section: Library + +Fix an uninitialized variable in `ctypes.util`. + +The bug only occurs on SunOS when the ctypes implementation searches for the +`crle` program. Patch by Xiang Zhang. Tested on SunOS by Kees Bos. + +.. + +.. bpo: 26864 +.. date: 9146 +.. nonce: 1KgGds +.. section: Library + +In urllib.request, change the proxy bypass host checking against no_proxy to +be case-insensitive, and to not match unrelated host names that happen to +have a bypassed hostname as a suffix. Patch by Xiang Zhang. + +.. + +.. bpo: 24902 +.. date: 9145 +.. nonce: bwWpLj +.. section: Library + +Print server URL on http.server startup. Initial patch by Felix Kaiser. + +.. + +.. bpo: 25788 +.. date: 9144 +.. nonce: 9weIV5 +.. section: Library + +fileinput.hook_encoded() now supports an "errors" argument for passing to +open. Original patch by Joseph Hackman. + +.. + +.. bpo: 26634 +.. date: 9143 +.. nonce: FZvsSb +.. section: Library + +recursive_repr() now sets __qualname__ of wrapper. Patch by Xiang Zhang. + +.. + +.. bpo: 26804 +.. date: 9142 +.. nonce: 9Orp-G +.. section: Library + +urllib.request will prefer lower_case proxy environment variables over +UPPER_CASE or Mixed_Case ones. Patch contributed by Hans-Peter Jansen. + +.. + +.. bpo: 26837 +.. date: 9141 +.. nonce: 2FXGsD +.. section: Library + +assertSequenceEqual() now correctly outputs non-stringified differing items +(like bytes in the -b mode). This affects assertListEqual() and +assertTupleEqual(). + +.. + +.. bpo: 26041 +.. date: 9140 +.. nonce: bVem-p +.. section: Library + +Remove "will be removed in Python 3.7" from deprecation messages of +platform.dist() and platform.linux_distribution(). Patch by Kumaripaba +Miyurusara Athukorala. + +.. + +.. bpo: 26822 +.. date: 9139 +.. nonce: rYSL4W +.. section: Library + +itemgetter, attrgetter and methodcaller objects no longer silently ignore +keyword arguments. + +.. + +.. bpo: 26733 +.. date: 9138 +.. nonce: YxaJmL +.. section: Library + +Disassembling a class now disassembles class and static methods. Patch by +Xiang Zhang. + +.. + +.. bpo: 26801 +.. date: 9137 +.. nonce: TQGY-7 +.. section: Library + +Fix error handling in :func:`shutil.get_terminal_size`, catch +:exc:`AttributeError` instead of :exc:`NameError`. Patch written by Emanuel +Barry. + +.. + +.. bpo: 24838 +.. date: 9136 +.. nonce: 3Pfx8T +.. section: Library + +tarfile's ustar and gnu formats now correctly calculate name and link field +limits for multibyte character encodings like utf-8. + +.. + +.. bpo: 26657 +.. date: 9135 +.. nonce: C_-XFg +.. original section: Library +.. section: Security + +Fix directory traversal vulnerability with http.server on Windows. This +fixes a regression that was introduced in 3.3.4rc1 and 3.4.0rc1. Based on +patch by Philipp Hagemeister. + +.. + +.. bpo: 26717 +.. date: 9134 +.. nonce: jngTdu +.. section: Library + +Stop encoding Latin-1-ized WSGI paths with UTF-8. Patch by Anthony Sottile. + +.. + +.. bpo: 26782 +.. date: 9133 +.. nonce: JWLPrH +.. section: Library + +Add STARTUPINFO to subprocess.__all__ on Windows. + +.. + +.. bpo: 26404 +.. date: 9132 +.. nonce: hXw7Bs +.. section: Library + +Add context manager to socketserver. Patch by Aviv Palivoda. + +.. + +.. bpo: 26735 +.. date: 9131 +.. nonce: riSl3b +.. section: Library + +Fix :func:`os.urandom` on Solaris 11.3 and newer when reading more than +1,024 bytes: call ``getrandom()`` multiple times with a limit of 1024 bytes +per call. + +.. + +.. bpo: 26585 +.. date: 9130 +.. nonce: kfb749 +.. section: Library + +Eliminate http.server._quote_html() and use html.escape(quote=False). Patch +by Xiang Zhang. + +.. + +.. bpo: 26685 +.. date: 9129 +.. nonce: sI_1Ff +.. section: Library + +Raise OSError if closing a socket fails. + +.. + +.. bpo: 16329 +.. date: 9128 +.. nonce: nuXD8W +.. section: Library + +Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. + +.. + +.. bpo: 13952 +.. date: 9127 +.. nonce: SOoTVE +.. section: Library + +Add .csv to mimetypes.types_map. Patch by Geoff Wilson. + +.. + +.. bpo: 26587 +.. date: 9126 +.. nonce: Qo-B6C +.. section: Library + +the site module now allows .pth files to specify files to be added to +sys.path (e.g. zip files). + +.. + +.. bpo: 25609 +.. date: 9125 +.. nonce: t1ydQM +.. section: Library + +Introduce contextlib.AbstractContextManager and typing.ContextManager. + +.. + +.. bpo: 26709 +.. date: 9124 +.. nonce: luOPbP +.. section: Library + +Fixed Y2038 problem in loading binary PLists. + +.. + +.. bpo: 23735 +.. date: 9123 +.. nonce: Y5oQ9r +.. section: Library + +Handle terminal resizing with Readline 6.3+ by installing our own SIGWINCH +handler. Patch by Eric Price. + +.. + +.. bpo: 25951 +.. date: 9122 +.. nonce: 1CUASJ +.. section: Library + +Change SSLSocket.sendall() to return None, as explicitly documented for +plain socket objects. Patch by Aviv Palivoda. + +.. + +.. bpo: 26586 +.. date: 9121 +.. nonce: V5pZNa +.. section: Library + +In http.server, respond with "413 Request header fields too large" if there +are too many header fields to parse, rather than killing the connection and +raising an unhandled exception. Patch by Xiang Zhang. + +.. + +.. bpo: 26676 +.. date: 9120 +.. nonce: zLRFed +.. section: Library + +Added missing XMLPullParser to ElementTree.__all__. + +.. + +.. bpo: 22854 +.. date: 9119 +.. nonce: K3rMEH +.. section: Library + +Change BufferedReader.writable() and BufferedWriter.readable() to always +return False. + +.. + +.. bpo: 26492 +.. date: 9118 +.. nonce: YN18iz +.. section: Library + +Exhausted iterator of array.array now conforms with the behavior of +iterators of other mutable sequences: it lefts exhausted even if iterated +array is extended. + +.. + +.. bpo: 26641 +.. date: 9117 +.. nonce: 1ICQz0 +.. section: Library + +doctest.DocFileTest and doctest.testfile() now support packages (module +splitted into multiple directories) for the package parameter. + +.. + +.. bpo: 25195 +.. date: 9116 +.. nonce: EOc4Po +.. section: Library + +Fix a regression in mock.MagicMock. _Call is a subclass of tuple (changeset +3603bae63c13 only works for classes) so we need to implement __ne__ +ourselves. Patch by Andrew Plummer. + +.. + +.. bpo: 26644 +.. date: 9115 +.. nonce: 7tt1tk +.. section: Library + +Raise ValueError rather than SystemError when a negative length is passed to +SSLSocket.recv() or read(). + +.. + +.. bpo: 23804 +.. date: 9114 +.. nonce: PP63Ff +.. section: Library + +Fix SSL recv(0) and read(0) methods to return zero bytes instead of up to +1024. + +.. + +.. bpo: 26616 +.. date: 9113 +.. nonce: v3QwdD +.. section: Library + +Fixed a bug in datetime.astimezone() method. + +.. + +.. bpo: 26637 +.. date: 9112 +.. nonce: ttiUf7 +.. section: Library + +The :mod:`importlib` module now emits an :exc:`ImportError` rather than a +:exc:`TypeError` if :func:`__import__` is tried during the Python shutdown +process but :data:`sys.path` is already cleared (set to ``None``). + +.. + +.. bpo: 21925 +.. date: 9111 +.. nonce: xFz-hR +.. section: Library + +:func:`warnings.formatwarning` now catches exceptions when calling +:func:`linecache.getline` and :func:`tracemalloc.get_object_traceback` to be +able to log :exc:`ResourceWarning` emitted late during the Python shutdown +process. + +.. + +.. bpo: 23848 +.. date: 9110 +.. nonce: RkKqPi +.. section: Library + +On Windows, faulthandler.enable() now also installs an exception handler to +dump the traceback of all Python threads on any Windows exception, not only +on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). + +.. + +.. bpo: 26530 +.. date: 9109 +.. nonce: RWN1jR +.. section: Library + +Add C functions :c:func:`_PyTraceMalloc_Track` and +:c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the +:mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get +the traceback of an object. + +.. + +.. bpo: 26588 +.. date: 9108 +.. nonce: uen0XP +.. section: Library + +The _tracemalloc now supports tracing memory allocations of multiple address +spaces (domains). + +.. + +.. bpo: 24266 +.. date: 9107 +.. nonce: YZgVyM +.. section: Library + +Ctrl+C during Readline history search now cancels the search mode when +compiled with Readline 7. + +.. + +.. bpo: 26590 +.. date: 9106 +.. nonce: qEy91x +.. section: Library + +Implement a safe finalizer for the _socket.socket type. It now releases the +GIL to close the socket. + +.. + +.. bpo: 18787 +.. date: 9105 +.. nonce: rWyzgA +.. section: Library + +spwd.getspnam() now raises a PermissionError if the user doesn't have +privileges. + +.. + +.. bpo: 26560 +.. date: 9104 +.. nonce: A4WXNz +.. section: Library + +Avoid potential ValueError in BaseHandler.start_response. Initial patch by +Peter Inglesby. + +.. + +.. bpo: 26567 +.. date: 9103 +.. nonce: kcC99B +.. section: Library + +Add a new function :c:func:`PyErr_ResourceWarning` function to pass the +destroyed object. Add a *source* attribute to +:class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses +tracemalloc to get the traceback where source object was allocated. + +.. + +.. bpo: 26313 +.. date: 9102 +.. nonce: LjZAjy +.. original section: Library +.. section: Security + +ssl.py _load_windows_store_certs fails if windows cert store is empty. Patch +by Baji. + +.. + +.. bpo: 26569 +.. date: 9101 +.. nonce: EX8vF1 +.. section: Library + +Fix :func:`pyclbr.readmodule` and :func:`pyclbr.readmodule_ex` to support +importing packages. + +.. + +.. bpo: 26499 +.. date: 9100 +.. nonce: NP08PI +.. section: Library + +Account for remaining Content-Length in HTTPResponse.readline() and read1(). +Based on patch by Silent Ghost. Also document that HTTPResponse now supports +these methods. + +.. + +.. bpo: 25320 +.. date: 9099 +.. nonce: V96LIy +.. section: Library + +Handle sockets in directories unittest discovery is scanning. Patch from +Victor van den Elzen. + +.. + +.. bpo: 16181 +.. date: 9098 +.. nonce: P7lLvo +.. section: Library + +cookiejar.http2time() now returns None if year is higher than +datetime.MAXYEAR. + +.. + +.. bpo: 26513 +.. date: 9097 +.. nonce: HoPepy +.. section: Library + +Fixes platform module detection of Windows Server + +.. + +.. bpo: 23718 +.. date: 9096 +.. nonce: AMPC0o +.. section: Library + +Fixed parsing time in week 0 before Jan 1. Original patch by Tamás Bence +Gedai. + +.. + +.. bpo: 26323 +.. date: 9095 +.. nonce: KkZqEj +.. section: Library + +Add Mock.assert_called() and Mock.assert_called_once() methods to +unittest.mock. Patch written by Amit Saha. + +.. + +.. bpo: 20589 +.. date: 9094 +.. nonce: NsQ_I1 +.. section: Library + +Invoking Path.owner() and Path.group() on Windows now raise +NotImplementedError instead of ImportError. + +.. + +.. bpo: 26177 +.. date: 9093 +.. nonce: HlSWer +.. section: Library + +Fixed the keys() method for Canvas and Scrollbar widgets. + +.. + +.. bpo: 15068 +.. date: 9092 +.. nonce: xokEVC +.. section: Library + +Got rid of excessive buffering in fileinput. The bufsize parameter is now +deprecated and ignored. + +.. + +.. bpo: 19475 +.. date: 9091 +.. nonce: MH2HH9 +.. section: Library + +Added an optional argument timespec to the datetime isoformat() method to +choose the precision of the time component. + +.. + +.. bpo: 2202 +.. date: 9090 +.. nonce: dk9sd0 +.. section: Library + +Fix UnboundLocalError in AbstractDigestAuthHandler.get_algorithm_impls. +Initial patch by Mathieu Dupuy. + +.. + +.. bpo: 26167 +.. date: 9089 +.. nonce: 3F-d12 +.. section: Library + +Minimized overhead in copy.copy() and copy.deepcopy(). Optimized copying and +deepcopying bytearrays, NotImplemented, slices, short lists, tuples, dicts, +sets. + +.. + +.. bpo: 25718 +.. date: 9088 +.. nonce: 4EjZyv +.. section: Library + +Fixed pickling and copying the accumulate() iterator with total is None. + +.. + +.. bpo: 26475 +.. date: 9087 +.. nonce: JXVccY +.. section: Library + +Fixed debugging output for regular expressions with the (?x) flag. + +.. + +.. bpo: 26482 +.. date: 9086 +.. nonce: d635gW +.. section: Library + +Allowed pickling recursive dequeues. + +.. + +.. bpo: 26335 +.. date: 9085 +.. nonce: iXw5Yb +.. section: Library + +Make mmap.write() return the number of bytes written like other write +methods. Patch by Jakub Stasiak. + +.. + +.. bpo: 26457 +.. date: 9084 +.. nonce: Xe6Clh +.. section: Library + +Fixed the subnets() methods in IP network classes for the case when +resulting prefix length is equal to maximal prefix length. Based on patch by +Xiang Zhang. + +.. + +.. bpo: 26385 +.. date: 9083 +.. nonce: 50bDXm +.. section: Library + +Remove the file if the internal open() call in NamedTemporaryFile() fails. +Patch by Silent Ghost. + +.. + +.. bpo: 26402 +.. date: 9082 +.. nonce: k7DVuU +.. section: Library + +Fix XML-RPC client to retry when the server shuts down a persistent +connection. This was a regression related to the new +http.client.RemoteDisconnected exception in 3.5.0a4. + +.. + +.. bpo: 25913 +.. date: 9081 +.. nonce: 5flb95 +.. section: Library + +Leading ``<~`` is optional now in base64.a85decode() with adobe=True. Patch +by Swati Jaiswal. + +.. + +.. bpo: 26186 +.. date: 9080 +.. nonce: R9rfiL +.. section: Library + +Remove an invalid type check in importlib.util.LazyLoader. + +.. + +.. bpo: 26367 +.. date: 9079 +.. nonce: Qct-9S +.. section: Library + +importlib.__import__() raises ImportError like builtins.__import__() when +``level`` is specified but without an accompanying package specified. + +.. + +.. bpo: 26309 +.. date: 9078 +.. nonce: ubEeiz +.. section: Library + +In the "socketserver" module, shut down the request (closing the connected +socket) when verify_request() returns false. Patch by Aviv Palivoda. + +.. + +.. bpo: 23430 +.. date: 9077 +.. nonce: s_mLiA +.. section: Library + +Change the socketserver module to only catch exceptions raised from a +request handler that are derived from Exception (instead of BaseException). +Therefore SystemExit and KeyboardInterrupt no longer trigger the +handle_error() method, and will now to stop a single-threaded server. + +.. + +.. bpo: 25939 +.. date: 9076 +.. nonce: X49Fqd +.. original section: Library +.. section: Security + +On Windows open the cert store readonly in ssl.enum_certificates. + +.. + +.. bpo: 25995 +.. date: 9075 +.. nonce: NfcimP +.. section: Library + +os.walk() no longer uses FDs proportional to the tree depth. + +.. + +.. bpo: 25994 +.. date: 9074 +.. nonce: ga9rT- +.. section: Library + +Added the close() method and the support of the context manager protocol for +the os.scandir() iterator. + +.. + +.. bpo: 23992 +.. date: 9073 +.. nonce: O0Hhvc +.. section: Library + +multiprocessing: make MapResult not fail-fast upon exception. + +.. + +.. bpo: 26243 +.. date: 9072 +.. nonce: 41WSpF +.. section: Library + +Support keyword arguments to zlib.compress(). Patch by Aviv Palivoda. + +.. + +.. bpo: 26117 +.. date: 9071 +.. nonce: ne6p11 +.. section: Library + +The os.scandir() iterator now closes file descriptor not only when the +iteration is finished, but when it was failed with error. + +.. + +.. bpo: 25949 +.. date: 9070 +.. nonce: -Lh9vz +.. section: Library + +__dict__ for an OrderedDict instance is now created only when needed. + +.. + +.. bpo: 25911 +.. date: 9069 +.. nonce: d4Zadh +.. section: Library + +Restored support of bytes paths in os.walk() on Windows. + +.. + +.. bpo: 26045 +.. date: 9068 +.. nonce: WmzUrX +.. section: Library + +Add UTF-8 suggestion to error message when posting a non-Latin-1 string with +http.client. + +.. + +.. bpo: 26039 +.. date: 9067 +.. nonce: a5Bxm4 +.. section: Library + +Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). Patch by +Thomas Kluyver. + +.. + +.. bpo: 12923 +.. date: 9066 +.. nonce: HPAu-B +.. section: Library + +Reset FancyURLopener's redirect counter even if there is an exception. +Based on patches by Brian Brazil and Daniel Rocco. + +.. + +.. bpo: 25945 +.. date: 9065 +.. nonce: guNgNM +.. section: Library + +Fixed a crash when unpickle the functools.partial object with wrong state. +Fixed a leak in failed functools.partial constructor. "args" and "keywords" +attributes of functools.partial have now always types tuple and dict +correspondingly. + +.. + +.. bpo: 26202 +.. date: 9064 +.. nonce: LPIXLg +.. section: Library + +copy.deepcopy() now correctly copies range() objects with non-atomic +attributes. + +.. + +.. bpo: 23076 +.. date: 9063 +.. nonce: 8rphoP +.. section: Library + +Path.glob() now raises a ValueError if it's called with an invalid pattern. +Patch by Thomas Nyberg. + +.. + +.. bpo: 19883 +.. date: 9062 +.. nonce: z9TsO6 +.. section: Library + +Fixed possible integer overflows in zipimport. + +.. + +.. bpo: 26227 +.. date: 9061 +.. nonce: Fe6oiB +.. section: Library + +On Windows, getnameinfo(), gethostbyaddr() and gethostbyname_ex() functions +of the socket module now decode the hostname from the ANSI code page rather +than UTF-8. + +.. + +.. bpo: 26099 +.. date: 9060 +.. nonce: CH5jer +.. section: Library + +The site module now writes an error into stderr if sitecustomize module can +be imported but executing the module raise an ImportError. Same change for +usercustomize. + +.. + +.. bpo: 26147 +.. date: 9059 +.. nonce: i-Jc01 +.. section: Library + +xmlrpc now works with strings not encodable with used non-UTF-8 encoding. + +.. + +.. bpo: 25935 +.. date: 9058 +.. nonce: cyni91 +.. section: Library + +Garbage collector now breaks reference loops with OrderedDict. + +.. + +.. bpo: 16620 +.. date: 9057 +.. nonce: rxpn_Y +.. section: Library + +Fixed AttributeError in msilib.Directory.glob(). + +.. + +.. bpo: 26013 +.. date: 9056 +.. nonce: 93RKNz +.. section: Library + +Added compatibility with broken protocol 2 pickles created in old Python 3 +versions (3.4.3 and lower). + +.. + +.. bpo: 26129 +.. date: 9055 +.. nonce: g4RQZd +.. section: Library + +Deprecated accepting non-integers in grp.getgrgid(). + +.. + +.. bpo: 25850 +.. date: 9054 +.. nonce: jwFPxj +.. section: Library + +Use cross-compilation by default for 64-bit Windows. + +.. + +.. bpo: 25822 +.. date: 9053 +.. nonce: 0Eafyi +.. section: Library + +Add docstrings to the fields of urllib.parse results. Patch contributed by +Swati Jaiswal. + +.. + +.. bpo: 22642 +.. date: 9052 +.. nonce: PEgS9F +.. section: Library + +Convert trace module option parsing mechanism to argparse. Patch contributed +by SilentGhost. + +.. + +.. bpo: 24705 +.. date: 9051 +.. nonce: IZYwjR +.. section: Library + +Fix sysconfig._parse_makefile not expanding ${} vars appearing before $() +vars. + +.. + +.. bpo: 26069 +.. date: 9050 +.. nonce: NaF4lN +.. section: Library + +Remove the deprecated apis in the trace module. + +.. + +.. bpo: 22138 +.. date: 9049 +.. nonce: nRNYkc +.. section: Library + +Fix mock.patch behavior when patching descriptors. Restore original values +after patching. Patch contributed by Sean McCully. + +.. + +.. bpo: 25672 +.. date: 9048 +.. nonce: fw9RJP +.. section: Library + +In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is +safe to do so. + +.. + +.. bpo: 26012 +.. date: 9047 +.. nonce: IFSXNm +.. section: Library + +Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). + +.. + +.. bpo: 24120 +.. date: 9046 +.. nonce: Yiwa0h +.. section: Library + +Ignore PermissionError when traversing a tree with pathlib.Path.[r]glob(). +Patch by Ulrich Petri. + +.. + +.. bpo: 21815 +.. date: 9045 +.. nonce: h7-UY8 +.. section: Library + +Accept ] characters in the data portion of imap responses, in order to +handle the flags with square brackets accepted and produced by servers such +as gmail. + +.. + +.. bpo: 25447 +.. date: 9044 +.. nonce: -4m4xO +.. section: Library + +fileinput now uses sys.stdin as-is if it does not have a buffer attribute +(restores backward compatibility). + +.. + +.. bpo: 25971 +.. date: 9043 +.. nonce: vhMeG0 +.. section: Library + +Optimized creating Fractions from floats by 2 times and from Decimals by 3 +times. + +.. + +.. bpo: 25802 +.. date: 9042 +.. nonce: Y2KOnA +.. section: Library + +Document as deprecated the remaining implementations of +importlib.abc.Loader.load_module(). + +.. + +.. bpo: 25928 +.. date: 9041 +.. nonce: JsQfKK +.. section: Library + +Add Decimal.as_integer_ratio(). + +.. + +.. bpo: 25447 +.. date: 9040 +.. nonce: ajPRDy +.. section: Library + +Copying the lru_cache() wrapper object now always works, independently from +the type of the wrapped object (by returning the original object unchanged). + +.. + +.. bpo: 25768 +.. date: 9039 +.. nonce: GDj2ip +.. section: Library + +Have the functions in compileall return booleans instead of ints and add +proper documentation and tests for the return values. + +.. + +.. bpo: 24103 +.. date: 9038 +.. nonce: WufqrQ +.. section: Library + +Fixed possible use after free in ElementTree.XMLPullParser. + +.. + +.. bpo: 25860 +.. date: 9037 +.. nonce: 0hActb +.. section: Library + +os.fwalk() no longer skips remaining directories when error occurs. +Original patch by Samson Lee. + +.. + +.. bpo: 25914 +.. date: 9036 +.. nonce: h0V61F +.. section: Library + +Fixed and simplified OrderedDict.__sizeof__. + +.. + +.. bpo: 25869 +.. date: 9035 +.. nonce: eAnRH5 +.. section: Library + +Optimized deepcopying ElementTree; it is now 20 times faster. + +.. + +.. bpo: 25873 +.. date: 9034 +.. nonce: L4Fgjm +.. section: Library + +Optimized iterating ElementTree. Iterating elements Element.iter() is now +40% faster, iterating text Element.itertext() is now up to 2.5 times faster. + +.. + +.. bpo: 25902 +.. date: 9033 +.. nonce: 6t2FmH +.. section: Library + +Fixed various refcount issues in ElementTree iteration. + +.. + +.. bpo: 22227 +.. date: 9032 +.. nonce: 5utM-Q +.. section: Library + +The TarFile iterator is reimplemented using generator. This implementation +is simpler that using class. + +.. + +.. bpo: 25638 +.. date: 9031 +.. nonce: yitRj4 +.. section: Library + +Optimized ElementTree.iterparse(); it is now 2x faster. Optimized +ElementTree parsing; it is now 10% faster. + +.. + +.. bpo: 25761 +.. date: 9030 +.. nonce: JGgMOP +.. section: Library + +Improved detecting errors in broken pickle data. + +.. + +.. bpo: 25717 +.. date: 9029 +.. nonce: 0_xjaK +.. section: Library + +Restore the previous behaviour of tolerating most fstat() errors when +opening files. This was a regression in 3.5a1, and stopped anonymous +temporary files from working in special cases. + +.. + +.. bpo: 24903 +.. date: 9028 +.. nonce: 3LBdzb +.. section: Library + +Fix regression in number of arguments compileall accepts when '-d' is +specified. The check on the number of arguments has been dropped completely +as it never worked correctly anyway. + +.. + +.. bpo: 25764 +.. date: 9027 +.. nonce: 7WWG07 +.. section: Library + +In the subprocess module, preserve any exception caused by fork() failure +when preexec_fn is used. + +.. + +.. bpo: 25771 +.. date: 9026 +.. nonce: It-7Qf +.. section: Library + +Tweak the exception message for importlib.util.resolve_name() when 'package' +isn't specified but necessary. + +.. + +.. bpo: 6478 +.. date: 9025 +.. nonce: -Bi9Hb +.. section: Library + +_strptime's regexp cache now is reset after changing timezone with +time.tzset(). + +.. + +.. bpo: 14285 +.. date: 9024 +.. nonce: UyG8Hj +.. section: Library + +When executing a package with the "python -m package" option, and package +initialization fails, a proper traceback is now reported. The "runpy" +module now lets exceptions from package initialization pass back to the +caller, rather than raising ImportError. + +.. + +.. bpo: 19771 +.. date: 9023 +.. nonce: 5NG-bg +.. section: Library + +Also in runpy and the "-m" option, omit the irrelevant message ". . . is a +package and cannot be directly executed" if the package could not even be +initialized (e.g. due to a bad ``*.pyc`` file). + +.. + +.. bpo: 25177 +.. date: 9022 +.. nonce: aNR4Ha +.. section: Library + +Fixed problem with the mean of very small and very large numbers. As a side +effect, statistics.mean and statistics.variance should be significantly +faster. + +.. + +.. bpo: 25718 +.. date: 9021 +.. nonce: D9mHZF +.. section: Library + +Fixed copying object with state with boolean value is false. + +.. + +.. bpo: 10131 +.. date: 9020 +.. nonce: a7tptz +.. section: Library + +Fixed deep copying of minidom documents. Based on patch by Marian Ganisin. + +.. + +.. bpo: 7990 +.. date: 9019 +.. nonce: fpvQxH +.. section: Library + +dir() on ElementTree.Element now lists properties: "tag", "text", "tail" and +"attrib". Original patch by Santoso Wijaya. + +.. + +.. bpo: 25725 +.. date: 9018 +.. nonce: XIKv3R +.. section: Library + +Fixed a reference leak in pickle.loads() when unpickling invalid data +including tuple instructions. + +.. + +.. bpo: 25663 +.. date: 9017 +.. nonce: Ofwfqa +.. section: Library + +In the Readline completer, avoid listing duplicate global names, and search +the global namespace before searching builtins. + +.. + +.. bpo: 25688 +.. date: 9016 +.. nonce: 8P1HOv +.. section: Library + +Fixed file leak in ElementTree.iterparse() raising an error. + +.. + +.. bpo: 23914 +.. date: 9015 +.. nonce: 1sEz4J +.. section: Library + +Fixed SystemError raised by unpickler on broken pickle data. + +.. + +.. bpo: 25691 +.. date: 9014 +.. nonce: ZEaapY +.. section: Library + +Fixed crash on deleting ElementTree.Element attributes. + +.. + +.. bpo: 25624 +.. date: 9013 +.. nonce: ed-fM0 +.. section: Library + +ZipFile now always writes a ZIP_STORED header for directory entries. Patch +by Dingyuan Wang. + +.. + +.. bpo: 25626 +.. date: 9012 +.. nonce: TQ3fvb +.. section: Library + +Change three zlib functions to accept sizes that fit in Py_ssize_t, but +internally cap those sizes to UINT_MAX. This resolves a regression in 3.5 +where GzipFile.read() failed to read chunks larger than 2 or 4 GiB. The +change affects the zlib.Decompress.decompress() max_length parameter, the +zlib.decompress() bufsize parameter, and the zlib.Decompress.flush() length +parameter. + +.. + +.. bpo: 25583 +.. date: 9011 +.. nonce: Gk-cim +.. section: Library + +Avoid incorrect errors raised by os.makedirs(exist_ok=True) when the OS +gives priority to errors such as EACCES over EEXIST. + +.. + +.. bpo: 25593 +.. date: 9010 +.. nonce: 56uegI +.. section: Library + +Change semantics of EventLoop.stop() in asyncio. + +.. + +.. bpo: 6973 +.. date: 9009 +.. nonce: nl5cHt +.. section: Library + +When we know a subprocess.Popen process has died, do not allow the +send_signal(), terminate(), or kill() methods to do anything as they could +potentially signal a different process. + +.. + +.. bpo: 23883 +.. date: 9008 +.. nonce: OQS5sS +.. section: Library + +Added missing APIs to __all__ to match the documented APIs for the following +modules: calendar, csv, enum, fileinput, ftplib, logging, optparse, tarfile, +threading and wave. Also added a test.support.check__all__() helper. +Patches by Jacek Kołodziej, Mauro S. M. Rodrigues and Joel Taddei. + +.. + +.. bpo: 25590 +.. date: 9007 +.. nonce: KPcnfv +.. section: Library + +In the Readline completer, only call getattr() once per attribute. Also +complete names of attributes such as properties and slots which are listed +by dir() but not yet created on an instance. + +.. + +.. bpo: 25498 +.. date: 9006 +.. nonce: AvqEBl +.. section: Library + +Fix a crash when garbage-collecting ctypes objects created by wrapping a +memoryview. This was a regression made in 3.5a1. Based on patch by +Eryksun. + +.. + +.. bpo: 25584 +.. date: 9005 +.. nonce: 124mYw +.. section: Library + +Added "escape" to the __all__ list in the glob module. + +.. + +.. bpo: 25584 +.. date: 9004 +.. nonce: ZeWX0J +.. section: Library + +Fixed recursive glob() with patterns starting with ``**``. + +.. + +.. bpo: 25446 +.. date: 9003 +.. nonce: k1DByx +.. section: Library + +Fix regression in smtplib's AUTH LOGIN support. + +.. + +.. bpo: 18010 +.. date: 9002 +.. nonce: Azyf1C +.. section: Library + +Fix the pydoc web server's module search function to handle exceptions from +importing packages. + +.. + +.. bpo: 25554 +.. date: 9001 +.. nonce: UM9MlR +.. section: Library + +Got rid of circular references in regular expression parsing. + +.. + +.. bpo: 18973 +.. date: 9000 +.. nonce: Am9jFL +.. section: Library + +Command-line interface of the calendar module now uses argparse instead of +optparse. + +.. + +.. bpo: 25510 +.. date: 8999 +.. nonce: 79g7LA +.. section: Library + +fileinput.FileInput.readline() now returns b'' instead of '' at the end if +the FileInput was opened with binary mode. Patch by Ryosuke Ito. + +.. + +.. bpo: 25503 +.. date: 8998 +.. nonce: Zea0Y7 +.. section: Library + +Fixed inspect.getdoc() for inherited docstrings of properties. Original +patch by John Mark Vandenberg. + +.. + +.. bpo: 25515 +.. date: 8997 +.. nonce: fQsyYG +.. section: Library + +Always use os.urandom as a source of randomness in uuid.uuid4. + +.. + +.. bpo: 21827 +.. date: 8996 +.. nonce: k2oreR +.. section: Library + +Fixed textwrap.dedent() for the case when largest common whitespace is a +substring of smallest leading whitespace. Based on patch by Robert Li. + +.. + +.. bpo: 25447 +.. date: 8995 +.. nonce: eDYc4t +.. section: Library + +The lru_cache() wrapper objects now can be copied and pickled (by returning +the original object unchanged). + +.. + +.. bpo: 25390 +.. date: 8994 +.. nonce: 6mSgRq +.. section: Library + +typing: Don't crash on Union[str, Pattern]. + +.. + +.. bpo: 25441 +.. date: 8993 +.. nonce: d7zph6 +.. section: Library + +asyncio: Raise error from drain() when socket is closed. + +.. + +.. bpo: 25410 +.. date: 8992 +.. nonce: QAs_3B +.. section: Library + +Cleaned up and fixed minor bugs in C implementation of OrderedDict. + +.. + +.. bpo: 25411 +.. date: 8991 +.. nonce: qsJTCb +.. section: Library + +Improved Unicode support in SMTPHandler through better use of the email +package. Thanks to user simon04 for the patch. + +.. + +.. bpo: 0 +.. date: 8990 +.. nonce: pFHJ0i +.. section: Library + +Move the imp module from a PendingDeprecationWarning to DeprecationWarning. + +.. + +.. bpo: 25407 +.. date: 8989 +.. nonce: ukNt1D +.. section: Library + +Remove mentions of the formatter module being removed in Python 3.6. + +.. + +.. bpo: 25406 +.. date: 8988 +.. nonce: 5MZKU_ +.. section: Library + +Fixed a bug in C implementation of OrderedDict.move_to_end() that caused +segmentation fault or hang in iterating after moving several items to the +start of ordered dict. + +.. + +.. bpo: 25382 +.. date: 8987 +.. nonce: XQ44yE +.. section: Library + +pickletools.dis() now outputs implicit memo index for the MEMOIZE opcode. + +.. + +.. bpo: 25357 +.. date: 8986 +.. nonce: ebqGy- +.. section: Library + +Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() +uses it to avoid a memory copy. + +.. + +.. bpo: 24164 +.. date: 8985 +.. nonce: oi6H3E +.. section: Library + +Objects that need calling ``__new__`` with keyword arguments, can now be +pickled using pickle protocols older than protocol version 4. + +.. + +.. bpo: 25364 +.. date: 8984 +.. nonce: u_1Wi6 +.. section: Library + +zipfile now works in threads disabled builds. + +.. + +.. bpo: 25328 +.. date: 8983 +.. nonce: Rja1Xg +.. section: Library + +smtpd's SMTPChannel now correctly raises a ValueError if both decode_data +and enable_SMTPUTF8 are set to true. + +.. + +.. bpo: 16099 +.. date: 8982 +.. nonce: _MTt3k +.. section: Library + +RobotFileParser now supports Crawl-delay and Request-rate extensions. Patch +by Nikolay Bogoychev. + +.. + +.. bpo: 25316 +.. date: 8981 +.. nonce: dHQHWI +.. section: Library + +distutils raises OSError instead of DistutilsPlatformError when MSVC is not +installed. + +.. + +.. bpo: 25380 +.. date: 8980 +.. nonce: sKZ6-I +.. section: Library + +Fixed protocol for the STACK_GLOBAL opcode in pickletools.opcodes. + +.. + +.. bpo: 23972 +.. date: 8979 +.. nonce: s2g30g +.. section: Library + +Updates asyncio datagram create method allowing reuseport and reuseaddr +socket options to be set prior to binding the socket. Mirroring the existing +asyncio create_server method the reuseaddr option for datagram sockets +defaults to True if the O/S is 'posix' (except if the platform is Cygwin). +Patch by Chris Laws. + +.. + +.. bpo: 25304 +.. date: 8978 +.. nonce: CsmLyI +.. section: Library + +Add asyncio.run_coroutine_threadsafe(). This lets you submit a coroutine to +a loop from another thread, returning a concurrent.futures.Future. By +Vincent Michel. + +.. + +.. bpo: 25232 +.. date: 8977 +.. nonce: KhKjCE +.. section: Library + +Fix CGIRequestHandler to split the query from the URL at the first question +mark (?) rather than the last. Patch from Xiang Zhang. + +.. + +.. bpo: 24657 +.. date: 8976 +.. nonce: h2Ag7y +.. section: Library + +Prevent CGIRequestHandler from collapsing slashes in the query part of the +URL as if it were a path. Patch from Xiang Zhang. + +.. + +.. bpo: 25287 +.. date: 8975 +.. nonce: KhzzMW +.. section: Library + +Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check +if it is supported, it may not be supported on OpenBSD for example. + +.. + +.. bpo: 23600 +.. date: 8974 +.. nonce: 7J_RD5 +.. section: Library + +Default implementation of tzinfo.fromutc() was returning wrong results in +some cases. + +.. + +.. bpo: 25203 +.. date: 8973 +.. nonce: IgDEbt +.. section: Library + +Failed readline.set_completer_delims() no longer left the module in +inconsistent state. + +.. + +.. bpo: 25011 +.. date: 8972 +.. nonce: VcaCd6 +.. section: Library + +rlcompleter now omits private and special attribute names unless the prefix +starts with underscores. + +.. + +.. bpo: 25209 +.. date: 8971 +.. nonce: WxKcdJ +.. section: Library + +rlcompleter now can add a space or a colon after completed keyword. + +.. + +.. bpo: 22241 +.. date: 8970 +.. nonce: a-Mtw2 +.. section: Library + +timezone.utc name is now plain 'UTC', not 'UTC-00:00'. + +.. + +.. bpo: 23517 +.. date: 8969 +.. nonce: 0ABp8q +.. section: Library + +fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now +round microseconds to nearest with ties going to nearest even integer +(ROUND_HALF_EVEN), as round(float), instead of rounding towards -Infinity +(ROUND_FLOOR). + +.. + +.. bpo: 23552 +.. date: 8968 +.. nonce: I0T-M- +.. section: Library + +Timeit now warns when there is substantial (4x) variance between best and +worst times. Patch from Serhiy Storchaka. + +.. + +.. bpo: 24633 +.. date: 8967 +.. nonce: 6Unn9B +.. section: Library + +site-packages/README -> README.txt. + +.. + +.. bpo: 24879 +.. date: 8966 +.. nonce: YUzg_z +.. section: Library + +help() and pydoc can now list named tuple fields in the order they were +defined rather than alphabetically. The ordering is determined by the +_fields attribute if present. + +.. + +.. bpo: 24874 +.. date: 8965 +.. nonce: luBfgA +.. section: Library + +Improve speed of itertools.cycle() and make its pickle more compact. + +.. + +.. bpo: 0 +.. date: 8964 +.. nonce: mD-_3v +.. section: Library + +Fix crash in itertools.cycle.__setstate__() when the first argument wasn't a +list. + +.. + +.. bpo: 20059 +.. date: 8963 +.. nonce: SHv0Ji +.. section: Library + +urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. + +.. + +.. bpo: 24360 +.. date: 8962 +.. nonce: 5RwH-e +.. section: Library + +Improve __repr__ of argparse.Namespace() for invalid identifiers. Patch by +Matthias Bussonnier. + +.. + +.. bpo: 23426 +.. date: 8961 +.. nonce: PUV-Cx +.. section: Library + +run_setup was broken in distutils. Patch from Alexander Belopolsky. + +.. + +.. bpo: 13938 +.. date: 8960 +.. nonce: e5NSE1 +.. section: Library + +2to3 converts StringTypes to a tuple. Patch from Mark Hammond. + +.. + +.. bpo: 2091 +.. date: 8959 +.. nonce: bp56pO +.. section: Library + +open() accepted a 'U' mode string containing '+', but 'U' can only be used +with 'r'. Patch from Jeff Balogh and John O'Connor. + +.. + +.. bpo: 8585 +.. date: 8958 +.. nonce: 78hPc2 +.. section: Library + +improved tests for zipimporter2. Patch from Mark Lawrence. + +.. + +.. bpo: 18622 +.. date: 8957 +.. nonce: i6nCCW +.. section: Library + +unittest.mock.mock_open().reset_mock would recurse infinitely. Patch from +Nicola Palumbo and Laurent De Buyst. + +.. + +.. bpo: 24426 +.. date: 8956 +.. nonce: yCtQfT +.. section: Library + +Fast searching optimization in regular expressions now works for patterns +that starts with capturing groups. Fast searching optimization now can't be +disabled at compile time. + +.. + +.. bpo: 23661 +.. date: 8955 +.. nonce: 5VHJmh +.. section: Library + +unittest.mock side_effects can now be exceptions again. This was a +regression vs Python 3.4. Patch from Ignacio Rossi + +.. + +.. bpo: 13248 +.. date: 8954 +.. nonce: SA2hvu +.. section: Library + +Remove deprecated inspect.getmoduleinfo function. + +.. + +.. bpo: 25578 +.. date: 8953 +.. nonce: G6S-ft +.. section: Library + +Fix (another) memory leak in SSLSocket.getpeercer(). + +.. + +.. bpo: 25530 +.. date: 8952 +.. nonce: hDFkwu +.. section: Library + +Disable the vulnerable SSLv3 protocol by default when creating +ssl.SSLContext. + +.. + +.. bpo: 25569 +.. date: 8951 +.. nonce: CfvQjK +.. section: Library + +Fix memory leak in SSLSocket.getpeercert(). + +.. + +.. bpo: 25471 +.. date: 8950 +.. nonce: T0A02M +.. section: Library + +Sockets returned from accept() shouldn't appear to be nonblocking. + +.. + +.. bpo: 25319 +.. date: 8949 +.. nonce: iyuglv +.. section: Library + +When threading.Event is reinitialized, the underlying condition should use a +regular lock rather than a recursive lock. + +.. + +.. bpo: 0 +.. date: 8948 +.. nonce: rtZyid +.. section: Library + +Skip getaddrinfo if host is already resolved. Patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26050 +.. date: 8947 +.. nonce: sclyvk +.. section: Library + +Add asyncio.StreamReader.readuntil() method. Patch by Марк Коренберг. + +.. + +.. bpo: 25924 +.. date: 8946 +.. nonce: Uxr2vt +.. section: Library + +Avoid unnecessary serialization of getaddrinfo(3) calls on OS X versions +10.5 or higher. Original patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26406 +.. date: 8945 +.. nonce: ihvhF4 +.. section: Library + +Avoid unnecessary serialization of getaddrinfo(3) calls on current versions +of OpenBSD and NetBSD. Patch by A. Jesse Jiryu Davis. + +.. + +.. bpo: 26848 +.. date: 8944 +.. nonce: ChBOpQ +.. section: Library + +Fix asyncio/subprocess.communicate() to handle empty input. Patch by Jack +O'Connor. + +.. + +.. bpo: 27040 +.. date: 8943 +.. nonce: UASyCC +.. section: Library + +Add loop.get_exception_handler method + +.. + +.. bpo: 27041 +.. date: 8942 +.. nonce: p3893U +.. section: Library + +asyncio: Add loop.create_future method + +.. + +.. bpo: 20640 +.. date: 8941 +.. nonce: PmI-G8 +.. section: IDLE + +Add tests for idlelib.configHelpSourceEdit. Patch by Saimadhav Heblikar. + +.. + +.. bpo: 0 +.. date: 8940 +.. nonce: _YJfG7 +.. section: IDLE + +In the 'IDLE-console differences' section of the IDLE doc, clarify how +running with IDLE affects sys.modules and the standard streams. + +.. + +.. bpo: 25507 +.. date: 8939 +.. nonce: i8bNpk +.. section: IDLE + +fix incorrect change in IOBinding that prevented printing. Augment IOBinding +htest to include all major IOBinding functions. + +.. + +.. bpo: 25905 +.. date: 8938 +.. nonce: FzNb3B +.. section: IDLE + +Revert unwanted conversion of ' to ’ RIGHT SINGLE QUOTATION MARK in +README.txt and open this and NEWS.txt with 'ascii'. Re-encode CREDITS.txt to +utf-8 and open it with 'utf-8'. + +.. + +.. bpo: 15348 +.. date: 8937 +.. nonce: d1Fg01 +.. section: IDLE + +Stop the debugger engine (normally in a user process) before closing the +debugger window (running in the IDLE process). This prevents the +RuntimeErrors that were being caught and ignored. + +.. + +.. bpo: 24455 +.. date: 8936 +.. nonce: x6YqtE +.. section: IDLE + +Prevent IDLE from hanging when a) closing the shell while the debugger is +active (15347); b) closing the debugger with the [X] button (15348); and c) +activating the debugger when already active (24455). The patch by Mark +Roseman does this by making two changes. 1. Suspend and resume the +gui.interaction method with the tcl vwait mechanism intended for this +purpose (instead of root.mainloop & .quit). 2. In gui.run, allow any +existing interaction to terminate first. + +.. + +.. bpo: 0 +.. date: 8935 +.. nonce: Yp9LRY +.. section: IDLE + +Change 'The program' to 'Your program' in an IDLE 'kill program?' message to +make it clearer that the program referred to is the currently running user +program, not IDLE itself. + +.. + +.. bpo: 24750 +.. date: 8934 +.. nonce: xgsi-K +.. section: IDLE + +Improve the appearance of the IDLE editor window status bar. Patch by Mark +Roseman. + +.. + +.. bpo: 25313 +.. date: 8933 +.. nonce: xMXHpO +.. section: IDLE + +Change the handling of new built-in text color themes to better address the +compatibility problem introduced by the addition of IDLE Dark. Consistently +use the revised idleConf.CurrentTheme everywhere in idlelib. + +.. + +.. bpo: 24782 +.. date: 8932 +.. nonce: RgIPYE +.. section: IDLE + +Extension configuration is now a tab in the IDLE Preferences dialog rather +than a separate dialog. The former tabs are now a sorted list. Patch by +Mark Roseman. + +.. + +.. bpo: 22726 +.. date: 8931 +.. nonce: x8T0dA +.. section: IDLE + +Re-activate the config dialog help button with some content about the other +buttons and the new IDLE Dark theme. + +.. + +.. bpo: 24820 +.. date: 8930 +.. nonce: TFPJhr +.. section: IDLE + +IDLE now has an 'IDLE Dark' built-in text color theme. It is more or less +IDLE Classic inverted, with a cobalt blue background. Strings, comments, +keywords, ... are still green, red, orange, ... . To use it with IDLEs +released before November 2015, hit the 'Save as New Custom Theme' button and +enter a new name, such as 'Custom Dark'. The custom theme will work with +any IDLE release, and can be modified. + +.. + +.. bpo: 25224 +.. date: 8929 +.. nonce: 5Llwo4 +.. section: IDLE + +README.txt is now an idlelib index for IDLE developers and curious users. +The previous user content is now in the IDLE doc chapter. 'IDLE' now means +'Integrated Development and Learning Environment'. + +.. + +.. bpo: 24820 +.. date: 8928 +.. nonce: ZUz9Fn +.. section: IDLE + +Users can now set breakpoint colors in Settings -> Custom Highlighting. +Original patch by Mark Roseman. + +.. + +.. bpo: 24972 +.. date: 8927 +.. nonce: uc0uNo +.. section: IDLE + +Inactive selection background now matches active selection background, as +configured by users, on all systems. Found items are now always highlighted +on Windows. Initial patch by Mark Roseman. + +.. + +.. bpo: 24570 +.. date: 8926 +.. nonce: s3EkNn +.. section: IDLE + +Idle: make calltip and completion boxes appear on Macs affected by a tk +regression. Initial patch by Mark Roseman. + +.. + +.. bpo: 24988 +.. date: 8925 +.. nonce: tXqq4T +.. section: IDLE + +Idle ScrolledList context menus (used in debugger) now work on Mac Aqua. +Patch by Mark Roseman. + +.. + +.. bpo: 24801 +.. date: 8924 +.. nonce: -bj_Ou +.. section: IDLE + +Make right-click for context menu work on Mac Aqua. Patch by Mark Roseman. + +.. + +.. bpo: 25173 +.. date: 8923 +.. nonce: EZzrPg +.. section: IDLE + +Associate tkinter messageboxes with a specific widget. For Mac OSX, make +them a 'sheet'. Patch by Mark Roseman. + +.. + +.. bpo: 25198 +.. date: 8922 +.. nonce: -j_BV7 +.. section: IDLE + +Enhance the initial html viewer now used for Idle Help. * Properly indent +fixed-pitch text (patch by Mark Roseman). * Give code snippet a very Sphinx- +like light blueish-gray background. * Re-use initial width and height set by +users for shell and editor. * When the Table of Contents (TOC) menu is used, +put the section header at the top of the screen. + +.. + +.. bpo: 25225 +.. date: 8921 +.. nonce: 9pvdq6 +.. section: IDLE + +Condense and rewrite Idle doc section on text colors. + +.. + +.. bpo: 21995 +.. date: 8920 +.. nonce: C5Rmzx +.. section: IDLE + +Explain some differences between IDLE and console Python. + +.. + +.. bpo: 22820 +.. date: 8919 +.. nonce: hix_8X +.. section: IDLE + +Explain need for *print* when running file from Idle editor. + +.. + +.. bpo: 25224 +.. date: 8918 +.. nonce: UVMYQq +.. section: IDLE + +Doc: augment Idle feature list and no-subprocess section. + +.. + +.. bpo: 25219 +.. date: 8917 +.. nonce: 8_9DYg +.. section: IDLE + +Update doc for Idle command line options. Some were missing and notes were +not correct. + +.. + +.. bpo: 24861 +.. date: 8916 +.. nonce: Ecg2yT +.. section: IDLE + +Most of idlelib is private and subject to change. Use idleib.idle.* to start +Idle. See idlelib.__init__.__doc__. + +.. + +.. bpo: 25199 +.. date: 8915 +.. nonce: ih7yY3 +.. section: IDLE + +Idle: add synchronization comments for future maintainers. + +.. + +.. bpo: 16893 +.. date: 8914 +.. nonce: uIi1oB +.. section: IDLE + +Replace help.txt with help.html for Idle doc display. The new +idlelib/help.html is rstripped Doc/build/html/library/idle.html. It looks +better than help.txt and will better document Idle as released. The tkinter +html viewer that works for this file was written by Rose Roseman. The now +unused EditorWindow.HelpDialog class and helt.txt file are deprecated. + +.. + +.. bpo: 24199 +.. date: 8913 +.. nonce: VKnZEv +.. section: IDLE + +Deprecate unused idlelib.idlever with possible removal in 3.6. + +.. + +.. bpo: 24790 +.. date: 8912 +.. nonce: hD1hlj +.. section: IDLE + +Remove extraneous code (which also create 2 & 3 conflicts). + +.. + +.. bpo: 26736 +.. date: 8911 +.. nonce: U_Hyqo +.. section: Documentation + +Used HTTPS for external links in the documentation if possible. + +.. + +.. bpo: 6953 +.. date: 8910 +.. nonce: Zk6rno +.. section: Documentation + +Rework the Readline module documentation to group related functions +together, and add more details such as what underlying Readline functions +and variables are accessed. + +.. + +.. bpo: 23606 +.. date: 8909 +.. nonce: 9MhIso +.. section: Documentation + +Adds note to ctypes documentation regarding cdll.msvcrt. + +.. + +.. bpo: 24952 +.. date: 8908 +.. nonce: RHhFPE +.. section: Documentation + +Clarify the default size argument of stack_size() in the "threading" and +"_thread" modules. Patch from Mattip. + +.. + +.. bpo: 26014 +.. date: 8907 +.. nonce: ptdZ_I +.. section: Documentation + +Update 3.x packaging documentation: * "See also" links to the new docs are +now provided in the legacy pages * links to setuptools documentation have +been updated + +.. + +.. bpo: 21916 +.. date: 8906 +.. nonce: muwCyp +.. section: Tests + +Added tests for the turtle module. Patch by ingrid, Gregory Loyse and Jelle +Zijlstra. + +.. + +.. bpo: 26295 +.. date: 8905 +.. nonce: sYBtj5 +.. section: Tests + +When using "python3 -m test --testdir=TESTDIR", regrtest doesn't add "test." +prefix to test module names. + +.. + +.. bpo: 26523 +.. date: 8904 +.. nonce: em_Uzt +.. section: Tests + +The multiprocessing thread pool (multiprocessing.dummy.Pool) was untested. + +.. + +.. bpo: 26015 +.. date: 8903 +.. nonce: p3oWK3 +.. section: Tests + +Added new tests for pickling iterators of mutable sequences. + +.. + +.. bpo: 26325 +.. date: 8902 +.. nonce: KOUc82 +.. section: Tests + +Added test.support.check_no_resource_warning() to check that no +ResourceWarning is emitted. + +.. + +.. bpo: 25940 +.. date: 8901 +.. nonce: MvBwSe +.. section: Tests + +Changed test_ssl to use its internal local server more. This avoids relying +on svn.python.org, which recently changed root certificate. + +.. + +.. bpo: 25616 +.. date: 8900 +.. nonce: Qr-60p +.. section: Tests + +Tests for OrderedDict are extracted from test_collections into separate file +test_ordered_dict. + +.. + +.. bpo: 25449 +.. date: 8899 +.. nonce: MP6KNs +.. section: Tests + +Added tests for OrderedDict subclasses. + +.. + +.. bpo: 25188 +.. date: 8898 +.. nonce: lnLnIW +.. section: Tests + +Add -P/--pgo to test.regrtest to suppress error output when running the test +suite for the purposes of a PGO build. Initial patch by Alecsandru Patrascu. + +.. + +.. bpo: 22806 +.. date: 8897 +.. nonce: _QHyyV +.. section: Tests + +Add ``python -m test --list-tests`` command to list tests. + +.. + +.. bpo: 18174 +.. date: 8896 +.. nonce: TzH9d_ +.. section: Tests + +``python -m test --huntrleaks ...`` now also checks for leak of file +descriptors. Patch written by Richard Oudkerk. + +.. + +.. bpo: 25260 +.. date: 8895 +.. nonce: jw3p83 +.. section: Tests + +Fix ``python -m test --coverage`` on Windows. Remove the list of ignored +directories. + +.. + +.. bpo: 0 +.. date: 8894 +.. nonce: X-Bk5l +.. section: Tests + +``PCbuild\rt.bat`` now accepts an unlimited number of arguments to pass +along to regrtest.py. Previously there was a limit of 9. + +.. + +.. bpo: 26583 +.. date: 8893 +.. nonce: Up7hTl +.. section: Tests + +Skip test_timestamp_overflow in test_import if bytecode files cannot be +written. + +.. + +.. bpo: 21277 +.. date: 8892 +.. nonce: 7y1j9a +.. section: Build + +Don't try to link _ctypes with a ffi_convenience library. + +.. + +.. bpo: 26884 +.. date: 8891 +.. nonce: O8-azL +.. section: Build + +Fix linking extension modules for cross builds. Patch by Xavier de Gaye. + +.. + +.. bpo: 26932 +.. date: 8890 +.. nonce: 5kzaG9 +.. section: Build + +Fixed support of RTLD_* constants defined as enum values, not via macros (in +particular on Android). Patch by Chi Hsuan Yen. + +.. + +.. bpo: 22359 +.. date: 8889 +.. nonce: HDjM4s +.. section: Build + +Disable the rules for running _freeze_importlib and pgen when cross- +compiling. The output of these programs is normally saved with the source +code anyway, and is still regenerated when doing a native build. Patch by +Xavier de Gaye. + +.. + +.. bpo: 21668 +.. date: 8888 +.. nonce: qWwBui +.. section: Build + +Link audioop, _datetime, _ctypes_test modules to libm, except on Mac OS X. +Patch written by Chi Hsuan Yen. + +.. + +.. bpo: 25702 +.. date: 8887 +.. nonce: ipxyJs +.. section: Build + +A --with-lto configure option has been added that will enable link time +optimizations at build time during a make profile-opt. Some compilers and +toolchains are known to not produce stable code when using LTO, be sure to +test things thoroughly before relying on it. It can provide a few % speed up +over profile-opt alone. + +.. + +.. bpo: 26624 +.. date: 8886 +.. nonce: 4fGrTl +.. section: Build + +Adds validation of ucrtbase[d].dll version with warning for old versions. + +.. + +.. bpo: 17603 +.. date: 8885 +.. nonce: 102DA- +.. section: Build + +Avoid error about nonexistant fileblocks.o file by using a lower-level check +for st_blocks in struct stat. + +.. + +.. bpo: 26079 +.. date: 8884 +.. nonce: mEzW0O +.. section: Build + +Fixing the build output folder for tix-8.4.3.6. Patch by Bjoern Thiel. + +.. + +.. bpo: 26465 +.. date: 8883 +.. nonce: _YR608 +.. section: Build + +Update Windows builds to use OpenSSL 1.0.2g. + +.. + +.. bpo: 25348 +.. date: 8882 +.. nonce: FLSPfp +.. section: Build + +Added ``--pgo`` and ``--pgo-job`` arguments to ``PCbuild\build.bat`` for +building with Profile-Guided Optimization. The old +``PCbuild\build_pgo.bat`` script is removed. + +.. + +.. bpo: 25827 +.. date: 8881 +.. nonce: yg3DMM +.. section: Build + +Add support for building with ICC to ``configure``, including a new +``--with-icc`` flag. + +.. + +.. bpo: 25696 +.. date: 8880 +.. nonce: 2R_wIv +.. section: Build + +Fix installation of Python on UNIX with make -j9. + +.. + +.. bpo: 24986 +.. date: 8879 +.. nonce: 1WyXeU +.. section: Build + +It is now possible to build Python on Windows without errors when external +libraries are not available. + +.. + +.. bpo: 24421 +.. date: 8878 +.. nonce: 2zY7vM +.. section: Build + +Compile Modules/_math.c once, before building extensions. Previously it +could fail to compile properly if the math and cmath builds were concurrent. + +.. + +.. bpo: 26465 +.. date: 8877 +.. nonce: PkIaV8 +.. section: Build + +Update OS X 10.5+ 32-bit-only installer to build and link with OpenSSL +1.0.2g. + +.. + +.. bpo: 26268 +.. date: 8876 +.. nonce: I3-YLh +.. section: Build + +Update Windows builds to use OpenSSL 1.0.2f. + +.. + +.. bpo: 25136 +.. date: 8875 +.. nonce: Vi-fmO +.. section: Build + +Support Apple Xcode 7's new textual SDK stub libraries. + +.. + +.. bpo: 24324 +.. date: 8874 +.. nonce: m6DZMx +.. section: Build + +Do not enable unreachable code warnings when using gcc as the option does +not work correctly in older versions of gcc and has been silently removed as +of gcc-4.5. + +.. + +.. bpo: 27053 +.. date: 8873 +.. nonce: 1IRbae +.. section: Windows + +Updates make_zip.py to correctly generate library ZIP file. + +.. + +.. bpo: 26268 +.. date: 8872 +.. nonce: Z-lJEh +.. section: Windows + +Update the prepare_ssl.py script to handle OpenSSL releases that don't +include the contents of the include directory (that is, 1.0.2e and later). + +.. + +.. bpo: 26071 +.. date: 8871 +.. nonce: wLxL2l +.. section: Windows + +bdist_wininst created binaries fail to start and find 32bit Python + +.. + +.. bpo: 26073 +.. date: 8870 +.. nonce: XwWgHp +.. section: Windows + +Update the list of magic numbers in launcher + +.. + +.. bpo: 26065 +.. date: 8869 +.. nonce: SkVLJp +.. section: Windows + +Excludes venv from library when generating embeddable distro. + +.. + +.. bpo: 25022 +.. date: 8868 +.. nonce: vAt_zr +.. section: Windows + +Removed very outdated PC/example_nt/ directory. + +.. + +.. bpo: 26799 +.. date: 8867 +.. nonce: gK2VXX +.. section: Tools/Demos + +Fix python-gdb.py: don't get C types once when the Python code is loaded, +but get C types on demand. The C types can change if python-gdb.py is loaded +before the Python executable. Patch written by Thomas Ilsche. + +.. + +.. bpo: 26271 +.. date: 8866 +.. nonce: wg-rzr +.. section: Tools/Demos + +Fix the Freeze tool to properly use flags passed through configure. Patch by +Daniel Shaulov. + +.. + +.. bpo: 26489 +.. date: 8865 +.. nonce: rJ_U5S +.. section: Tools/Demos + +Add dictionary unpacking support to Tools/parser/unparse.py. Patch by Guo Ci +Teo. + +.. + +.. bpo: 26316 +.. date: 8864 +.. nonce: QJvVOi +.. section: Tools/Demos + +Fix variable name typo in Argument Clinic. + +.. + +.. bpo: 25440 +.. date: 8863 +.. nonce: 5xhyGr +.. section: Tools/Demos + +Fix output of python-config --extension-suffix. + +.. + +.. bpo: 25154 +.. date: 8862 +.. nonce: yLO-r4 +.. section: Tools/Demos + +The pyvenv script has been deprecated in favour of `python3 -m venv`. + +.. + +.. bpo: 26312 +.. date: 8861 +.. nonce: h1T61B +.. section: C API + +SystemError is now raised in all programming bugs with using +PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some +programming bugs. + +.. + +.. bpo: 26198 +.. date: 8860 +.. nonce: lVn1HX +.. section: C API + +ValueError is now raised instead of TypeError on buffer overflow in parsing +"es#" and "et#" format units. SystemError is now raised instead of +TypeError on programmical error in parsing format string. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst new file mode 100644 index 00000000000000..46387a5c0b1538 --- /dev/null +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -0,0 +1,792 @@ +.. bpo: 27095 +.. date: 9332 +.. nonce: 92UoyH +.. release date: 2016-06-13 +.. section: Core and Builtins + +Simplified MAKE_FUNCTION and removed MAKE_CLOSURE opcodes. Patch by Demur +Rumed. + +.. + +.. bpo: 27190 +.. date: 9331 +.. nonce: DHDFeD +.. section: Core and Builtins + +Raise NotSupportedError if sqlite3 is older than 3.3.1. Patch by Dave +Sawyer. + +.. + +.. bpo: 27286 +.. date: 9330 +.. nonce: U8q6B1 +.. section: Core and Builtins + +Fixed compiling BUILD_MAP_UNPACK_WITH_CALL opcode. Calling function with +generalized unpacking (PEP 448) and conflicting keyword names could cause +undefined behavior. + +.. + +.. bpo: 27140 +.. date: 9329 +.. nonce: uc39-1 +.. section: Core and Builtins + +Added BUILD_CONST_KEY_MAP opcode. + +.. + +.. bpo: 27186 +.. date: 9328 +.. nonce: EAnCS7 +.. section: Core and Builtins + +Add support for os.PathLike objects to open() (part of PEP 519). + +.. + +.. bpo: 27066 +.. date: 9327 +.. nonce: SNExZi +.. section: Core and Builtins + +Fixed SystemError if a custom opener (for open()) returns a negative number +without setting an exception. + +.. + +.. bpo: 26983 +.. date: 9326 +.. nonce: A0f3fK +.. section: Core and Builtins + +float() now always return an instance of exact float. The deprecation +warning is emitted if __float__ returns an instance of a strict subclass of +float. In a future versions of Python this can be an error. + +.. + +.. bpo: 27097 +.. date: 9325 +.. nonce: woRKey +.. section: Core and Builtins + +Python interpreter is now about 7% faster due to optimized instruction +decoding. Based on patch by Demur Rumed. + +.. + +.. bpo: 26647 +.. date: 9324 +.. nonce: DLSzRi +.. section: Core and Builtins + +Python interpreter now uses 16-bit wordcode instead of bytecode. Patch by +Demur Rumed. + +.. + +.. bpo: 23275 +.. date: 9323 +.. nonce: YGPb_y +.. section: Core and Builtins + +Allow assigning to an empty target list in round brackets: () = iterable. + +.. + +.. bpo: 27243 +.. date: 9322 +.. nonce: U36M4E +.. section: Core and Builtins + +Update the __aiter__ protocol: instead of returning an awaitable that +resolves to an asynchronous iterator, the asynchronous iterator should be +returned directly. Doing the former will trigger a +PendingDeprecationWarning. + +.. + +.. bpo: 0 +.. date: 9321 +.. nonce: nBpVM1 +.. section: Library + +Comment out socket (SO_REUSEPORT) and posix (O_SHLOCK, O_EXLOCK) constants +exposed on the API which are not implemented on GNU/Hurd. They would not +work at runtime anyway. + +.. + +.. bpo: 27025 +.. date: 9320 +.. nonce: ffzxpX +.. section: Library + +Generated names for Tkinter widgets are now more meanful and recognizirable. + +.. + +.. bpo: 25455 +.. date: 9319 +.. nonce: k10GoO +.. section: Library + +Fixed crashes in repr of recursive ElementTree.Element and functools.partial +objects. + +.. + +.. bpo: 27294 +.. date: 9318 +.. nonce: XPCURr +.. section: Library + +Improved repr for Tkinter event objects. + +.. + +.. bpo: 20508 +.. date: 9317 +.. nonce: 3NMbT2 +.. section: Library + +Improve exception message of IPv{4,6}Network.__getitem__. Patch by Gareth +Rees. + +.. + +.. bpo: 26556 +.. date: 9316 +.. nonce: v5j2uL +.. original section: Library +.. section: Security + +Update expat to 2.1.1, fixes CVE-2015-1283. + +.. + +.. bpo: 0 +.. date: 9315 +.. nonce: PHOAdg +.. original section: Library +.. section: Security + +Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team +Oststrom. + +.. + +.. bpo: 21386 +.. date: 9314 +.. nonce: DjV72U +.. section: Library + +Implement missing IPv4Address.is_global property. It was documented since +07a5610bae9d. Initial patch by Roger Luethi. + +.. + +.. bpo: 27029 +.. date: 9313 +.. nonce: dmycvw +.. section: Library + +Removed deprecated support of universal newlines mode from ZipFile.open(). + +.. + +.. bpo: 27030 +.. date: 9312 +.. nonce: p29J7m +.. section: Library + +Unknown escapes consisting of ``'\'`` and an ASCII letter in regular +expressions now are errors. The re.LOCALE flag now can be used only with +bytes patterns. + +.. + +.. bpo: 27186 +.. date: 9311 +.. nonce: UYiwoh +.. section: Library + +Add os.PathLike support to DirEntry (part of PEP 519). Initial patch by +Jelle Zijlstra. + +.. + +.. bpo: 20900 +.. date: 9310 +.. nonce: H5YQPR +.. section: Library + +distutils register command now decodes HTTP responses correctly. Initial +patch by ingrid. + +.. + +.. bpo: 27186 +.. date: 9309 +.. nonce: Xo4c_F +.. section: Library + +Add os.PathLike support to pathlib, removing its provisional status (part of +PEP 519). Initial patch by Dusty Phillips. + +.. + +.. bpo: 27186 +.. date: 9308 +.. nonce: ZD1wpp +.. section: Library + +Add support for os.PathLike objects to os.fsencode() and os.fsdecode() (part +of PEP 519). + +.. + +.. bpo: 27186 +.. date: 9307 +.. nonce: y7YRfj +.. section: Library + +Introduce os.PathLike and os.fspath() (part of PEP 519). + +.. + +.. bpo: 0 +.. date: 9306 +.. nonce: iYIeng +.. section: Library + +A new version of typing.py provides several new classes and features: +@overload outside stubs, Reversible, DefaultDict, Text, ContextManager, +Type[], NewType(), TYPE_CHECKING, and numerous bug fixes (note that some of +the new features are not yet implemented in mypy or other static analyzers). +Also classes for PEP 492 (Awaitable, AsyncIterable, AsyncIterator) have been +added (in fact they made it into 3.5.1 but were never mentioned). + +.. + +.. bpo: 25738 +.. date: 9305 +.. nonce: mED9w4 +.. section: Library + +Stop http.server.BaseHTTPRequestHandler.send_error() from sending a message +body for 205 Reset Content. Also, don't send Content header fields in +responses that don't have a body. Patch by Susumu Koshiba. + +.. + +.. bpo: 21313 +.. date: 9304 +.. nonce: W30MBr +.. section: Library + +Fix the "platform" module to tolerate when sys.version contains truncated +build information. + +.. + +.. bpo: 26839 +.. date: 9303 +.. nonce: yVvy7R +.. original section: Library +.. section: Security + +On Linux, :func:`os.urandom` now calls ``getrandom()`` with +``GRND_NONBLOCK`` to fall back on reading ``/dev/urandom`` if the urandom +entropy pool is not initialized yet. Patch written by Colm Buckley. + +.. + +.. bpo: 23883 +.. date: 9302 +.. nonce: tsZUiM +.. section: Library + +Added missing APIs to __all__ to match the documented APIs for the following +modules: cgi, mailbox, mimetypes, plistlib and smtpd. Patches by Jacek +Kołodziej. + +.. + +.. bpo: 27164 +.. date: 9301 +.. nonce: 6wmjx2 +.. section: Library + +In the zlib module, allow decompressing raw Deflate streams with a +predefined zdict. Based on patch by Xiang Zhang. + +.. + +.. bpo: 24291 +.. date: 9300 +.. nonce: Ac6HvL +.. section: Library + +Fix wsgiref.simple_server.WSGIRequestHandler to completely write data to the +client. Previously it could do partial writes and truncate data. Also, +wsgiref.handler.ServerHandler can now handle stdout doing partial writes, +but this is deprecated. + +.. + +.. bpo: 21272 +.. date: 9299 +.. nonce: unScIG +.. section: Library + +Use _sysconfigdata.py to initialize distutils.sysconfig. + +.. + +.. bpo: 19611 +.. date: 9298 +.. nonce: MT-Qga +.. section: Library + +:mod:`inspect` now reports the implicit ``.0`` parameters generated by the +compiler for comprehension and generator expression scopes as if they were +positional-only parameters called ``implicit0``. Patch by Jelle Zijlstra. + +.. + +.. bpo: 26809 +.. date: 9297 +.. nonce: ya7JMb +.. section: Library + +Add ``__all__`` to :mod:`string`. Patch by Emanuel Barry. + +.. + +.. bpo: 26373 +.. date: 9296 +.. nonce: P6qz6o +.. section: Library + +subprocess.Popen.communicate now correctly ignores BrokenPipeError when the +child process dies before .communicate() is called in more/all +circumstances. + +.. + +.. bpo: 0 +.. date: 9295 +.. nonce: eKchPz +.. section: Library + +signal, socket, and ssl module IntEnum constant name lookups now return a +consistent name for values having multiple names. Ex: signal.Signals(6) now +refers to itself as signal.SIGALRM rather than flipping between that and +signal.SIGIOT based on the interpreter's hash randomization seed. + +.. + +.. bpo: 27167 +.. date: 9294 +.. nonce: orA_j0 +.. section: Library + +Clarify the subprocess.CalledProcessError error message text when the child +process died due to a signal. + +.. + +.. bpo: 25931 +.. date: 9293 +.. nonce: W7h6Am +.. section: Library + +Don't define socketserver.Forking* names on platforms such as Windows that +do not support os.fork(). + +.. + +.. bpo: 21776 +.. date: 9292 +.. nonce: 04eQfa +.. section: Library + +distutils.upload now correctly handles HTTPError. Initial patch by Claudiu +Popa. + +.. + +.. bpo: 26526 +.. date: 9291 +.. nonce: ScewjJ +.. section: Library + +Replace custom parse tree validation in the parser module with a simple DFA +validator. + +.. + +.. bpo: 27114 +.. date: 9290 +.. nonce: bGCuAM +.. section: Library + +Fix SSLContext._load_windows_store_certs fails with PermissionError + +.. + +.. bpo: 18383 +.. date: 9289 +.. nonce: jr-b0l +.. section: Library + +Avoid creating duplicate filters when using filterwarnings and simplefilter. +Based on patch by Alex Shkop. + +.. + +.. bpo: 23026 +.. date: 9288 +.. nonce: V2rgYX +.. section: Library + +winreg.QueryValueEx() now return an integer for REG_QWORD type. + +.. + +.. bpo: 26741 +.. date: 9287 +.. nonce: fsbb42 +.. section: Library + +subprocess.Popen destructor now emits a ResourceWarning warning if the child +process is still running. + +.. + +.. bpo: 27056 +.. date: 9286 +.. nonce: rk-BBL +.. section: Library + +Optimize pickle.load() and pickle.loads(), up to 10% faster to deserialize a +lot of small objects. + +.. + +.. bpo: 21271 +.. date: 9285 +.. nonce: bHIfFA +.. section: Library + +New keyword only parameters in reset_mock call. + +.. + +.. bpo: 5124 +.. date: 9284 +.. nonce: 4kwBvM +.. section: IDLE + +Paste with text selected now replaces the selection on X11. This matches how +paste works on Windows, Mac, most modern Linux apps, and ttk widgets. +Original patch by Serhiy Storchaka. + +.. + +.. bpo: 24750 +.. date: 9283 +.. nonce: wA-pc9 +.. section: IDLE + +Switch all scrollbars in IDLE to ttk versions. Where needed, minimal tests +are added to cover changes. + +.. + +.. bpo: 24759 +.. date: 9282 +.. nonce: 76HB4w +.. section: IDLE + +IDLE requires tk 8.5 and availability ttk widgets. Delete now unneeded tk +version tests and code for older versions. Add test for IDLE syntax +colorizoer. + +.. + +.. bpo: 27239 +.. date: 9281 +.. nonce: fToURh +.. section: IDLE + +idlelib.macosx.isXyzTk functions initialize as needed. + +.. + +.. bpo: 27262 +.. date: 9280 +.. nonce: t7ckly +.. section: IDLE + +move Aqua unbinding code, which enable context menus, to maxosx. + +.. + +.. bpo: 24759 +.. date: 9279 +.. nonce: ccmySu +.. section: IDLE + +Make clear in idlelib.idle_test.__init__ that the directory is a private +implementation of test.test_idle and tool for maintainers. + +.. + +.. bpo: 27196 +.. date: 9278 +.. nonce: 3yp8TF +.. section: IDLE + +Stop 'ThemeChanged' warnings when running IDLE tests. These persisted after +other warnings were suppressed in #20567. Apply Serhiy Storchaka's +update_idletasks solution to four test files. Record this additional advice +in idle_test/README.txt + +.. + +.. bpo: 20567 +.. date: 9277 +.. nonce: hhT32b +.. section: IDLE + +Revise idle_test/README.txt with advice about avoiding tk warning messages +from tests. Apply advice to several IDLE tests. + +.. + +.. bpo: 24225 +.. date: 9276 +.. nonce: NxQCka +.. section: IDLE + +Update idlelib/README.txt with new file names and event handlers. + +.. + +.. bpo: 27156 +.. date: 9275 +.. nonce: j1N9br +.. section: IDLE + +Remove obsolete code not used by IDLE. + +.. + +.. bpo: 27117 +.. date: 9274 +.. nonce: YrLPf4 +.. section: IDLE + +Make colorizer htest and turtledemo work with dark themes. Move code for +configuring text widget colors to a new function. + +.. + +.. bpo: 24225 +.. date: 9273 +.. nonce: RbyFuV +.. section: IDLE + +Rename many `idlelib/*.py` and `idle_test/test_*.py` files. Edit files to +replace old names with new names when the old name referred to the module +rather than the class it contained. See the issue and IDLE section in What's +New in 3.6 for more. + +.. + +.. bpo: 26673 +.. date: 9272 +.. nonce: dh0_Ij +.. section: IDLE + +When tk reports font size as 0, change to size 10. Such fonts on Linux +prevented the configuration dialog from opening. + +.. + +.. bpo: 21939 +.. date: 9271 +.. nonce: pWz-OK +.. section: IDLE + +Add test for IDLE's percolator. Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 21676 +.. date: 9270 +.. nonce: hqy6Qh +.. section: IDLE + +Add test for IDLE's replace dialog. Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 18410 +.. date: 9269 +.. nonce: DLSPZo +.. section: IDLE + +Add test for IDLE's search dialog. Original patch by Westley Martínez. + +.. + +.. bpo: 21703 +.. date: 9268 +.. nonce: bEU8sP +.. section: IDLE + +Add test for undo delegator. Patch mostly by Saimadhav Heblikar . + +.. + +.. bpo: 27044 +.. date: 9267 +.. nonce: 4y7tyM +.. section: IDLE + +Add ConfigDialog.remove_var_callbacks to stop memory leaks. + +.. + +.. bpo: 23977 +.. date: 9266 +.. nonce: miDjj8 +.. section: IDLE + +Add more asserts to test_delegator. + +.. + +.. bpo: 16484 +.. date: 9265 +.. nonce: ITzcGg +.. section: Documentation + +Change the default PYTHONDOCS URL to "https:", and fix the resulting links +to use lowercase. Patch by Sean Rodman, test by Kaushik Nadikuditi. + +.. + +.. bpo: 24136 +.. date: 9264 +.. nonce: MUK0zK +.. section: Documentation + +Document the new PEP 448 unpacking syntax of 3.5. + +.. + +.. bpo: 22558 +.. date: 9263 +.. nonce: Pk02YC +.. section: Documentation + +Add remaining doc links to source code for Python-coded modules. Patch by +Yoni Lavi. + +.. + +.. bpo: 25285 +.. date: 9262 +.. nonce: 6CxIBo +.. section: Tests + +regrtest now uses subprocesses when the -j1 command line option is used: +each test file runs in a fresh child process. Before, the -j1 option was +ignored. + +.. + +.. bpo: 25285 +.. date: 9261 +.. nonce: ENYqUQ +.. section: Tests + +Tools/buildbot/test.bat script now uses -j1 by default to run each test file +in fresh child process. + +.. + +.. bpo: 27064 +.. date: 9260 +.. nonce: xeY1WF +.. section: Windows + +The py.exe launcher now defaults to Python 3. The Windows launcher +``py.exe`` no longer prefers an installed Python 2 version over Python 3 by +default when used interactively. + +.. + +.. bpo: 27229 +.. date: 9259 +.. nonce: C2NDch +.. section: Build + +Fix the cross-compiling pgen rule for in-tree builds. Patch by Xavier de +Gaye. + +.. + +.. bpo: 26930 +.. date: 9258 +.. nonce: Sqz2O3 +.. section: Build + +Update OS X 10.5+ 32-bit-only installer to build and link with OpenSSL +1.0.2h. + +.. + +.. bpo: 17500 +.. date: 9257 +.. nonce: QTZbRV +.. section: Windows + +Remove unused and outdated icons. (See also: +https://github.com/python/pythondotorg/issues/945) + +.. + +.. bpo: 27186 +.. date: 9256 +.. nonce: Ll8R-t +.. section: C API + +Add the PyOS_FSPath() function (part of PEP 519). + +.. + +.. bpo: 26282 +.. date: 9255 +.. nonce: Rp-R6L +.. section: C API + +PyArg_ParseTupleAndKeywords() now supports positional-only parameters. + +.. + +.. bpo: 26282 +.. date: 9254 +.. nonce: DRRV-- +.. section: Tools/Demos + +Argument Clinic now supports positional-only and keyword parameters in the +same function. diff --git a/Misc/NEWS.d/3.6.0a3.rst b/Misc/NEWS.d/3.6.0a3.rst new file mode 100644 index 00000000000000..251e6aa0e128d9 --- /dev/null +++ b/Misc/NEWS.d/3.6.0a3.rst @@ -0,0 +1,537 @@ +.. bpo: 27473 +.. date: 9385 +.. nonce: _nOtTA +.. release date: 2016-07-11 +.. section: Core and Builtins + +Fixed possible integer overflow in bytes and bytearray concatenations. +Patch by Xiang Zhang. + +.. + +.. bpo: 23034 +.. date: 9384 +.. nonce: GWaUqn +.. section: Core and Builtins + +The output of a special Python build with defined COUNT_ALLOCS, +SHOW_ALLOC_COUNT or SHOW_TRACK_COUNT macros is now off by default. It can +be re-enabled using the "-X showalloccount" option. It now outputs to +stderr instead of stdout. + +.. + +.. bpo: 27443 +.. date: 9383 +.. nonce: 87ZwZ1 +.. section: Core and Builtins + +__length_hint__() of bytearray iterators no longer return a negative integer +for a resized bytearray. + +.. + +.. bpo: 27007 +.. date: 9382 +.. nonce: Gg8Um4 +.. section: Core and Builtins + +The fromhex() class methods of bytes and bytearray subclasses now return an +instance of corresponding subclass. + +.. + +.. bpo: 26844 +.. date: 9381 +.. nonce: I0wdnY +.. section: Library + +Fix error message for imp.find_module() to refer to 'path' instead of +'name'. Patch by Lev Maximov. + +.. + +.. bpo: 23804 +.. date: 9380 +.. nonce: ipFvxc +.. section: Library + +Fix SSL zero-length recv() calls to not block and not raise an error about +unclean EOF. + +.. + +.. bpo: 27466 +.. date: 9379 +.. nonce: C_3a8E +.. section: Library + +Change time format returned by http.cookie.time2netscape, confirming the +netscape cookie format and making it consistent with documentation. + +.. + +.. bpo: 21708 +.. date: 9378 +.. nonce: RpPYiv +.. section: Library + +Deprecated dbm.dumb behavior that differs from common dbm behavior: creating +a database in 'r' and 'w' modes and modifying a database in 'r' mode. + +.. + +.. bpo: 26721 +.. date: 9377 +.. nonce: L37Y7r +.. section: Library + +Change the socketserver.StreamRequestHandler.wfile attribute to implement +BufferedIOBase. In particular, the write() method no longer does partial +writes. + +.. + +.. bpo: 22115 +.. date: 9376 +.. nonce: vG5UQW +.. section: Library + +Added methods trace_add, trace_remove and trace_info in the tkinter.Variable +class. They replace old methods trace_variable, trace, trace_vdelete and +trace_vinfo that use obsolete Tcl commands and might not work in future +versions of Tcl. Fixed old tracing methods: trace_vdelete() with wrong mode +no longer break tracing, trace_vinfo() now always returns a list of pairs of +strings, tracing in the "u" mode now works. + +.. + +.. bpo: 26243 +.. date: 9375 +.. nonce: dBtlhI +.. section: Library + +Only the level argument to zlib.compress() is keyword argument now. The +first argument is positional-only. + +.. + +.. bpo: 27038 +.. date: 9374 +.. nonce: yGMV4h +.. section: Library + +Expose the DirEntry type as os.DirEntry. Code patch by Jelle Zijlstra. + +.. + +.. bpo: 27186 +.. date: 9373 +.. nonce: OtorpF +.. section: Library + +Update os.fspath()/PyOS_FSPath() to check the return value of __fspath__() +to be either str or bytes. + +.. + +.. bpo: 18726 +.. date: 9372 +.. nonce: eIXHIl +.. section: Library + +All optional parameters of the dump(), dumps(), load() and loads() functions +and JSONEncoder and JSONDecoder class constructors in the json module are +now keyword-only. + +.. + +.. bpo: 27319 +.. date: 9371 +.. nonce: vDl2zm +.. section: Library + +Methods selection_set(), selection_add(), selection_remove() and +selection_toggle() of ttk.TreeView now allow passing multiple items as +multiple arguments instead of passing them as a tuple. Deprecated +undocumented ability of calling the selection() method with arguments. + +.. + +.. bpo: 27079 +.. date: 9370 +.. nonce: c7d0Ym +.. section: Library + +Fixed curses.ascii functions isblank(), iscntrl() and ispunct(). + +.. + +.. bpo: 27294 +.. date: 9369 +.. nonce: 0WSp9y +.. section: Library + +Numerical state in the repr for Tkinter event objects is now represented as +a combination of known flags. + +.. + +.. bpo: 27177 +.. date: 9368 +.. nonce: U6jRnd +.. section: Library + +Match objects in the re module now support index-like objects as group +indices. Based on patches by Jeroen Demeyer and Xiang Zhang. + +.. + +.. bpo: 26754 +.. date: 9367 +.. nonce: J3n0QW +.. section: Library + +Some functions (compile() etc) accepted a filename argument encoded as an +iterable of integers. Now only strings and byte-like objects are accepted. + +.. + +.. bpo: 26536 +.. date: 9366 +.. nonce: DgLWm- +.. section: Library + +socket.ioctl now supports SIO_LOOPBACK_FAST_PATH. Patch by Daniel Stokes. + +.. + +.. bpo: 27048 +.. date: 9365 +.. nonce: EVe-Bk +.. section: Library + +Prevents distutils failing on Windows when environment variables contain +non-ASCII characters + +.. + +.. bpo: 27330 +.. date: 9364 +.. nonce: GJaFCV +.. section: Library + +Fixed possible leaks in the ctypes module. + +.. + +.. bpo: 27238 +.. date: 9363 +.. nonce: Q6v6Qv +.. section: Library + +Got rid of bare excepts in the turtle module. Original patch by Jelle +Zijlstra. + +.. + +.. bpo: 27122 +.. date: 9362 +.. nonce: 06t7zN +.. section: Library + +When an exception is raised within the context being managed by a +contextlib.ExitStack() and one of the exit stack generators catches and +raises it in a chain, do not re-raise the original exception when exiting, +let the new chained one through. This avoids the PEP 479 bug described in +issue25782. + +.. + +.. bpo: 27278 +.. date: 9361 +.. nonce: y_HkGw +.. original section: Library +.. section: Security + +Fix os.urandom() implementation using getrandom() on Linux. Truncate size +to INT_MAX and loop until we collected enough random bytes, instead of +casting a directly Py_ssize_t to int. + +.. + +.. bpo: 16864 +.. date: 9360 +.. nonce: W7tJDa +.. section: Library + +sqlite3.Cursor.lastrowid now supports REPLACE statement. Initial patch by +Alex LordThorsen. + +.. + +.. bpo: 26386 +.. date: 9359 +.. nonce: 9L3Ut4 +.. section: Library + +Fixed ttk.TreeView selection operations with item id's containing spaces. + +.. + +.. bpo: 8637 +.. date: 9358 +.. nonce: lHiUSA +.. section: Library + +Honor a pager set by the env var MANPAGER (in preference to one set by the +env var PAGER). + +.. + +.. bpo: 22636 +.. date: 9357 +.. nonce: 3fQW_g +.. original section: Library +.. section: Security + +Avoid shell injection problems with ctypes.util.find_library(). + +.. + +.. bpo: 16182 +.. date: 9356 +.. nonce: RgFXyr +.. section: Library + +Fix various functions in the "readline" module to use the locale encoding, +and fix get_begidx() and get_endidx() to return code point indexes. + +.. + +.. bpo: 27392 +.. date: 9355 +.. nonce: obfni7 +.. section: Library + +Add loop.connect_accepted_socket(). Patch by Jim Fulton. + +.. + +.. bpo: 27477 +.. date: 9354 +.. nonce: iEuL-9 +.. section: IDLE + +IDLE search dialogs now use ttk widgets. + +.. + +.. bpo: 27173 +.. date: 9353 +.. nonce: M-fYaV +.. section: IDLE + +Add 'IDLE Modern Unix' to the built-in key sets. Make the default key set +depend on the platform. Add tests for the changes to the config module. + +.. + +.. bpo: 27452 +.. date: 9352 +.. nonce: dLxZ8W +.. section: IDLE + +make command line "idle-test> python test_help.py" work. __file__ is +relative when python is started in the file's directory. + +.. + +.. bpo: 27452 +.. date: 9351 +.. nonce: RtWnyR +.. section: IDLE + +add line counter and crc to IDLE configHandler test dump. + +.. + +.. bpo: 27380 +.. date: 9350 +.. nonce: Q39r9U +.. section: IDLE + +IDLE: add query.py with base Query dialog and ttk widgets. Module had +subclasses SectionName, ModuleName, and HelpSource, which are used to get +information from users by configdialog and file =>Load Module. Each subclass +has itw own validity checks. Using ModuleName allows users to edit bad +module names instead of starting over. Add tests and delete the two files +combined into the new one. + +.. + +.. bpo: 27372 +.. date: 9349 +.. nonce: k3Wj2V +.. section: IDLE + +Test_idle no longer changes the locale. + +.. + +.. bpo: 27365 +.. date: 9348 +.. nonce: y7ys_A +.. section: IDLE + +Allow non-ascii chars in IDLE NEWS.txt, for contributor names. + +.. + +.. bpo: 27245 +.. date: 9347 +.. nonce: u9aKO1 +.. section: IDLE + +IDLE: Cleanly delete custom themes and key bindings. Previously, when IDLE +was started from a console or by import, a cascade of warnings was emitted. +Patch by Serhiy Storchaka. + +.. + +.. bpo: 24137 +.. date: 9346 +.. nonce: v8o-IT +.. section: IDLE + +Run IDLE, test_idle, and htest with tkinter default root disabled. Fix code +and tests that fail with this restriction. Fix htests to not create a +second and redundant root and mainloop. + +.. + +.. bpo: 27310 +.. date: 9345 +.. nonce: KiURpC +.. section: IDLE + +Fix IDLE.app failure to launch on OS X due to vestigial import. + +.. + +.. bpo: 26754 +.. date: 9344 +.. nonce: Qm_N79 +.. section: C API + +PyUnicode_FSDecoder() accepted a filename argument encoded as an iterable of +integers. Now only strings and byte-like objects are accepted. + +.. + +.. bpo: 28066 +.. date: 9343 +.. nonce: _3xImV +.. section: Build + +Fix the logic that searches build directories for generated include files +when building outside the source tree. + +.. + +.. bpo: 27442 +.. date: 9342 +.. nonce: S2M0cz +.. section: Build + +Expose the Android API level that python was built against, in +sysconfig.get_config_vars() as 'ANDROID_API_LEVEL'. + +.. + +.. bpo: 27434 +.. date: 9341 +.. nonce: 4nRZmn +.. section: Build + +The interpreter that runs the cross-build, found in PATH, must now be of the +same feature version (e.g. 3.6) as the source being built. + +.. + +.. bpo: 26930 +.. date: 9340 +.. nonce: 9JUeSD +.. section: Build + +Update Windows builds to use OpenSSL 1.0.2h. + +.. + +.. bpo: 23968 +.. date: 9339 +.. nonce: 7AuSK9 +.. section: Build + +Rename the platform directory from plat-$(MACHDEP) to +plat-$(PLATFORM_TRIPLET). Rename the config directory (LIBPL) from +config-$(LDVERSION) to config-$(LDVERSION)-$(PLATFORM_TRIPLET). Install the +platform specifc _sysconfigdata module into the platform directory and +rename it to include the ABIFLAGS. + +.. + +.. bpo: 0 +.. date: 9338 +.. nonce: U46i2u +.. section: Build + +Don't use largefile support for GNU/Hurd. + +.. + +.. bpo: 27332 +.. date: 9337 +.. nonce: OuRZp9 +.. section: Tools/Demos + +Fixed the type of the first argument of module-level functions generated by +Argument Clinic. Patch by Petr Viktorin. + +.. + +.. bpo: 27418 +.. date: 9336 +.. nonce: W2m_8I +.. section: Tools/Demos + +Fixed Tools/importbench/importbench.py. + +.. + +.. bpo: 19489 +.. date: 9335 +.. nonce: jvzuO7 +.. section: Documentation + +Moved the search box from the sidebar to the header and footer of each page. +Patch by Ammar Askar. + +.. + +.. bpo: 27285 +.. date: 9334 +.. nonce: wZur0b +.. section: Documentation + +Update documentation to reflect the deprecation of ``pyvenv`` and normalize +on the term "virtual environment". Patch by Steve Piercy. + +.. + +.. bpo: 27027 +.. date: 9333 +.. nonce: 5oRSGL +.. section: Tests + +Added test.support.is_android that is True when this is an Android build. diff --git a/Misc/NEWS.d/3.6.0a4.rst b/Misc/NEWS.d/3.6.0a4.rst new file mode 100644 index 00000000000000..e2480335c5d405 --- /dev/null +++ b/Misc/NEWS.d/3.6.0a4.rst @@ -0,0 +1,685 @@ +.. bpo: 27704 +.. date: 9455 +.. nonce: RUxzHf +.. release date: 2016-08-15 +.. section: Core and Builtins + +Optimized creating bytes and bytearray from byte-like objects and iterables. +Speed up to 3 times for short objects. Original patch by Naoki Inada. + +.. + +.. bpo: 26823 +.. date: 9454 +.. nonce: UWORiU +.. section: Core and Builtins + +Large sections of repeated lines in tracebacks are now abbreviated as +"[Previous line repeated {count} more times]" by the builtin traceback +rendering. Patch by Emanuel Barry. + +.. + +.. bpo: 27574 +.. date: 9453 +.. nonce: q73Tss +.. section: Core and Builtins + +Decreased an overhead of parsing keyword arguments in functions implemented +with using Argument Clinic. + +.. + +.. bpo: 22557 +.. date: 9452 +.. nonce: Hta2Rz +.. section: Core and Builtins + +Now importing already imported modules is up to 2.5 times faster. + +.. + +.. bpo: 17596 +.. date: 9451 +.. nonce: XgbA9V +.. section: Core and Builtins + +Include to help with Min GW building. + +.. + +.. bpo: 17599 +.. date: 9450 +.. nonce: noy7o1 +.. section: Core and Builtins + +On Windows, rename the privately defined REPARSE_DATA_BUFFER structure to +avoid conflicting with the definition from Min GW. + +.. + +.. bpo: 27507 +.. date: 9449 +.. nonce: 3pX0Be +.. section: Core and Builtins + +Add integer overflow check in bytearray.extend(). Patch by Xiang Zhang. + +.. + +.. bpo: 27581 +.. date: 9448 +.. nonce: KezjNt +.. section: Core and Builtins + +Don't rely on wrapping for overflow check in PySequence_Tuple(). Patch by +Xiang Zhang. + +.. + +.. bpo: 1621 +.. date: 9447 +.. nonce: _FZWTr +.. section: Core and Builtins + +Avoid signed integer overflow in list and tuple operations. Patch by Xiang +Zhang. + +.. + +.. bpo: 27419 +.. date: 9446 +.. nonce: YaGodL +.. section: Core and Builtins + +Standard __import__() no longer look up "__import__" in globals or builtins +for importing submodules or "from import". Fixed a crash if raise a warning +about unabling to resolve package from __spec__ or __package__. + +.. + +.. bpo: 27083 +.. date: 9445 +.. nonce: F4ZT1C +.. section: Core and Builtins + +Respect the PYTHONCASEOK environment variable under Windows. + +.. + +.. bpo: 27514 +.. date: 9444 +.. nonce: NLbwPG +.. section: Core and Builtins + +Make having too many statically nested blocks a SyntaxError instead of +SystemError. + +.. + +.. bpo: 27366 +.. date: 9443 +.. nonce: VrInsj +.. section: Core and Builtins + +Implemented PEP 487 (Simpler customization of class creation). Upon +subclassing, the __init_subclass__ classmethod is called on the base class. +Descriptors are initialized with __set_name__ after class creation. + +.. + +.. bpo: 26027 +.. date: 9442 +.. nonce: nfVMKM +.. section: Library + +Add PEP 519/__fspath__() support to the os and os.path modules. Includes +code from Jelle Zijlstra. (See also: bpo-27524) + +.. + +.. bpo: 27598 +.. date: 9441 +.. nonce: y7PtEV +.. section: Library + +Add Collections to collections.abc. Patch by Ivan Levkivskyi, docs by Neil +Girdhar. + +.. + +.. bpo: 25958 +.. date: 9440 +.. nonce: X-V4U1 +.. section: Library + +Support "anti-registration" of special methods from various ABCs, like +__hash__, __iter__ or __len__. All these (and several more) can be set to +None in an implementation class and the behavior will be as if the method is +not defined at all. (Previously, this mechanism existed only for __hash__, +to make mutable classes unhashable.) Code contributed by Andrew Barnert and +Ivan Levkivskyi. + +.. + +.. bpo: 16764 +.. date: 9439 +.. nonce: cPbNjL +.. section: Library + +Support keyword arguments to zlib.decompress(). Patch by Xiang Zhang. + +.. + +.. bpo: 27736 +.. date: 9438 +.. nonce: 8kMhpQ +.. section: Library + +Prevent segfault after interpreter re-initialization due to ref count +problem introduced in code for Issue #27038 in 3.6.0a3. Patch by Xiang +Zhang. + +.. + +.. bpo: 25628 +.. date: 9437 +.. nonce: UcQnHF +.. section: Library + +The *verbose* and *rename* parameters for collections.namedtuple are now +keyword-only. + +.. + +.. bpo: 12345 +.. date: 9436 +.. nonce: nbAEM8 +.. section: Library + +Add mathematical constant tau to math and cmath. See also PEP 628. + +.. + +.. bpo: 26823 +.. date: 9435 +.. nonce: HcO8tR +.. section: Library + +traceback.StackSummary.format now abbreviates large sections of repeated +lines as "[Previous line repeated {count} more times]" (this change then +further affects other traceback display operations in the module). Patch by +Emanuel Barry. + +.. + +.. bpo: 27664 +.. date: 9434 +.. nonce: 6DJPxw +.. section: Library + +Add to concurrent.futures.thread.ThreadPoolExecutor() the ability to specify +a thread name prefix. + +.. + +.. bpo: 27181 +.. date: 9433 +.. nonce: 8aw9TZ +.. section: Library + +Add geometric_mean and harmonic_mean to statistics module. + +.. + +.. bpo: 27573 +.. date: 9432 +.. nonce: B7XhTs +.. section: Library + +code.interact now prints an message when exiting. + +.. + +.. bpo: 6422 +.. date: 9431 +.. nonce: iBSc45 +.. section: Library + +Add autorange method to timeit.Timer objects. + +.. + +.. bpo: 27773 +.. date: 9430 +.. nonce: hMSSeX +.. section: Library + +Correct some memory management errors server_hostname in _ssl.wrap_socket(). + +.. + +.. bpo: 26750 +.. date: 9429 +.. nonce: OQn3fr +.. section: Library + +unittest.mock.create_autospec() now works properly for subclasses of +property() and other data descriptors. Removes the never publicly used, +never documented unittest.mock.DescriptorTypes tuple. + +.. + +.. bpo: 26754 +.. date: 9428 +.. nonce: XZqomf +.. section: Library + +Undocumented support of general bytes-like objects as path in compile() and +similar functions is now deprecated. + +.. + +.. bpo: 26800 +.. date: 9427 +.. nonce: QDcK8u +.. section: Library + +Undocumented support of general bytes-like objects as paths in os functions +is now deprecated. + +.. + +.. bpo: 26981 +.. date: 9426 +.. nonce: yhNTCf +.. section: Library + +Add _order_ compatibility shim to enum.Enum for Python 2/3 code bases. + +.. + +.. bpo: 27661 +.. date: 9425 +.. nonce: 3JZckO +.. section: Library + +Added tzinfo keyword argument to datetime.combine. + +.. + +.. bpo: 0 +.. date: 9424 +.. nonce: Ny9oPv +.. section: Library + +In the curses module, raise an error if window.getstr() or window.instr() is +passed a negative value. + +.. + +.. bpo: 27783 +.. date: 9423 +.. nonce: cR1jXH +.. section: Library + +Fix possible usage of uninitialized memory in operator.methodcaller. + +.. + +.. bpo: 27774 +.. date: 9422 +.. nonce: FDcik1 +.. section: Library + +Fix possible Py_DECREF on unowned object in _sre. + +.. + +.. bpo: 27760 +.. date: 9421 +.. nonce: gxMjp4 +.. section: Library + +Fix possible integer overflow in binascii.b2a_qp. + +.. + +.. bpo: 27758 +.. date: 9420 +.. nonce: 0NRV03 +.. section: Library + +Fix possible integer overflow in the _csv module for large record lengths. + +.. + +.. bpo: 27568 +.. date: 9419 +.. nonce: OnuO9s +.. section: Library + +Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +when REQUEST_METHOD environment is set, which indicates that the script is +in CGI mode. + +.. + +.. bpo: 7063 +.. date: 9418 +.. nonce: nXsVKB +.. section: Library + +Remove dead code from the "array" module's slice handling. Patch by Chuck. + +.. + +.. bpo: 27656 +.. date: 9417 +.. nonce: joTscM +.. section: Library + +Do not assume sched.h defines any SCHED_* constants. + +.. + +.. bpo: 27130 +.. date: 9416 +.. nonce: SUxwXZ +.. section: Library + +In the "zlib" module, fix handling of large buffers (typically 4 GiB) when +compressing and decompressing. Previously, inputs were limited to 4 GiB, +and compression and decompression operations did not properly handle results +of 4 GiB. + +.. + +.. bpo: 24773 +.. date: 9415 +.. nonce: IDW05R +.. section: Library + +Implemented PEP 495 (Local Time Disambiguation). + +.. + +.. bpo: 0 +.. date: 9414 +.. nonce: lOkwM8 +.. section: Library + +Expose the EPOLLEXCLUSIVE constant (when it is defined) in the select +module. + +.. + +.. bpo: 27567 +.. date: 9413 +.. nonce: bYOgyw +.. section: Library + +Expose the EPOLLRDHUP and POLLRDHUP constants in the select module. + +.. + +.. bpo: 1621 +.. date: 9412 +.. nonce: 0nclmI +.. section: Library + +Avoid signed int negation overflow in the "audioop" module. + +.. + +.. bpo: 27533 +.. date: 9411 +.. nonce: iDmKzV +.. section: Library + +Release GIL in nt._isdir + +.. + +.. bpo: 17711 +.. date: 9410 +.. nonce: 47AILJ +.. section: Library + +Fixed unpickling by the persistent ID with protocol 0. Original patch by +Alexandre Vassalotti. + +.. + +.. bpo: 27522 +.. date: 9409 +.. nonce: 8vVz_t +.. section: Library + +Avoid an unintentional reference cycle in email.feedparser. + +.. + +.. bpo: 27512 +.. date: 9408 +.. nonce: FaGwup +.. section: Library + +Fix a segfault when os.fspath() called an __fspath__() method that raised an +exception. Patch by Xiang Zhang. + +.. + +.. bpo: 27714 +.. date: 9407 +.. nonce: bUEDsI +.. section: IDLE + +text_textview and test_autocomplete now pass when re-run in the same +process. This occurs when test_idle fails when run with the -w option but +without -jn. Fix warning from test_config. + +.. + +.. bpo: 27621 +.. date: 9406 +.. nonce: BcpOPU +.. section: IDLE + +Put query response validation error messages in the query box itself instead +of in a separate massagebox. Redo tests to match. Add Mac OSX refinements. +Original patch by Mark Roseman. + +.. + +.. bpo: 27620 +.. date: 9405 +.. nonce: TXRR6x +.. section: IDLE + +Escape key now closes Query box as cancelled. + +.. + +.. bpo: 27609 +.. date: 9404 +.. nonce: MbTuKa +.. section: IDLE + +IDLE: tab after initial whitespace should tab, not autocomplete. This fixes +problem with writing docstrings at least twice indented. + +.. + +.. bpo: 27609 +.. date: 9403 +.. nonce: OBYgv_ +.. section: IDLE + +Explicitly return None when there are also non-None returns. In a few cases, +reverse a condition and eliminate a return. + +.. + +.. bpo: 25507 +.. date: 9402 +.. nonce: lxf68d +.. section: IDLE + +IDLE no longer runs buggy code because of its tkinter imports. Users must +include the same imports required to run directly in Python. + +.. + +.. bpo: 27173 +.. date: 9401 +.. nonce: M-fYaV +.. section: IDLE + +Add 'IDLE Modern Unix' to the built-in key sets. Make the default key set +depend on the platform. Add tests for the changes to the config module. + +.. + +.. bpo: 27452 +.. date: 9400 +.. nonce: RtWnyR +.. section: IDLE + +add line counter and crc to IDLE configHandler test dump. + +.. + +.. bpo: 25805 +.. date: 9399 +.. nonce: 9SVxXQ +.. section: Tests + +Skip a test in test_pkgutil as needed that doesn't work when ``__name__ == +__main__``. Patch by SilentGhost. + +.. + +.. bpo: 27472 +.. date: 9398 +.. nonce: NS3L93 +.. section: Tests + +Add test.support.unix_shell as the path to the default shell. + +.. + +.. bpo: 27369 +.. date: 9397 +.. nonce: LG7U2D +.. section: Tests + +In test_pyexpat, avoid testing an error message detail that changed in Expat +2.2.0. + +.. + +.. bpo: 27594 +.. date: 9396 +.. nonce: w3F57B +.. section: Tests + +Prevent assertion error when running test_ast with coverage enabled: ensure +code object has a valid first line number. Patch suggested by Ivan +Levkivskyi. + +.. + +.. bpo: 27647 +.. date: 9395 +.. nonce: -1HUR6 +.. section: Windows + +Update bundled Tcl/Tk to 8.6.6. + +.. + +.. bpo: 27610 +.. date: 9394 +.. nonce: O0o0mB +.. section: Windows + +Adds PEP 514 metadata to Windows installer + +.. + +.. bpo: 27469 +.. date: 9393 +.. nonce: 0GwDkX +.. section: Windows + +Adds a shell extension to the launcher so that drag and drop works +correctly. + +.. + +.. bpo: 27309 +.. date: 9392 +.. nonce: chiOo6 +.. section: Windows + +Enables proper Windows styles in python[w].exe manifest. + +.. + +.. bpo: 27713 +.. date: 9391 +.. nonce: _3DgXG +.. section: Build + +Suppress spurious build warnings when updating importlib's bootstrap files. +Patch by Xiang Zhang + +.. + +.. bpo: 25825 +.. date: 9390 +.. nonce: MLbdVU +.. section: Build + +Correct the references to Modules/python.exp, which is required on AIX. The +references were accidentally changed in 3.5.0a1. + +.. + +.. bpo: 27453 +.. date: 9389 +.. nonce: Pb5DBi +.. section: Build + +CPP invocation in configure must use CPPFLAGS. Patch by Chi Hsuan Yen. + +.. + +.. bpo: 27641 +.. date: 9388 +.. nonce: eGzgCk +.. section: Build + +The configure script now inserts comments into the makefile to prevent the +pgen and _freeze_importlib executables from being cross- compiled. + +.. + +.. bpo: 26662 +.. date: 9387 +.. nonce: XkwRxM +.. section: Build + +Set PYTHON_FOR_GEN in configure as the Python program to be used for file +generation during the build. + +.. + +.. bpo: 10910 +.. date: 9386 +.. nonce: ZdRayb +.. section: Build + +Avoid C++ compilation errors on FreeBSD and OS X. Also update FreedBSD +version checks for the original ctype UTF-8 workaround. diff --git a/Misc/NEWS.d/3.6.0b1.rst b/Misc/NEWS.d/3.6.0b1.rst new file mode 100644 index 00000000000000..655f6f749e466f --- /dev/null +++ b/Misc/NEWS.d/3.6.0b1.rst @@ -0,0 +1,1608 @@ +.. bpo: 23722 +.. date: 9619 +.. nonce: C-8boi +.. release date: 2016-09-12 +.. section: Core and Builtins + +The __class__ cell used by zero-argument super() is now initialized from +type.__new__ rather than __build_class__, so class methods relying on that +will now work correctly when called from metaclass methods during class +creation. Patch by Martin Teichmann. + +.. + +.. bpo: 25221 +.. date: 9618 +.. nonce: 9YbOxB +.. section: Core and Builtins + +Fix corrupted result from PyLong_FromLong(0) when Python is compiled with +NSMALLPOSINTS = 0. + +.. + +.. bpo: 27080 +.. date: 9617 +.. nonce: Te4Tjb +.. section: Core and Builtins + +Implement formatting support for PEP 515. Initial patch by Chris Angelico. + +.. + +.. bpo: 27199 +.. date: 9616 +.. nonce: GheADD +.. section: Core and Builtins + +In tarfile, expose copyfileobj bufsize to improve throughput. Patch by Jason +Fried. + +.. + +.. bpo: 27948 +.. date: 9615 +.. nonce: Rpw5nq +.. section: Core and Builtins + +In f-strings, only allow backslashes inside the braces (where the +expressions are). This is a breaking change from the 3.6 alpha releases, +where backslashes are allowed anywhere in an f-string. Also, require that +expressions inside f-strings be enclosed within literal braces, and not +escapes like ``f'\x7b"hi"\x7d'``. + +.. + +.. bpo: 28046 +.. date: 9614 +.. nonce: liHxFW +.. section: Core and Builtins + +Remove platform-specific directories from sys.path. + +.. + +.. bpo: 28071 +.. date: 9613 +.. nonce: PffE44 +.. section: Core and Builtins + +Add early-out for differencing from an empty set. + +.. + +.. bpo: 25758 +.. date: 9612 +.. nonce: yR-YTD +.. section: Core and Builtins + +Prevents zipimport from unnecessarily encoding a filename (patch by Eryk +Sun) + +.. + +.. bpo: 25856 +.. date: 9611 +.. nonce: neCvXl +.. section: Core and Builtins + +The __module__ attribute of extension classes and functions now is interned. +This leads to more compact pickle data with protocol 4. + +.. + +.. bpo: 27213 +.. date: 9610 +.. nonce: VCfkkp +.. section: Core and Builtins + +Rework CALL_FUNCTION* opcodes to produce shorter and more efficient +bytecode. Patch by Demur Rumed, design by Serhiy Storchaka, reviewed by +Serhiy Storchaka and Victor Stinner. + +.. + +.. bpo: 26331 +.. date: 9609 +.. nonce: TdJp8_ +.. section: Core and Builtins + +Implement tokenizing support for PEP 515. Patch by Georg Brandl. + +.. + +.. bpo: 27999 +.. date: 9608 +.. nonce: 8aacQj +.. section: Core and Builtins + +Make "global after use" a SyntaxError, and ditto for nonlocal. Patch by Ivan +Levkivskyi. + +.. + +.. bpo: 28003 +.. date: 9607 +.. nonce: noeoav +.. section: Core and Builtins + +Implement PEP 525 -- Asynchronous Generators. + +.. + +.. bpo: 27985 +.. date: 9606 +.. nonce: 0ayJ5k +.. section: Core and Builtins + +Implement PEP 526 -- Syntax for Variable Annotations. Patch by Ivan +Levkivskyi. + +.. + +.. bpo: 26058 +.. date: 9605 +.. nonce: UR_ojv +.. section: Core and Builtins + +Add a new private version to the builtin dict type, incremented at each +dictionary creation and at each dictionary change. Implementation of the PEP +509. + +.. + +.. bpo: 27364 +.. date: 9604 +.. nonce: 8u_LoD +.. section: Core and Builtins + +A backslash-character pair that is not a valid escape sequence now generates +a DeprecationWarning. Patch by Emanuel Barry. + +.. + +.. bpo: 27350 +.. date: 9603 +.. nonce: aABzcL +.. section: Core and Builtins + +`dict` implementation is changed like PyPy. It is more compact and preserves +insertion order. (Concept developed by Raymond Hettinger and patch by Inada +Naoki.) + +.. + +.. bpo: 27911 +.. date: 9602 +.. nonce: 1eaHRd +.. section: Core and Builtins + +Remove unnecessary error checks in ``exec_builtin_or_dynamic()``. + +.. + +.. bpo: 27078 +.. date: 9601 +.. nonce: ZevPQR +.. section: Core and Builtins + +Added BUILD_STRING opcode. Optimized f-strings evaluation. + +.. + +.. bpo: 17884 +.. date: 9600 +.. nonce: wGy0dr +.. section: Core and Builtins + +Python now requires systems with inttypes.h and stdint.h + +.. + +.. bpo: 27961 +.. date: 9599 +.. nonce: EYS8oe +.. section: Core and Builtins + +Require platforms to support ``long long``. Python hasn't compiled without +``long long`` for years, so this is basically a formality. + +.. + +.. bpo: 27355 +.. date: 9598 +.. nonce: qdIpxm +.. section: Core and Builtins + +Removed support for Windows CE. It was never finished, and Windows CE is no +longer a relevant platform for Python. + +.. + +.. bpo: 0 +.. date: 9597 +.. nonce: rdhhVw +.. section: Core and Builtins + +Implement PEP 523. + +.. + +.. bpo: 27870 +.. date: 9596 +.. nonce: Y0u34u +.. section: Core and Builtins + +A left shift of zero by a large integer no longer attempts to allocate large +amounts of memory. + +.. + +.. bpo: 25402 +.. date: 9595 +.. nonce: naeRHq +.. section: Core and Builtins + +In int-to-decimal-string conversion, improve the estimate of the +intermediate memory required, and remove an unnecessarily strict overflow +check. Patch by Serhiy Storchaka. + +.. + +.. bpo: 27214 +.. date: 9594 +.. nonce: CDh8S4 +.. section: Core and Builtins + +In long_invert, be more careful about modifying object returned by long_add, +and remove an unnecessary check for small longs. Thanks Oren Milman for +analysis and patch. + +.. + +.. bpo: 27506 +.. date: 9593 +.. nonce: eK87PI +.. section: Core and Builtins + +Support passing the bytes/bytearray.translate() "delete" argument by +keyword. + +.. + +.. bpo: 27812 +.. date: 9592 +.. nonce: sidcs8 +.. section: Core and Builtins + +Properly clear out a generator's frame's backreference to the generator to +prevent crashes in frame.clear(). + +.. + +.. bpo: 27811 +.. date: 9591 +.. nonce: T4AuBo +.. section: Core and Builtins + +Fix a crash when a coroutine that has not been awaited is finalized with +warnings-as-errors enabled. + +.. + +.. bpo: 27587 +.. date: 9590 +.. nonce: mbavY2 +.. section: Core and Builtins + +Fix another issue found by PVS-Studio: Null pointer check after use of 'def' +in _PyState_AddModule(). Initial patch by Christian Heimes. + +.. + +.. bpo: 27792 +.. date: 9589 +.. nonce: Np6_Hl +.. section: Core and Builtins + +The modulo operation applied to ``bool`` and other ``int`` subclasses now +always returns an ``int``. Previously the return type depended on the input +values. Patch by Xiang Zhang. + +.. + +.. bpo: 26984 +.. date: 9588 +.. nonce: 7--80J +.. section: Core and Builtins + +int() now always returns an instance of exact int. + +.. + +.. bpo: 25604 +.. date: 9587 +.. nonce: UkeHGy +.. section: Core and Builtins + +Fix a minor bug in integer true division; this bug could potentially have +caused off-by-one-ulp results on platforms with unreliable ldexp +implementations. + +.. + +.. bpo: 24254 +.. date: 9586 +.. nonce: 368r1U +.. section: Core and Builtins + +Make class definition namespace ordered by default. + +.. + +.. bpo: 27662 +.. date: 9585 +.. nonce: a8cBpq +.. section: Core and Builtins + +Fix an overflow check in ``List_New``: the original code was checking +against ``Py_SIZE_MAX`` instead of the correct upper bound of +``Py_SSIZE_T_MAX``. Patch by Xiang Zhang. + +.. + +.. bpo: 27782 +.. date: 9584 +.. nonce: C8OBQD +.. section: Core and Builtins + +Multi-phase extension module import now correctly allows the ``m_methods`` +field to be used to add module level functions to instances of non-module +types returned from ``Py_create_mod``. Patch by Xiang Zhang. + +.. + +.. bpo: 27936 +.. date: 9583 +.. nonce: AdOann +.. section: Core and Builtins + +The round() function accepted a second None argument for some types but not +for others. Fixed the inconsistency by accepting None for all numeric +types. + +.. + +.. bpo: 27487 +.. date: 9582 +.. nonce: jeTQNr +.. section: Core and Builtins + +Warn if a submodule argument to "python -m" or runpy.run_module() is found +in sys.modules after parent packages are imported, but before the submodule +is executed. + +.. + +.. bpo: 27157 +.. date: 9581 +.. nonce: Wf_eFE +.. section: Core and Builtins + +Make only type() itself accept the one-argument form. Patch by Eryk Sun and +Emanuel Barry. + +.. + +.. bpo: 27558 +.. date: 9580 +.. nonce: VmltMh +.. section: Core and Builtins + +Fix a SystemError in the implementation of "raise" statement. In a brand new +thread, raise a RuntimeError since there is no active exception to reraise. +Patch written by Xiang Zhang. + +.. + +.. bpo: 28008 +.. date: 9579 +.. nonce: 0DdIrA +.. section: Core and Builtins + +Implement PEP 530 -- asynchronous comprehensions. + +.. + +.. bpo: 27942 +.. date: 9578 +.. nonce: wCAkW5 +.. section: Core and Builtins + +Fix memory leak in codeobject.c + +.. + +.. bpo: 28732 +.. date: 9577 +.. nonce: xkG8k7 +.. section: Library + +Fix crash in os.spawnv() with no elements in args + +.. + +.. bpo: 28485 +.. date: 9576 +.. nonce: WuKqKh +.. section: Library + +Always raise ValueError for negative compileall.compile_dir(workers=...) +parameter, even when multithreading is unavailable. + +.. + +.. bpo: 28037 +.. date: 9575 +.. nonce: -3u7zq +.. section: Library + +Use sqlite3_get_autocommit() instead of setting Connection->inTransaction +manually. + +.. + +.. bpo: 25283 +.. date: 9574 +.. nonce: qwQDX2 +.. section: Library + +Attributes tm_gmtoff and tm_zone are now available on all platforms in the +return values of time.localtime() and time.gmtime(). + +.. + +.. bpo: 24454 +.. date: 9573 +.. nonce: pUTKOA +.. section: Library + +Regular expression match object groups are now accessible using __getitem__. +"mo[x]" is equivalent to "mo.group(x)". + +.. + +.. bpo: 10740 +.. date: 9572 +.. nonce: 8iGFan +.. section: Library + +sqlite3 no longer implicitly commit an open transaction before DDL +statements. + +.. + +.. bpo: 17941 +.. date: 9571 +.. nonce: E9rm_o +.. section: Library + +Add a *module* parameter to collections.namedtuple(). + +.. + +.. bpo: 22493 +.. date: 9570 +.. nonce: yDfUrj +.. section: Library + +Inline flags now should be used only at the start of the regular expression. +Deprecation warning is emitted if uses them in the middle of the regular +expression. + +.. + +.. bpo: 26885 +.. date: 9569 +.. nonce: TJ779X +.. section: Library + +xmlrpc now supports unmarshalling additional data types used by Apache XML- +RPC implementation for numerics and None. + +.. + +.. bpo: 28070 +.. date: 9568 +.. nonce: Kot8Hu +.. section: Library + +Fixed parsing inline verbose flag in regular expressions. + +.. + +.. bpo: 19500 +.. date: 9567 +.. nonce: H7q5im +.. section: Library + +Add client-side SSL session resumption to the ssl module. + +.. + +.. bpo: 28022 +.. date: 9566 +.. nonce: 08kTMg +.. section: Library + +Deprecate ssl-related arguments in favor of SSLContext. The deprecation +include manual creation of SSLSocket and certfile/keyfile (or similar) in +ftplib, httplib, imaplib, smtplib, poplib and urllib. + +.. + +.. bpo: 28043 +.. date: 9565 +.. nonce: 588Oy3 +.. section: Library + +SSLContext has improved default settings: OP_NO_SSLv2, OP_NO_SSLv3, +OP_NO_COMPRESSION, OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE, +OP_SINGLE_ECDH_USE and HIGH ciphers without MD5. + +.. + +.. bpo: 24693 +.. date: 9564 +.. nonce: a63Shp +.. section: Library + +Changed some RuntimeError's in the zipfile module to more appropriate types. +Improved some error messages and debugging output. + +.. + +.. bpo: 17909 +.. date: 9563 +.. nonce: SMNkt6 +.. section: Library + +``json.load`` and ``json.loads`` now support binary input encoded as UTF-8, +UTF-16 or UTF-32. Patch by Serhiy Storchaka. + +.. + +.. bpo: 27137 +.. date: 9562 +.. nonce: frjG8W +.. section: Library + +the pure Python fallback implementation of ``functools.partial`` now matches +the behaviour of its accelerated C counterpart for subclassing, pickling and +text representation purposes. Patch by Emanuel Barry and Serhiy Storchaka. + +.. + +.. bpo: 0 +.. date: 9561 +.. nonce: 81jNns +.. section: Library + +Fix possible integer overflows and crashes in the mmap module with unusual +usage patterns. + +.. + +.. bpo: 1703178 +.. date: 9560 +.. nonce: meb49K +.. section: Library + +Fix the ability to pass the --link-objects option to the distutils build_ext +command. + +.. + +.. bpo: 28019 +.. date: 9559 +.. nonce: KUhBaS +.. section: Library + +itertools.count() no longer rounds non-integer step in range between 1.0 and +2.0 to 1. + +.. + +.. bpo: 18401 +.. date: 9558 +.. nonce: _12WDV +.. section: Library + +Pdb now supports the 'readrc' keyword argument to control whether .pdbrc +files should be read. Patch by Martin Matusiak and Sam Kimbrel. + +.. + +.. bpo: 25969 +.. date: 9557 +.. nonce: qSPkl- +.. section: Library + +Update the lib2to3 grammar to handle the unpacking generalizations added in +3.5. + +.. + +.. bpo: 14977 +.. date: 9556 +.. nonce: 4MvALg +.. section: Library + +mailcap now respects the order of the lines in the mailcap files ("first +match"), as required by RFC 1542. Patch by Michael Lazar. + +.. + +.. bpo: 28082 +.. date: 9555 +.. nonce: EICw4d +.. section: Library + +Convert re flag constants to IntFlag. + +.. + +.. bpo: 28025 +.. date: 9554 +.. nonce: YxcZHY +.. section: Library + +Convert all ssl module constants to IntEnum and IntFlags. SSLContext +properties now return flags and enums. + +.. + +.. bpo: 23591 +.. date: 9553 +.. nonce: 7gSXAN +.. section: Library + +Add Flag, IntFlag, and auto() to enum module. + +.. + +.. bpo: 433028 +.. date: 9552 +.. nonce: yGjT0q +.. section: Library + +Added support of modifier spans in regular expressions. + +.. + +.. bpo: 24594 +.. date: 9551 +.. nonce: 9CnFVS +.. section: Library + +Validates persist parameter when opening MSI database + +.. + +.. bpo: 17582 +.. date: 9550 +.. nonce: MXEHxQ +.. section: Library + +xml.etree.ElementTree nows preserves whitespaces in attributes (Patch by +Duane Griffin. Reviewed and approved by Stefan Behnel.) + +.. + +.. bpo: 28047 +.. date: 9549 +.. nonce: pDu3Fm +.. section: Library + +Fixed calculation of line length used for the base64 CTE in the new email +policies. + +.. + +.. bpo: 27576 +.. date: 9548 +.. nonce: tqZxYv +.. section: Library + +Fix call order in OrderedDict.__init__(). + +.. + +.. bpo: 0 +.. date: 9547 +.. nonce: cxHuUo +.. section: Library + +email.generator.DecodedGenerator now supports the policy keyword. + +.. + +.. bpo: 28027 +.. date: 9546 +.. nonce: v39s1z +.. section: Library + +Remove undocumented modules from ``Lib/plat-*``: IN, CDROM, DLFCN, TYPES, +CDIO, and STROPTS. + +.. + +.. bpo: 27445 +.. date: 9545 +.. nonce: wOG0C0 +.. section: Library + +Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. + +.. + +.. bpo: 24277 +.. date: 9544 +.. nonce: OgDA28 +.. section: Library + +The new email API is no longer provisional, and the docs have been +reorganized and rewritten to emphasize the new API. + +.. + +.. bpo: 22450 +.. date: 9543 +.. nonce: T3Sn_J +.. section: Library + +urllib now includes an ``Accept: */*`` header among the default headers. +This makes the results of REST API requests more consistent and predictable +especially when proxy servers are involved. + +.. + +.. bpo: 0 +.. date: 9542 +.. nonce: PVZStR +.. section: Library + +lib2to3.pgen3.driver.load_grammar() now creates a stable cache file between +runs given the same Grammar.txt input regardless of the hash randomization +setting. + +.. + +.. bpo: 28005 +.. date: 9541 +.. nonce: oJLK1w +.. section: Library + +Allow ImportErrors in encoding implementation to propagate. + +.. + +.. bpo: 26667 +.. date: 9540 +.. nonce: hWs9wA +.. section: Library + +Support path-like objects in importlib.util. + +.. + +.. bpo: 27570 +.. date: 9539 +.. nonce: pU0Zie +.. section: Library + +Avoid zero-length memcpy() etc calls with null source pointers in the +"ctypes" and "array" modules. + +.. + +.. bpo: 22233 +.. date: 9538 +.. nonce: uXSN0R +.. section: Library + +Break email header lines *only* on the RFC specified CR and LF characters, +not on arbitrary unicode line breaks. This also fixes a bug in HTTP header +parsing. + +.. + +.. bpo: 27331 +.. date: 9537 +.. nonce: akOxfh +.. section: Library + +The email.mime classes now all accept an optional policy keyword. + +.. + +.. bpo: 27988 +.. date: 9536 +.. nonce: VfMzZH +.. section: Library + +Fix email iter_attachments incorrect mutation of payload list. + +.. + +.. bpo: 16113 +.. date: 9535 +.. nonce: jyKRxs +.. section: Library + +Add SHA-3 and SHAKE support to hashlib module. + +.. + +.. bpo: 0 +.. date: 9534 +.. nonce: j7npJi +.. section: Library + +Eliminate a tautological-pointer-compare warning in _scproxy.c. + +.. + +.. bpo: 27776 +.. date: 9533 +.. nonce: dOJcUU +.. section: Library + +The :func:`os.urandom` function does now block on Linux 3.17 and newer until +the system urandom entropy pool is initialized to increase the security. +This change is part of the :pep:`524`. + +.. + +.. bpo: 27778 +.. date: 9532 +.. nonce: gvbf3F +.. section: Library + +Expose the Linux ``getrandom()`` syscall as a new :func:`os.getrandom` +function. This change is part of the :pep:`524`. + +.. + +.. bpo: 27691 +.. date: 9531 +.. nonce: TMYF5_ +.. section: Library + +Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 +certs. + +.. + +.. bpo: 18844 +.. date: 9530 +.. nonce: OZnLOi +.. section: Library + +Add random.choices(). + +.. + +.. bpo: 25761 +.. date: 9529 +.. nonce: qd--Ta +.. section: Library + +Improved error reporting about truncated pickle data in C implementation of +unpickler. UnpicklingError is now raised instead of AttributeError and +ValueError in some cases. + +.. + +.. bpo: 26798 +.. date: 9528 +.. nonce: he58yl +.. section: Library + +Add BLAKE2 (blake2b and blake2s) to hashlib. + +.. + +.. bpo: 26032 +.. date: 9527 +.. nonce: v5ByZW +.. section: Library + +Optimized globbing in pathlib by using os.scandir(); it is now about 1.5--4 +times faster. + +.. + +.. bpo: 25596 +.. date: 9526 +.. nonce: TFtyjC +.. section: Library + +Optimized glob() and iglob() functions in the glob module; they are now +about 3--6 times faster. + +.. + +.. bpo: 27928 +.. date: 9525 +.. nonce: vG2f6q +.. section: Library + +Add scrypt (password-based key derivation function) to hashlib module +(requires OpenSSL 1.1.0). + +.. + +.. bpo: 27850 +.. date: 9524 +.. nonce: kIVQ0m +.. section: Library + +Remove 3DES from ssl module's default cipher list to counter measure sweet32 +attack (CVE-2016-2183). + +.. + +.. bpo: 27766 +.. date: 9523 +.. nonce: WI70Tc +.. section: Library + +Add ChaCha20 Poly1305 to ssl module's default ciper list. (Required OpenSSL +1.1.0 or LibreSSL). + +.. + +.. bpo: 25387 +.. date: 9522 +.. nonce: -wsV59 +.. section: Library + +Check return value of winsound.MessageBeep. + +.. + +.. bpo: 27866 +.. date: 9521 +.. nonce: FM3-BZ +.. section: Library + +Add SSLContext.get_ciphers() method to get a list of all enabled ciphers. + +.. + +.. bpo: 27744 +.. date: 9520 +.. nonce: 2cVMpG +.. section: Library + +Add AF_ALG (Linux Kernel crypto) to socket module. + +.. + +.. bpo: 26470 +.. date: 9519 +.. nonce: QGu_wo +.. section: Library + +Port ssl and hashlib module to OpenSSL 1.1.0. + +.. + +.. bpo: 11620 +.. date: 9518 +.. nonce: JyL-Po +.. section: Library + +Fix support for SND_MEMORY in winsound.PlaySound. Based on a patch by Tim +Lesher. + +.. + +.. bpo: 11734 +.. date: 9517 +.. nonce: AQoy-q +.. section: Library + +Add support for IEEE 754 half-precision floats to the struct module. Based +on a patch by Eli Stevens. + +.. + +.. bpo: 27919 +.. date: 9516 +.. nonce: NRqNEW +.. section: Library + +Deprecated ``extra_path`` distribution option in distutils packaging. + +.. + +.. bpo: 23229 +.. date: 9515 +.. nonce: gXhSFh +.. section: Library + +Add new ``cmath`` constants: ``cmath.inf`` and ``cmath.nan`` to match +``math.inf`` and ``math.nan``, and also ``cmath.infj`` and ``cmath.nanj`` to +match the format used by complex repr. + +.. + +.. bpo: 27842 +.. date: 9514 +.. nonce: qlhp0- +.. section: Library + +The csv.DictReader now returns rows of type OrderedDict. (Contributed by +Steve Holden.) + +.. + +.. bpo: 0 +.. date: 9513 +.. nonce: 6TjEgz +.. section: Library + +Remove support for passing a file descriptor to os.access. It never worked +but previously didn't raise. + +.. + +.. bpo: 12885 +.. date: 9512 +.. nonce: r-IV1g +.. section: Library + +Fix error when distutils encounters symlink. + +.. + +.. bpo: 27881 +.. date: 9511 +.. nonce: fkETd9 +.. section: Library + +Fixed possible bugs when setting sqlite3.Connection.isolation_level. Based +on patch by Xiang Zhang. + +.. + +.. bpo: 27861 +.. date: 9510 +.. nonce: DBYuo9 +.. section: Library + +Fixed a crash in sqlite3.Connection.cursor() when a factory creates not a +cursor. Patch by Xiang Zhang. + +.. + +.. bpo: 19884 +.. date: 9509 +.. nonce: MO8AWH +.. section: Library + +Avoid spurious output on OS X with Gnu Readline. + +.. + +.. bpo: 27706 +.. date: 9508 +.. nonce: ZY67yu +.. section: Library + +Restore deterministic behavior of random.Random().seed() for string seeds +using seeding version 1. Allows sequences of calls to random() to exactly +match those obtained in Python 2. Patch by Nofar Schnider. + +.. + +.. bpo: 10513 +.. date: 9507 +.. nonce: tQIQD_ +.. section: Library + +Fix a regression in Connection.commit(). Statements should not be reset +after a commit. + +.. + +.. bpo: 12319 +.. date: 9506 +.. nonce: Wc4oUu +.. section: Library + +Chunked transfer encoding support added to http.client.HTTPConnection +requests. The urllib.request.AbstractHTTPHandler class does not enforce a +Content-Length header any more. If a HTTP request has a file or iterable +body, but no Content-Length header, the library now falls back to use +chunked transfer- encoding. + +.. + +.. bpo: 0 +.. date: 9505 +.. nonce: cYraeH +.. section: Library + +A new version of typing.py from https://github.com/python/typing: - +Collection (only for 3.6) (Issue #27598) - Add FrozenSet to __all__ +(upstream #261) - fix crash in _get_type_vars() (upstream #259) - Remove the +dict constraint in ForwardRef._eval_type (upstream #252) + +.. + +.. bpo: 27832 +.. date: 9504 +.. nonce: hxh6_h +.. section: Library + +Make ``_normalize`` parameter to ``Fraction`` constuctor keyword-only, so +that ``Fraction(2, 3, 4)`` now raises ``TypeError``. + +.. + +.. bpo: 27539 +.. date: 9503 +.. nonce: S4L1cq +.. section: Library + +Fix unnormalised ``Fraction.__pow__`` result in the case of negative +exponent and negative base. + +.. + +.. bpo: 21718 +.. date: 9502 +.. nonce: FUJd-7 +.. section: Library + +cursor.description is now available for queries using CTEs. + +.. + +.. bpo: 27819 +.. date: 9501 +.. nonce: -A_u1x +.. section: Library + +In distutils sdists, simply produce the "gztar" (gzipped tar format) +distributions on all platforms unless "formats" is supplied. + +.. + +.. bpo: 2466 +.. date: 9500 +.. nonce: VRNlkg +.. section: Library + +posixpath.ismount now correctly recognizes mount points which the user does +not have permission to access. + +.. + +.. bpo: 9998 +.. date: 9499 +.. nonce: SNIoPr +.. section: Library + +On Linux, ctypes.util.find_library now looks in LD_LIBRARY_PATH for shared +libraries. + +.. + +.. bpo: 27573 +.. date: 9498 +.. nonce: yuXLnW +.. section: Library + +exit message for code.interact is now configurable. + +.. + +.. bpo: 27930 +.. date: 9497 +.. nonce: BkOfSi +.. section: Library + +Improved behaviour of logging.handlers.QueueListener. Thanks to Paulo +Andrade and Petr Viktorin for the analysis and patch. + +.. + +.. bpo: 6766 +.. date: 9496 +.. nonce: _zO4cV +.. section: Library + +Distributed reference counting added to multiprocessing to support nesting +of shared values / proxy objects. + +.. + +.. bpo: 21201 +.. date: 9495 +.. nonce: wLCKiA +.. section: Library + +Improves readability of multiprocessing error message. Thanks to Wojciech +Walczak for patch. + +.. + +.. bpo: 0 +.. date: 9494 +.. nonce: hgCs-W +.. section: Library + +asyncio: Add set_protocol / get_protocol to Transports. + +.. + +.. bpo: 27456 +.. date: 9493 +.. nonce: lI_IE7 +.. section: Library + +asyncio: Set TCP_NODELAY by default. + +.. + +.. bpo: 15308 +.. date: 9492 +.. nonce: zZxn8m +.. section: IDLE + +Add 'interrupt execution' (^C) to Shell menu. Patch by Roger Serwy, updated +by Bayard Randel. + +.. + +.. bpo: 27922 +.. date: 9491 +.. nonce: UEtEv9 +.. section: IDLE + +Stop IDLE tests from 'flashing' gui widgets on the screen. + +.. + +.. bpo: 27891 +.. date: 9490 +.. nonce: 7W5cAj +.. section: IDLE + +Consistently group and sort imports within idlelib modules. + +.. + +.. bpo: 17642 +.. date: 9489 +.. nonce: B0BNOB +.. section: IDLE + +add larger font sizes for classroom projection. + +.. + +.. bpo: 0 +.. date: 9488 +.. nonce: zWZs6o +.. section: IDLE + +Add version to title of IDLE help window. + +.. + +.. bpo: 25564 +.. date: 9487 +.. nonce: GN0p14 +.. section: IDLE + +In section on IDLE -- console differences, mention that using exec means +that __builtins__ is defined for each statement. + +.. + +.. bpo: 27821 +.. date: 9486 +.. nonce: Vzr42u +.. section: IDLE + +Fix 3.6.0a3 regression that prevented custom key sets from being selected +when no custom theme was defined. + +.. + +.. bpo: 26900 +.. date: 9485 +.. nonce: 0erSIc +.. section: C API + +Excluded underscored names and other private API from limited API. + +.. + +.. bpo: 26027 +.. date: 9484 +.. nonce: 5uVb7n +.. section: C API + +Add support for path-like objects in PyUnicode_FSConverter() & +PyUnicode_FSDecoder(). + +.. + +.. bpo: 27427 +.. date: 9483 +.. nonce: OGhkYQ +.. section: Tests + +Additional tests for the math module. Patch by Francisco Couzo. + +.. + +.. bpo: 27953 +.. date: 9482 +.. nonce: oP3nuf +.. section: Tests + +Skip math and cmath tests that fail on OS X 10.4 due to a poor libm +implementation of tan. + +.. + +.. bpo: 26040 +.. date: 9481 +.. nonce: RvSU5I +.. section: Tests + +Improve test_math and test_cmath coverage and rigour. Patch by Jeff Allen. + +.. + +.. bpo: 27787 +.. date: 9480 +.. nonce: kf0YAt +.. section: Tests + +Call gc.collect() before checking each test for "dangling threads", since +the dangling threads are weak references. + +.. + +.. bpo: 27566 +.. date: 9479 +.. nonce: xDWjEb +.. section: Build + +Fix clean target in freeze makefile (patch by Lisa Roach) + +.. + +.. bpo: 27705 +.. date: 9478 +.. nonce: 8C2Ms3 +.. section: Build + +Update message in validate_ucrtbase.py + +.. + +.. bpo: 27976 +.. date: 9477 +.. nonce: z0CT-3 +.. section: Build + +Deprecate building _ctypes with the bundled copy of libffi on non-OSX UNIX +platforms. + +.. + +.. bpo: 27983 +.. date: 9476 +.. nonce: jL_1n8 +.. section: Build + +Cause lack of llvm-profdata tool when using clang as required for PGO +linking to be a configure time error rather than make time when --with- +optimizations is enabled. Also improve our ability to find the llvm- +profdata tool on MacOS and some Linuxes. + +.. + +.. bpo: 21590 +.. date: 9475 +.. nonce: haPolL +.. section: Build + +Support for DTrace and SystemTap probes. + +.. + +.. bpo: 26307 +.. date: 9474 +.. nonce: Puk2rd +.. section: Build + +The profile-opt build now applies PGO to the built-in modules. + +.. + +.. bpo: 26359 +.. date: 9473 +.. nonce: uxKCqR +.. section: Build + +Add the --with-optimizations flag to turn on LTO and PGO build support when +available. + +.. + +.. bpo: 27917 +.. date: 9472 +.. nonce: 8V2esX +.. section: Build + +Set platform triplets for Android builds. + +.. + +.. bpo: 25825 +.. date: 9471 +.. nonce: PwGiUI +.. section: Build + +Update references to the $(LIBPL) installation path on AIX. This path was +changed in 3.2a4. + +.. + +.. bpo: 0 +.. date: 9470 +.. nonce: G27B6T +.. section: Build + +Update OS X installer to use SQLite 3.14.1 and XZ 5.2.2. + +.. + +.. bpo: 21122 +.. date: 9469 +.. nonce: 98ovv8 +.. section: Build + +Fix LTO builds on OS X. + +.. + +.. bpo: 17128 +.. date: 9468 +.. nonce: jd3Cll +.. section: Build + +Build OS X installer with a private copy of OpenSSL. Also provide a sample +Install Certificates command script to install a set of root certificates +from the third-party certifi module. + +.. + +.. bpo: 27952 +.. date: 9467 +.. nonce: WX9Ufc +.. section: Tools/Demos + +Get Tools/scripts/fixcid.py working with Python 3 and the current "re" +module, avoid invalid Python backslash escapes, and fix a bug parsing +escaped C quote signs. + +.. + +.. bpo: 28065 +.. date: 9466 +.. nonce: TUW63o +.. section: Windows + +Update xz dependency to 5.2.2 and build it from source. + +.. + +.. bpo: 25144 +.. date: 9465 +.. nonce: iUha52 +.. section: Windows + +Ensures TargetDir is set before continuing with custom install. + +.. + +.. bpo: 1602 +.. date: 9464 +.. nonce: 5Kowx0 +.. section: Windows + +Windows console doesn't input or print Unicode (PEP 528) + +.. + +.. bpo: 27781 +.. date: 9463 +.. nonce: 21eQH2 +.. section: Windows + +Change file system encoding on Windows to UTF-8 (PEP 529) + +.. + +.. bpo: 27731 +.. date: 9462 +.. nonce: U2HSrC +.. section: Windows + +Opt-out of MAX_PATH on Windows 10 + +.. + +.. bpo: 6135 +.. date: 9461 +.. nonce: pACuPJ +.. section: Windows + +Adds encoding and errors parameters to subprocess. + +.. + +.. bpo: 27959 +.. date: 9460 +.. nonce: JamSoC +.. section: Windows + +Adds oem encoding, alias ansi to mbcs, move aliasmbcs to codec lookup. + +.. + +.. bpo: 27982 +.. date: 9459 +.. nonce: xrUa9R +.. section: Windows + +The functions of the winsound module now accept keyword arguments. + +.. + +.. bpo: 20366 +.. date: 9458 +.. nonce: s6b-ut +.. section: Windows + +Build full text search support into SQLite on Windows. + +.. + +.. bpo: 27756 +.. date: 9457 +.. nonce: PDAoGy +.. section: Windows + +Adds new icons for Python files and processes on Windows. Designs by Cherry +Wang. + +.. + +.. bpo: 27883 +.. date: 9456 +.. nonce: vyOnxj +.. section: Windows + +Update sqlite to 3.14.1.0 on Windows. diff --git a/Misc/NEWS.d/3.6.0b2.rst b/Misc/NEWS.d/3.6.0b2.rst new file mode 100644 index 00000000000000..ebf4a96e635541 --- /dev/null +++ b/Misc/NEWS.d/3.6.0b2.rst @@ -0,0 +1,840 @@ +.. bpo: 28183 +.. date: 9707 +.. nonce: MJZeNd +.. release date: 2016-10-10 +.. section: Core and Builtins + +Optimize and cleanup dict iteration. + +.. + +.. bpo: 26081 +.. date: 9706 +.. nonce: _x5vjl +.. section: Core and Builtins + +Added C implementation of asyncio.Future. Original patch by Yury Selivanov. + +.. + +.. bpo: 28379 +.. date: 9705 +.. nonce: DuXlco +.. section: Core and Builtins + +Added sanity checks and tests for PyUnicode_CopyCharacters(). Patch by Xiang +Zhang. + +.. + +.. bpo: 28376 +.. date: 9704 +.. nonce: oPD-5D +.. section: Core and Builtins + +The type of long range iterator is now registered as Iterator. Patch by Oren +Milman. + +.. + +.. bpo: 28376 +.. date: 9703 +.. nonce: YEy-uG +.. section: Core and Builtins + +Creating instances of range_iterator by calling range_iterator type now is +deprecated. Patch by Oren Milman. + +.. + +.. bpo: 28376 +.. date: 9702 +.. nonce: fLeHM2 +.. section: Core and Builtins + +The constructor of range_iterator now checks that step is not 0. Patch by +Oren Milman. + +.. + +.. bpo: 26906 +.. date: 9701 +.. nonce: YBjcwI +.. section: Core and Builtins + +Resolving special methods of uninitialized type now causes implicit +initialization of the type instead of a fail. + +.. + +.. bpo: 18287 +.. date: 9700 +.. nonce: k6jffS +.. section: Core and Builtins + +PyType_Ready() now checks that tp_name is not NULL. Original patch by Niklas +Koep. + +.. + +.. bpo: 24098 +.. date: 9699 +.. nonce: XqlP_1 +.. section: Core and Builtins + +Fixed possible crash when AST is changed in process of compiling it. + +.. + +.. bpo: 28201 +.. date: 9698 +.. nonce: GWUxAy +.. section: Core and Builtins + +Dict reduces possibility of 2nd conflict in hash table when hashes have same +lower bits. + +.. + +.. bpo: 28350 +.. date: 9697 +.. nonce: 8M5Eg9 +.. section: Core and Builtins + +String constants with null character no longer interned. + +.. + +.. bpo: 26617 +.. date: 9696 +.. nonce: Gh5LvN +.. section: Core and Builtins + +Fix crash when GC runs during weakref callbacks. + +.. + +.. bpo: 27942 +.. date: 9695 +.. nonce: ZGuhns +.. section: Core and Builtins + +String constants now interned recursively in tuples and frozensets. + +.. + +.. bpo: 21578 +.. date: 9694 +.. nonce: GI1bhj +.. section: Core and Builtins + +Fixed misleading error message when ImportError called with invalid keyword +args. + +.. + +.. bpo: 28203 +.. date: 9693 +.. nonce: LRn5vp +.. section: Core and Builtins + +Fix incorrect type in complex(1.0, {2:3}) error message. Patch by Soumya +Sharma. + +.. + +.. bpo: 28086 +.. date: 9692 +.. nonce: JsQPMQ +.. section: Core and Builtins + +Single var-positional argument of tuple subtype was passed unscathed to the +C-defined function. Now it is converted to exact tuple. + +.. + +.. bpo: 28214 +.. date: 9691 +.. nonce: zQF8Em +.. section: Core and Builtins + +Now __set_name__ is looked up on the class instead of the instance. + +.. + +.. bpo: 27955 +.. date: 9690 +.. nonce: HC4pZ4 +.. section: Core and Builtins + +Fallback on reading /dev/urandom device when the getrandom() syscall fails +with EPERM, for example when blocked by SECCOMP. + +.. + +.. bpo: 28192 +.. date: 9689 +.. nonce: eR6stU +.. section: Core and Builtins + +Don't import readline in isolated mode. + +.. + +.. bpo: 0 +.. date: 9688 +.. nonce: 9EbOiD +.. section: Core and Builtins + +Upgrade internal unicode databases to Unicode version 9.0.0. + +.. + +.. bpo: 28131 +.. date: 9687 +.. nonce: owq0wW +.. section: Core and Builtins + +Fix a regression in zipimport's compile_source(). zipimport should use the +same optimization level as the interpreter. + +.. + +.. bpo: 28126 +.. date: 9686 +.. nonce: Qf6-uQ +.. section: Core and Builtins + +Replace Py_MEMCPY with memcpy(). Visual Studio can properly optimize +memcpy(). + +.. + +.. bpo: 28120 +.. date: 9685 +.. nonce: e5xc1i +.. section: Core and Builtins + +Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +(Not yet inserted in split-table). Patch by Xiang Zhang. + +.. + +.. bpo: 26182 +.. date: 9684 +.. nonce: jYlqTO +.. section: Core and Builtins + +Raise DeprecationWarning when async and await keywords are used as +variable/attribute/class/function name. + +.. + +.. bpo: 27998 +.. date: 9683 +.. nonce: CPhy4H +.. section: Library + +Fixed bytes path support in os.scandir() on Windows. Patch by Eryk Sun. + +.. + +.. bpo: 28317 +.. date: 9682 +.. nonce: LgHleA +.. section: Library + +The disassembler now decodes FORMAT_VALUE argument. + +.. + +.. bpo: 26293 +.. date: 9681 +.. nonce: 2mjvwX +.. section: Library + +Fixed writing ZIP files that starts not from the start of the file. Offsets +in ZIP file now are relative to the start of the archive in conforming to +the specification. + +.. + +.. bpo: 28380 +.. date: 9680 +.. nonce: jKPMzH +.. section: Library + +unittest.mock Mock autospec functions now properly support assert_called, +assert_not_called, and assert_called_once. + +.. + +.. bpo: 27181 +.. date: 9679 +.. nonce: SQyDpC +.. section: Library + +remove statistics.geometric_mean and defer until 3.7. + +.. + +.. bpo: 28229 +.. date: 9678 +.. nonce: BKAxcS +.. section: Library + +lzma module now supports pathlib. + +.. + +.. bpo: 28321 +.. date: 9677 +.. nonce: bQ-IIX +.. section: Library + +Fixed writing non-BMP characters with binary format in plistlib. + +.. + +.. bpo: 28225 +.. date: 9676 +.. nonce: 6N28nu +.. section: Library + +bz2 module now supports pathlib. Initial patch by Ethan Furman. + +.. + +.. bpo: 28227 +.. date: 9675 +.. nonce: 7lUz8i +.. section: Library + +gzip now supports pathlib. Patch by Ethan Furman. + +.. + +.. bpo: 27358 +.. date: 9674 +.. nonce: t288Iv +.. section: Library + +Optimized merging var-keyword arguments and improved error message when +passing a non-mapping as a var-keyword argument. + +.. + +.. bpo: 28257 +.. date: 9673 +.. nonce: SVD_IH +.. section: Library + +Improved error message when passing a non-iterable as a var-positional +argument. Added opcode BUILD_TUPLE_UNPACK_WITH_CALL. + +.. + +.. bpo: 28322 +.. date: 9672 +.. nonce: l9hzap +.. section: Library + +Fixed possible crashes when unpickle itertools objects from incorrect pickle +data. Based on patch by John Leitch. + +.. + +.. bpo: 28228 +.. date: 9671 +.. nonce: 1qBwdM +.. section: Library + +imghdr now supports pathlib. + +.. + +.. bpo: 28226 +.. date: 9670 +.. nonce: nMXiwU +.. section: Library + +compileall now supports pathlib. + +.. + +.. bpo: 28314 +.. date: 9669 +.. nonce: N7YrkN +.. section: Library + +Fix function declaration (C flags) for the getiterator() method of +xml.etree.ElementTree.Element. + +.. + +.. bpo: 28148 +.. date: 9668 +.. nonce: Flzndx +.. section: Library + +Stop using localtime() and gmtime() in the time module. + +Introduced platform independent _PyTime_localtime API that is similar to +POSIX localtime_r, but available on all platforms. Patch by Ed Schouten. + +.. + +.. bpo: 28253 +.. date: 9667 +.. nonce: aLfmhe +.. section: Library + +Fixed calendar functions for extreme months: 0001-01 and 9999-12. + +Methods itermonthdays() and itermonthdays2() are reimplemented so that they +don't call itermonthdates() which can cause datetime.date under/overflow. + +.. + +.. bpo: 28275 +.. date: 9666 +.. nonce: EhWIsz +.. section: Library + +Fixed possible use after free in the decompress() methods of the +LZMADecompressor and BZ2Decompressor classes. Original patch by John Leitch. + +.. + +.. bpo: 27897 +.. date: 9665 +.. nonce: I0Ppmx +.. section: Library + +Fixed possible crash in sqlite3.Connection.create_collation() if pass +invalid string-like object as a name. Patch by Xiang Zhang. + +.. + +.. bpo: 18844 +.. date: 9664 +.. nonce: fQsEdn +.. section: Library + +random.choices() now has k as a keyword-only argument to improve the +readability of common cases and come into line with the signature used in +other languages. + +.. + +.. bpo: 18893 +.. date: 9663 +.. nonce: osiX5c +.. section: Library + +Fix invalid exception handling in Lib/ctypes/macholib/dyld.py. Patch by +Madison May. + +.. + +.. bpo: 27611 +.. date: 9662 +.. nonce: A_ArH_ +.. section: Library + +Fixed support of default root window in the tkinter.tix module. Added the +master parameter in the DisplayStyle constructor. + +.. + +.. bpo: 27348 +.. date: 9661 +.. nonce: tDx7Vw +.. section: Library + +In the traceback module, restore the formatting of exception messages like +"Exception: None". This fixes a regression introduced in 3.5a2. + +.. + +.. bpo: 25651 +.. date: 9660 +.. nonce: 3UhyPo +.. section: Library + +Allow falsy values to be used for msg parameter of subTest(). + +.. + +.. bpo: 27778 +.. date: 9659 +.. nonce: Yyo1aP +.. section: Library + +Fix a memory leak in os.getrandom() when the getrandom() is interrupted by a +signal and a signal handler raises a Python exception. + +.. + +.. bpo: 28200 +.. date: 9658 +.. nonce: 4IEbr7 +.. section: Library + +Fix memory leak on Windows in the os module (fix path_converter() function). + +.. + +.. bpo: 25400 +.. date: 9657 +.. nonce: d9Qn0E +.. section: Library + +RobotFileParser now correctly returns default values for crawl_delay and +request_rate. Initial patch by Peter Wirtz. + +.. + +.. bpo: 27932 +.. date: 9656 +.. nonce: mtgl-6 +.. section: Library + +Prevent memory leak in win32_ver(). + +.. + +.. bpo: 0 +.. date: 9655 +.. nonce: iPpjqX +.. section: Library + +Fix UnboundLocalError in socket._sendfile_use_sendfile. + +.. + +.. bpo: 28075 +.. date: 9654 +.. nonce: aLiUs9 +.. section: Library + +Check for ERROR_ACCESS_DENIED in Windows implementation of os.stat(). Patch +by Eryk Sun. + +.. + +.. bpo: 22493 +.. date: 9653 +.. nonce: Mv_hZf +.. section: Library + +Warning message emitted by using inline flags in the middle of regular +expression now contains a (truncated) regex pattern. Patch by Tim Graham. + +.. + +.. bpo: 25270 +.. date: 9652 +.. nonce: jrZruM +.. section: Library + +Prevent codecs.escape_encode() from raising SystemError when an empty +bytestring is passed. + +.. + +.. bpo: 28181 +.. date: 9651 +.. nonce: NGc4Yv +.. section: Library + +Get antigravity over HTTPS. Patch by Kaartic Sivaraam. + +.. + +.. bpo: 25895 +.. date: 9650 +.. nonce: j92qoQ +.. section: Library + +Enable WebSocket URL schemes in urllib.parse.urljoin. Patch by Gergely Imreh +and Markus Holtermann. + +.. + +.. bpo: 28114 +.. date: 9649 +.. nonce: gmFXsA +.. section: Library + +Fix a crash in parse_envlist() when env contains byte strings. Patch by Eryk +Sun. + +.. + +.. bpo: 27599 +.. date: 9648 +.. nonce: itvm8T +.. section: Library + +Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). + +.. + +.. bpo: 27906 +.. date: 9647 +.. nonce: TBBXrv +.. section: Library + +Fix socket accept exhaustion during high TCP traffic. Patch by Kevin Conway. + +.. + +.. bpo: 28174 +.. date: 9646 +.. nonce: CV1UdI +.. section: Library + +Handle when SO_REUSEPORT isn't properly supported. Patch by Seth Michael +Larson. + +.. + +.. bpo: 26654 +.. date: 9645 +.. nonce: XtzTE9 +.. section: Library + +Inspect functools.partial in asyncio.Handle.__repr__. Patch by iceboy. + +.. + +.. bpo: 26909 +.. date: 9644 +.. nonce: ASiakT +.. section: Library + +Fix slow pipes IO in asyncio. Patch by INADA Naoki. + +.. + +.. bpo: 28176 +.. date: 9643 +.. nonce: sU8R6L +.. section: Library + +Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +.. + +.. bpo: 27759 +.. date: 9642 +.. nonce: qpMDGq +.. section: Library + +Fix selectors incorrectly retain invalid file descriptors. Patch by Mark +Williams. + +.. + +.. bpo: 28368 +.. date: 9641 +.. nonce: fGl9y4 +.. section: Library + +Refuse monitoring processes if the child watcher has no loop attached. Patch +by Vincent Michel. + +.. + +.. bpo: 28369 +.. date: 9640 +.. nonce: 8DTANe +.. section: Library + +Raise RuntimeError when transport's FD is used with add_reader, add_writer, +etc. + +.. + +.. bpo: 28370 +.. date: 9639 +.. nonce: 18jBuZ +.. section: Library + +Speedup asyncio.StreamReader.readexactly. Patch by Коренберг Марк. + +.. + +.. bpo: 28371 +.. date: 9638 +.. nonce: U9Zqdk +.. section: Library + +Deprecate passing asyncio.Handles to run_in_executor. + +.. + +.. bpo: 28372 +.. date: 9637 +.. nonce: njcIPk +.. section: Library + +Fix asyncio to support formatting of non-python coroutines. + +.. + +.. bpo: 28399 +.. date: 9636 +.. nonce: QKIqRX +.. section: Library + +Remove UNIX socket from FS before binding. Patch by Коренберг Марк. + +.. + +.. bpo: 27972 +.. date: 9635 +.. nonce: ZK-GFm +.. section: Library + +Prohibit Tasks to await on themselves. + +.. + +.. bpo: 28402 +.. date: 9634 +.. nonce: v9zETJ +.. section: Windows + +Adds signed catalog files for stdlib on Windows. + +.. + +.. bpo: 28333 +.. date: 9633 +.. nonce: KnpeO4 +.. section: Windows + +Enables Unicode for ps1/ps2 and input() prompts. (Patch by Eryk Sun) + +.. + +.. bpo: 28251 +.. date: 9632 +.. nonce: tR_AFs +.. section: Windows + +Improvements to help manuals on Windows. + +.. + +.. bpo: 28110 +.. date: 9631 +.. nonce: cnkP5F +.. section: Windows + +launcher.msi has different product codes between 32-bit and 64-bit + +.. + +.. bpo: 28161 +.. date: 9630 +.. nonce: hF91LI +.. section: Windows + +Opening CON for write access fails + +.. + +.. bpo: 28162 +.. date: 9629 +.. nonce: 3FHPVD +.. section: Windows + +WindowsConsoleIO readall() fails if first line starts with Ctrl+Z + +.. + +.. bpo: 28163 +.. date: 9628 +.. nonce: -DUgJw +.. section: Windows + +WindowsConsoleIO fileno() passes wrong flags to _open_osfhandle + +.. + +.. bpo: 28164 +.. date: 9627 +.. nonce: 5MfN0J +.. section: Windows + +_PyIO_get_console_type fails for various paths + +.. + +.. bpo: 28137 +.. date: 9626 +.. nonce: C1uvzY +.. section: Windows + +Renames Windows path file to ._pth + +.. + +.. bpo: 28138 +.. date: 9625 +.. nonce: pNdv64 +.. section: Windows + +Windows ._pth file should allow import site + +.. + +.. bpo: 28426 +.. date: 9624 +.. nonce: zPwvbI +.. section: C API + +Deprecated undocumented functions PyUnicode_AsEncodedObject(), +PyUnicode_AsDecodedObject(), PyUnicode_AsDecodedUnicode() and +PyUnicode_AsEncodedUnicode(). + +.. + +.. bpo: 28258 +.. date: 9623 +.. nonce: iKtAHd +.. section: Build + +Fixed build with Estonian locale (python-config and distclean targets in +Makefile). Patch by Arfrever Frehtes Taifersar Arahesis. + +.. + +.. bpo: 26661 +.. date: 9622 +.. nonce: Z_HNbs +.. section: Build + +setup.py now detects system libffi with multiarch wrapper. + +.. + +.. bpo: 15819 +.. date: 9621 +.. nonce: QVDr3E +.. section: Build + +Remove redundant include search directory option for building outside the +source tree. + +.. + +.. bpo: 28217 +.. date: 9620 +.. nonce: Y37OKV +.. section: Tests + +Adds _testconsole module to test console input. diff --git a/Misc/NEWS.d/3.6.0b3.rst b/Misc/NEWS.d/3.6.0b3.rst new file mode 100644 index 00000000000000..4233708aedbd41 --- /dev/null +++ b/Misc/NEWS.d/3.6.0b3.rst @@ -0,0 +1,355 @@ +.. bpo: 28128 +.. date: 9744 +.. nonce: Lc2sFu +.. release date: 2016-10-31 +.. section: Core and Builtins + +Deprecation warning for invalid str and byte escape sequences now prints +better information about where the error occurs. Patch by Serhiy Storchaka +and Eric Smith. + +.. + +.. bpo: 28509 +.. date: 9743 +.. nonce: _Fa4Uq +.. section: Core and Builtins + +dict.update() no longer allocate unnecessary large memory. + +.. + +.. bpo: 28426 +.. date: 9742 +.. nonce: E_quyK +.. section: Core and Builtins + +Fixed potential crash in PyUnicode_AsDecodedObject() in debug build. + +.. + +.. bpo: 28517 +.. date: 9741 +.. nonce: ExPkm9 +.. section: Core and Builtins + +Fixed of-by-one error in the peephole optimizer that caused keeping +unreachable code. + +.. + +.. bpo: 28214 +.. date: 9740 +.. nonce: 6ECJox +.. section: Core and Builtins + +Improved exception reporting for problematic __set_name__ attributes. + +.. + +.. bpo: 23782 +.. date: 9739 +.. nonce: lonDzj +.. section: Core and Builtins + +Fixed possible memory leak in _PyTraceback_Add() and exception loss in +PyTraceBack_Here(). + +.. + +.. bpo: 28471 +.. date: 9738 +.. nonce: Vd5pv7 +.. section: Core and Builtins + +Fix "Python memory allocator called without holding the GIL" crash in +socket.setblocking. + +.. + +.. bpo: 27517 +.. date: 9737 +.. nonce: 1CYM8A +.. section: Library + +LZMA compressor and decompressor no longer raise exceptions if given empty +data twice. Patch by Benjamin Fogle. + +.. + +.. bpo: 28549 +.. date: 9736 +.. nonce: ShnM2y +.. section: Library + +Fixed segfault in curses's addch() with ncurses6. + +.. + +.. bpo: 28449 +.. date: 9735 +.. nonce: 5JK6ES +.. section: Library + +tarfile.open() with mode "r" or "r:" now tries to open a tar file with +compression before trying to open it without compression. Otherwise it had +50% chance failed with ignore_zeros=True. + +.. + +.. bpo: 23262 +.. date: 9734 +.. nonce: 6EVB7N +.. section: Library + +The webbrowser module now supports Firefox 36+ and derived browsers. Based +on patch by Oleg Broytman. + +.. + +.. bpo: 27939 +.. date: 9733 +.. nonce: mTfADV +.. section: Library + +Fixed bugs in tkinter.ttk.LabeledScale and tkinter.Scale caused by +representing the scale as float value internally in Tk. tkinter.IntVar now +works if float value is set to underlying Tk variable. + +.. + +.. bpo: 18844 +.. date: 9732 +.. nonce: oif1-H +.. section: Library + +The various ways of specifying weights for random.choices() now produce the +same result sequences. + +.. + +.. bpo: 28255 +.. date: 9731 +.. nonce: _ZH4wm +.. section: Library + +calendar.TextCalendar().prmonth() no longer prints a space at the start of +new line after printing a month's calendar. Patch by Xiang Zhang. + +.. + +.. bpo: 20491 +.. date: 9730 +.. nonce: ObgnQ2 +.. section: Library + +The textwrap.TextWrapper class now honors non-breaking spaces. Based on +patch by Kaarle Ritvanen. + +.. + +.. bpo: 28353 +.. date: 9729 +.. nonce: sKGbLL +.. section: Library + +os.fwalk() no longer fails on broken links. + +.. + +.. bpo: 28430 +.. date: 9728 +.. nonce: 4MiEYT +.. section: Library + +Fix iterator of C implemented asyncio.Future doesn't accept non-None value +is passed to it.send(val). + +.. + +.. bpo: 27025 +.. date: 9727 +.. nonce: foAViS +.. section: Library + +Generated names for Tkinter widgets now start by the "!" prefix for +readability. + +.. + +.. bpo: 25464 +.. date: 9726 +.. nonce: HDUTCu +.. section: Library + +Fixed HList.header_exists() in tkinter.tix module by addin a workaround to +Tix library bug. + +.. + +.. bpo: 28488 +.. date: 9725 +.. nonce: TgO112 +.. section: Library + +shutil.make_archive() no longer adds entry "./" to ZIP archive. + +.. + +.. bpo: 25953 +.. date: 9724 +.. nonce: EKKJAQ +.. section: Library + +re.sub() now raises an error for invalid numerical group reference in +replacement template even if the pattern is not found in the string. Error +message for invalid group reference now includes the group index and the +position of the reference. Based on patch by SilentGhost. + +.. + +.. bpo: 18219 +.. date: 9723 +.. nonce: 1ANQN1 +.. section: Library + +Optimize csv.DictWriter for large number of columns. Patch by Mariatta +Wijaya. + +.. + +.. bpo: 28448 +.. date: 9722 +.. nonce: 5bduWe +.. section: Library + +Fix C implemented asyncio.Future didn't work on Windows. + +.. + +.. bpo: 28480 +.. date: 9721 +.. nonce: 9lHw6m +.. section: Library + +Fix error building socket module when multithreading is disabled. + +.. + +.. bpo: 24452 +.. date: 9720 +.. nonce: m9Kyg3 +.. section: Library + +Make webbrowser support Chrome on Mac OS X. + +.. + +.. bpo: 20766 +.. date: 9719 +.. nonce: 4kvCzx +.. section: Library + +Fix references leaked by pdb in the handling of SIGINT handlers. + +.. + +.. bpo: 28492 +.. date: 9718 +.. nonce: pFRLQE +.. section: Library + +Fix how StopIteration exception is raised in _asyncio.Future. + +.. + +.. bpo: 28500 +.. date: 9717 +.. nonce: NINKzZ +.. section: Library + +Fix asyncio to handle async gens GC from another thread. + +.. + +.. bpo: 26923 +.. date: 9716 +.. nonce: 8dh3AV +.. section: Library + +Fix asyncio.Gather to refuse being cancelled once all children are done. +Patch by Johannes Ebke. + +.. + +.. bpo: 26796 +.. date: 9715 +.. nonce: TZyAfJ +.. section: Library + +Don't configure the number of workers for default threadpool executor. +Initial patch by Hans Lawrenz. + +.. + +.. bpo: 28544 +.. date: 9714 +.. nonce: KD1oFP +.. section: Library + +Implement asyncio.Task in C. + +.. + +.. bpo: 28522 +.. date: 9713 +.. nonce: XHMQa7 +.. section: Windows + +Fixes mishandled buffer reallocation in getpathp.c + +.. + +.. bpo: 28444 +.. date: 9712 +.. nonce: zkc9nT +.. section: Build + +Fix missing extensions modules when cross compiling. + +.. + +.. bpo: 28208 +.. date: 9711 +.. nonce: DtoP1i +.. section: Build + +Update Windows build and OS X installers to use SQLite 3.14.2. + +.. + +.. bpo: 28248 +.. date: 9710 +.. nonce: KY_-en +.. section: Build + +Update Windows build and OS X installers to use OpenSSL 1.0.2j. + +.. + +.. bpo: 26944 +.. date: 9709 +.. nonce: ChZ_BO +.. section: Tests + +Fix test_posix for Android where 'id -G' is entirely wrong or missing the +effective gid. + +.. + +.. bpo: 28409 +.. date: 9708 +.. nonce: Q2IlxJ +.. section: Tests + +regrtest: fix the parser of command line arguments. diff --git a/Misc/NEWS.d/3.6.0b4.rst b/Misc/NEWS.d/3.6.0b4.rst new file mode 100644 index 00000000000000..c725ffd44b2582 --- /dev/null +++ b/Misc/NEWS.d/3.6.0b4.rst @@ -0,0 +1,327 @@ +.. bpo: 28532 +.. date: 9778 +.. nonce: KEYJny +.. release date: 2016-11-21 +.. section: Core and Builtins + +Show sys.version when -V option is supplied twice. + +.. + +.. bpo: 27100 +.. date: 9777 +.. nonce: poVjXq +.. section: Core and Builtins + +The with-statement now checks for __enter__ before it checks for __exit__. +This gives less confusing error messages when both methods are missing. +Patch by Jonathan Ellington. + +.. + +.. bpo: 28746 +.. date: 9776 +.. nonce: r5MXdB +.. section: Core and Builtins + +Fix the set_inheritable() file descriptor method on platforms that do not +have the ioctl FIOCLEX and FIONCLEX commands. + +.. + +.. bpo: 26920 +.. date: 9775 +.. nonce: 1URwGb +.. section: Core and Builtins + +Fix not getting the locale's charset upon initializing the interpreter, on +platforms that do not have langinfo. + +.. + +.. bpo: 28648 +.. date: 9774 +.. nonce: z7B52W +.. section: Core and Builtins + +Fixed crash in Py_DecodeLocale() in debug build on Mac OS X when decode +astral characters. Patch by Xiang Zhang. + +.. + +.. bpo: 19398 +.. date: 9773 +.. nonce: RYbEGH +.. section: Core and Builtins + +Extra slash no longer added to sys.path components in case of empty compile- +time PYTHONPATH components. + +.. + +.. bpo: 28665 +.. date: 9772 +.. nonce: v4nx86 +.. section: Core and Builtins + +Improve speed of the STORE_DEREF opcode by 40%. + +.. + +.. bpo: 28583 +.. date: 9771 +.. nonce: F-QAx1 +.. section: Core and Builtins + +PyDict_SetDefault didn't combine split table when needed. Patch by Xiang +Zhang. + +.. + +.. bpo: 27243 +.. date: 9770 +.. nonce: 61E6K5 +.. section: Core and Builtins + +Change PendingDeprecationWarning -> DeprecationWarning. As it was agreed in +the issue, __aiter__ returning an awaitable should result in +PendingDeprecationWarning in 3.5 and in DeprecationWarning in 3.6. + +.. + +.. bpo: 26182 +.. date: 9769 +.. nonce: a8JXK2 +.. section: Core and Builtins + +Fix a refleak in code that raises DeprecationWarning. + +.. + +.. bpo: 28721 +.. date: 9768 +.. nonce: BO9BUF +.. section: Core and Builtins + +Fix asynchronous generators aclose() and athrow() to handle +StopAsyncIteration propagation properly. + +.. + +.. bpo: 28752 +.. date: 9767 +.. nonce: Q-4oRE +.. section: Library + +Restored the __reduce__() methods of datetime objects. + +.. + +.. bpo: 28727 +.. date: 9766 +.. nonce: ubZP_b +.. section: Library + +Regular expression patterns, _sre.SRE_Pattern objects created by +re.compile(), become comparable (only x==y and x!=y operators). This change +should fix the issue #18383: don't duplicate warning filters when the +warnings module is reloaded (thing usually only done in unit tests). + +.. + +.. bpo: 20572 +.. date: 9765 +.. nonce: lGXaH9 +.. section: Library + +The subprocess.Popen.wait method's undocumented endtime parameter now raises +a DeprecationWarning. + +.. + +.. bpo: 25659 +.. date: 9764 +.. nonce: lE2IlT +.. section: Library + +In ctypes, prevent a crash calling the from_buffer() and from_buffer_copy() +methods on abstract classes like Array. + +.. + +.. bpo: 19717 +.. date: 9763 +.. nonce: HXCAIz +.. section: Library + +Makes Path.resolve() succeed on paths that do not exist. Patch by Vajrasky +Kok + +.. + +.. bpo: 28563 +.. date: 9762 +.. nonce: iweEiw +.. section: Library + +Fixed possible DoS and arbitrary code execution when handle plural form +selections in the gettext module. The expression parser now supports exact +syntax supported by GNU gettext. + +.. + +.. bpo: 28387 +.. date: 9761 +.. nonce: 1clJu7 +.. section: Library + +Fixed possible crash in _io.TextIOWrapper deallocator when the garbage +collector is invoked in other thread. Based on patch by Sebastian Cufre. + +.. + +.. bpo: 28600 +.. date: 9760 +.. nonce: wMVrjN +.. section: Library + +Optimize loop.call_soon. + +.. + +.. bpo: 28613 +.. date: 9759 +.. nonce: sqUPrv +.. section: Library + +Fix get_event_loop() return the current loop if called from +coroutines/callbacks. + +.. + +.. bpo: 28634 +.. date: 9758 +.. nonce: YlRydz +.. section: Library + +Fix asyncio.isfuture() to support unittest.Mock. + +.. + +.. bpo: 26081 +.. date: 9757 +.. nonce: 2Y8-a9 +.. section: Library + +Fix refleak in _asyncio.Future.__iter__().throw. + +.. + +.. bpo: 28639 +.. date: 9756 +.. nonce: WUPo1o +.. section: Library + +Fix inspect.isawaitable to always return bool Patch by Justin Mayfield. + +.. + +.. bpo: 28652 +.. date: 9755 +.. nonce: f5M8FG +.. section: Library + +Make loop methods reject socket kinds they do not support. + +.. + +.. bpo: 28653 +.. date: 9754 +.. nonce: S5bA9i +.. section: Library + +Fix a refleak in functools.lru_cache. + +.. + +.. bpo: 28703 +.. date: 9753 +.. nonce: CRLTJc +.. section: Library + +Fix asyncio.iscoroutinefunction to handle Mock objects. + +.. + +.. bpo: 28704 +.. date: 9752 +.. nonce: EFWBII +.. section: Library + +Fix create_unix_server to support Path-like objects (PEP 519). + +.. + +.. bpo: 28720 +.. date: 9751 +.. nonce: Fsz-Lf +.. section: Library + +Add collections.abc.AsyncGenerator. + +.. + +.. bpo: 28513 +.. date: 9750 +.. nonce: L3joAz +.. section: Documentation + +Documented command-line interface of zipfile. + +.. + +.. bpo: 28666 +.. date: 9749 +.. nonce: RtTk-4 +.. section: Tests + +Now test.support.rmtree is able to remove unwritable or unreadable +directories. + +.. + +.. bpo: 23839 +.. date: 9748 +.. nonce: zsT_L9 +.. section: Tests + +Various caches now are cleared before running every test file. + +.. + +.. bpo: 10656 +.. date: 9747 +.. nonce: pR8FFU +.. section: Build + +Fix out-of-tree building on AIX. Patch by Tristan Carel and Michael +Haubenwallner. + +.. + +.. bpo: 26359 +.. date: 9746 +.. nonce: CLz6qy +.. section: Build + +Rename --with-optimiations to --enable-optimizations. + +.. + +.. bpo: 28676 +.. date: 9745 +.. nonce: Wxf6Ds +.. section: Build + +Prevent missing 'getentropy' declaration warning on macOS. Patch by Gareth +Rees. diff --git a/Misc/NEWS.d/3.6.0rc1.rst b/Misc/NEWS.d/3.6.0rc1.rst new file mode 100644 index 00000000000000..17220ca6df626f --- /dev/null +++ b/Misc/NEWS.d/3.6.0rc1.rst @@ -0,0 +1,122 @@ +.. bpo: 23722 +.. date: 9790 +.. nonce: e8BH5h +.. release date: 2016-12-06 +.. section: Core and Builtins + +Rather than silently producing a class that doesn't support zero-argument +``super()`` in methods, failing to pass the new ``__classcell__`` namespace +entry up to ``type.__new__`` now results in a ``DeprecationWarning`` and a +class that supports zero-argument ``super()``. + +.. + +.. bpo: 28797 +.. date: 9789 +.. nonce: _A0_Z5 +.. section: Core and Builtins + +Modifying the class __dict__ inside the __set_name__ method of a descriptor +that is used inside that class no longer prevents calling the __set_name__ +method of other descriptors. + +.. + +.. bpo: 28782 +.. date: 9788 +.. nonce: foJV_E +.. section: Core and Builtins + +Fix a bug in the implementation ``yield from`` when checking if the next +instruction is YIELD_FROM. Regression introduced by WORDCODE (issue #26647). + +.. + +.. bpo: 27030 +.. date: 9787 +.. nonce: 88FOrz +.. section: Library + +Unknown escapes in re.sub() replacement template are allowed again. But +they still are deprecated and will be disabled in 3.7. + +.. + +.. bpo: 28835 +.. date: 9786 +.. nonce: Fv7Dr1 +.. section: Library + +Fix a regression introduced in warnings.catch_warnings(): call +warnings.showwarning() if it was overriden inside the context manager. + +.. + +.. bpo: 27172 +.. date: 9785 +.. nonce: mVKfLT +.. section: Library + +To assist with upgrades from 2.7, the previously documented deprecation of +``inspect.getfullargspec()`` has been reversed. This decision may be +revisited again after the Python 2.7 branch is no longer officially +supported. + +.. + +.. bpo: 26273 +.. date: 9784 +.. nonce: ilNIWN +.. section: Library + +Add new :data:`socket.TCP_CONGESTION` (Linux 2.6.13) and +:data:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37) constants. Patch written by +Omar Sandoval. + +.. + +.. bpo: 24142 +.. date: 9783 +.. nonce: IrZnFs +.. section: Library + +Reading a corrupt config file left configparser in an invalid state. +Original patch by Florian Höch. + +.. + +.. bpo: 28843 +.. date: 9782 +.. nonce: O7M0LE +.. section: Library + +Fix asyncio C Task to handle exceptions __traceback__. + +.. + +.. bpo: 28808 +.. date: 9781 +.. nonce: A03X6r +.. section: C API + +PyUnicode_CompareWithASCIIString() now never raises exceptions. + +.. + +.. bpo: 23722 +.. date: 9780 +.. nonce: 6HX6fk +.. section: Documentation + +The data model reference and the porting section in the What's New guide now +cover the additional ``__classcell__`` handling needed for custom +metaclasses to fully support PEP 487 and zero-argument ``super()``. + +.. + +.. bpo: 28023 +.. date: 9779 +.. nonce: 4gzSGp +.. section: Tools/Demos + +Fix python-gdb.py didn't support new dict implementation. diff --git a/Misc/NEWS.d/3.6.0rc2.rst b/Misc/NEWS.d/3.6.0rc2.rst new file mode 100644 index 00000000000000..4a78694167eab2 --- /dev/null +++ b/Misc/NEWS.d/3.6.0rc2.rst @@ -0,0 +1,45 @@ +.. bpo: 28147 +.. date: 9795 +.. nonce: CnK_xf +.. release date: 2016-12-16 +.. section: Core and Builtins + +Fix a memory leak in split-table dictionaries: setattr() must not convert +combined table into split table. Patch written by INADA Naoki. + +.. + +.. bpo: 28990 +.. date: 9794 +.. nonce: m8xRMJ +.. section: Core and Builtins + +Fix asyncio SSL hanging if connection is closed before handshake is +completed. (Patch by HoHo-Ho) + +.. + +.. bpo: 28770 +.. date: 9793 +.. nonce: N9GQsz +.. section: Tools/Demos + +Fix python-gdb.py for fastcalls. + +.. + +.. bpo: 28896 +.. date: 9792 +.. nonce: ymAbmH +.. section: Windows + +Deprecate WindowsRegistryFinder. + +.. + +.. bpo: 28898 +.. date: 9791 +.. nonce: YGUd_i +.. section: Build + +Prevent gdb build errors due to HAVE_LONG_LONG redefinition. diff --git a/Misc/NEWS.d/3.6.1.rst b/Misc/NEWS.d/3.6.1.rst new file mode 100644 index 00000000000000..a7bf84ef4ad0f6 --- /dev/null +++ b/Misc/NEWS.d/3.6.1.rst @@ -0,0 +1,31 @@ +.. bpo: 29723 +.. date: 9895 +.. nonce: M5omgP +.. release date: 2017-03-21 +.. section: Core and Builtins + +The ``sys.path[0]`` initialization change for bpo-29139 caused a regression +by revealing an inconsistency in how sys.path is initialized when executing +``__main__`` from a zipfile, directory, or other import location. The +interpreter now consistently avoids ever adding the import location's parent +directory to ``sys.path``, and ensures no other ``sys.path`` entries are +inadvertently modified when inserting the import location named on the +command line. + +.. + +.. bpo: 27593 +.. date: 9894 +.. nonce: nk7Etn +.. section: Build + +fix format of git information used in sys.version + +.. + +.. bpo: 0 +.. date: 9893 +.. nonce: usKKNQ +.. section: Build + +Fix incompatible comment in python.h diff --git a/Misc/NEWS.d/3.6.1rc1.rst b/Misc/NEWS.d/3.6.1rc1.rst new file mode 100644 index 00000000000000..9137df6c1435e6 --- /dev/null +++ b/Misc/NEWS.d/3.6.1rc1.rst @@ -0,0 +1,940 @@ +.. bpo: 28893 +.. date: 9892 +.. nonce: WTKnpj +.. release date: 2017-03-04 +.. section: Core and Builtins + +Set correct __cause__ for errors about invalid awaitables returned from +__aiter__ and __anext__. + +.. + +.. bpo: 29683 +.. date: 9891 +.. nonce: G5iS-P +.. section: Core and Builtins + +Fixes to memory allocation in _PyCode_SetExtra. Patch by Brian Coleman. + +.. + +.. bpo: 29684 +.. date: 9890 +.. nonce: wTgEoh +.. section: Core and Builtins + +Fix minor regression of PyEval_CallObjectWithKeywords. It should raise +TypeError when kwargs is not a dict. But it might cause segv when args=NULL +and kwargs is not a dict. + +.. + +.. bpo: 28598 +.. date: 9889 +.. nonce: QxbzQn +.. section: Core and Builtins + +Support __rmod__ for subclasses of str being called before str.__mod__. +Patch by Martijn Pieters. + +.. + +.. bpo: 29607 +.. date: 9888 +.. nonce: 7NvBA1 +.. section: Core and Builtins + +Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu +Dartiailh. + +.. + +.. bpo: 29602 +.. date: 9887 +.. nonce: qyyskC +.. section: Core and Builtins + +Fix incorrect handling of signed zeros in complex constructor for complex +subclasses and for inputs having a __complex__ method. Patch by Serhiy +Storchaka. + +.. + +.. bpo: 29347 +.. date: 9886 +.. nonce: 1RPPGN +.. section: Core and Builtins + +Fixed possibly dereferencing undefined pointers when creating weakref +objects. + +.. + +.. bpo: 29438 +.. date: 9885 +.. nonce: IKxD6I +.. section: Core and Builtins + +Fixed use-after-free problem in key sharing dict. + +.. + +.. bpo: 29319 +.. date: 9884 +.. nonce: KLDUZf +.. section: Core and Builtins + +Prevent RunMainFromImporter overwriting sys.path[0]. + +.. + +.. bpo: 29337 +.. date: 9883 +.. nonce: bjX8AE +.. section: Core and Builtins + +Fixed possible BytesWarning when compare the code objects. Warnings could be +emitted at compile time. + +.. + +.. bpo: 29327 +.. date: 9882 +.. nonce: XXQarW +.. section: Core and Builtins + +Fixed a crash when pass the iterable keyword argument to sorted(). + +.. + +.. bpo: 29034 +.. date: 9881 +.. nonce: 7-uEDT +.. section: Core and Builtins + +Fix memory leak and use-after-free in os module (path_converter). + +.. + +.. bpo: 29159 +.. date: 9880 +.. nonce: gEn_kP +.. section: Core and Builtins + +Fix regression in bytes(x) when x.__index__() raises Exception. + +.. + +.. bpo: 28932 +.. date: 9879 +.. nonce: QnLx8A +.. section: Core and Builtins + +Do not include if it does not exist. + +.. + +.. bpo: 25677 +.. date: 9878 +.. nonce: RWhZrb +.. section: Core and Builtins + +Correct the positioning of the syntax error caret for indented blocks. +Based on patch by Michael Layzell. + +.. + +.. bpo: 29000 +.. date: 9877 +.. nonce: K6wQ-3 +.. section: Core and Builtins + +Fixed bytes formatting of octals with zero padding in alternate form. + +.. + +.. bpo: 26919 +.. date: 9876 +.. nonce: Cm7MSa +.. section: Core and Builtins + +On Android, operating system data is now always encoded/decoded to/from +UTF-8, instead of the locale encoding to avoid inconsistencies with +os.fsencode() and os.fsdecode() which are already using UTF-8. + +.. + +.. bpo: 28991 +.. date: 9875 +.. nonce: lGA0FK +.. section: Core and Builtins + +functools.lru_cache() was susceptible to an obscure reentrancy bug +triggerable by a monkey-patched len() function. + +.. + +.. bpo: 28739 +.. date: 9874 +.. nonce: w1fvhk +.. section: Core and Builtins + +f-string expressions are no longer accepted as docstrings and by +ast.literal_eval() even if they do not include expressions. + +.. + +.. bpo: 28512 +.. date: 9873 +.. nonce: i-pv6d +.. section: Core and Builtins + +Fixed setting the offset attribute of SyntaxError by +PyErr_SyntaxLocationEx() and PyErr_SyntaxLocationObject(). + +.. + +.. bpo: 28918 +.. date: 9872 +.. nonce: SFVuPz +.. section: Core and Builtins + +Fix the cross compilation of xxlimited when Python has been built with +Py_DEBUG defined. + +.. + +.. bpo: 28731 +.. date: 9871 +.. nonce: oNF59u +.. section: Core and Builtins + +Optimize _PyDict_NewPresized() to create correct size dict. Improve speed of +dict literal with constant keys up to 30%. + +.. + +.. bpo: 29169 +.. date: 9870 +.. nonce: 8ypApm +.. section: Library + +Update zlib to 1.2.11. + +.. + +.. bpo: 29623 +.. date: 9869 +.. nonce: D3-NP2 +.. section: Library + +Allow use of path-like object as a single argument in ConfigParser.read(). +Patch by David Ellis. + +.. + +.. bpo: 28963 +.. date: 9868 +.. nonce: tPl8dq +.. section: Library + +Fix out of bound iteration in asyncio.Future.remove_done_callback +implemented in C. + +.. + +.. bpo: 29704 +.. date: 9867 +.. nonce: r-kWqv +.. section: Library + +asyncio.subprocess.SubprocessStreamProtocol no longer closes before all +pipes are closed. + +.. + +.. bpo: 29271 +.. date: 9866 +.. nonce: y8Vj2v +.. section: Library + +Fix Task.current_task and Task.all_tasks implemented in C to accept None +argument as their pure Python implementation. + +.. + +.. bpo: 29703 +.. date: 9865 +.. nonce: ZdsPCR +.. section: Library + +Fix asyncio to support instantiation of new event loops in child processes. + +.. + +.. bpo: 29376 +.. date: 9864 +.. nonce: rrJhJy +.. section: Library + +Fix assertion error in threading._DummyThread.is_alive(). + +.. + +.. bpo: 28624 +.. date: 9863 +.. nonce: 43TJib +.. section: Library + +Add a test that checks that cwd parameter of Popen() accepts PathLike +objects. Patch by Sayan Chowdhury. + +.. + +.. bpo: 28518 +.. date: 9862 +.. nonce: o-Q2Nw +.. section: Library + +Start a transaction implicitly before a DML statement. Patch by Aviv +Palivoda. + +.. + +.. bpo: 29532 +.. date: 9861 +.. nonce: YCwVQn +.. section: Library + +Altering a kwarg dictionary passed to functools.partial() no longer affects +a partial object after creation. + +.. + +.. bpo: 29110 +.. date: 9860 +.. nonce: wmE-_T +.. section: Library + +Fix file object leak in aifc.open() when file is given as a filesystem path +and is not in valid AIFF format. Patch by Anthony Zhang. + +.. + +.. bpo: 28556 +.. date: 9859 +.. nonce: p6967e +.. section: Library + +Various updates to typing module: typing.Counter, typing.ChainMap, improved +ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel +Krebber, and Łukasz Langa. + +.. + +.. bpo: 29100 +.. date: 9858 +.. nonce: LAAERS +.. section: Library + +Fix datetime.fromtimestamp() regression introduced in Python 3.6.0: check +minimum and maximum years. + +.. + +.. bpo: 29519 +.. date: 9857 +.. nonce: oGGgZ4 +.. section: Library + +Fix weakref spewing exceptions during interpreter shutdown when used with a +rare combination of multiprocessing and custom codecs. + +.. + +.. bpo: 29416 +.. date: 9856 +.. nonce: KJGyI_ +.. section: Library + +Prevent infinite loop in pathlib.Path.mkdir + +.. + +.. bpo: 29444 +.. date: 9855 +.. nonce: cEwgmk +.. section: Library + +Fixed out-of-bounds buffer access in the group() method of the match object. +Based on patch by WGH. + +.. + +.. bpo: 29335 +.. date: 9854 +.. nonce: _KC7IK +.. section: Library + +Fix subprocess.Popen.wait() when the child process has exited to a stopped +instead of terminated state (ex: when under ptrace). + +.. + +.. bpo: 29290 +.. date: 9853 +.. nonce: XBqptF +.. section: Library + +Fix a regression in argparse that help messages would wrap at non-breaking +spaces. + +.. + +.. bpo: 28735 +.. date: 9852 +.. nonce: admHLO +.. section: Library + +Fixed the comparison of mock.MagickMock with mock.ANY. + +.. + +.. bpo: 29316 +.. date: 9851 +.. nonce: OeOQw5 +.. section: Library + +Restore the provisional status of typing module, add corresponding note to +documentation. Patch by Ivan L. + +.. + +.. bpo: 29219 +.. date: 9850 +.. nonce: kxui7t +.. section: Library + +Fixed infinite recursion in the repr of uninitialized ctypes.CDLL instances. + +.. + +.. bpo: 29011 +.. date: 9849 +.. nonce: MI5f2R +.. section: Library + +Fix an important omission by adding Deque to the typing module. + +.. + +.. bpo: 28969 +.. date: 9848 +.. nonce: j3HJYO +.. section: Library + +Fixed race condition in C implementation of functools.lru_cache. KeyError +could be raised when cached function with full cache was simultaneously +called from differen threads with the same uncached arguments. + +.. + +.. bpo: 29142 +.. date: 9847 +.. nonce: xo6kAv +.. section: Library + +In urllib.request, suffixes in no_proxy environment variable with leading +dots could match related hostnames again (e.g. .b.c matches a.b.c). Patch by +Milan Oberkirch. + +.. + +.. bpo: 28961 +.. date: 9846 +.. nonce: Rt93vg +.. section: Library + +Fix unittest.mock._Call helper: don't ignore the name parameter anymore. +Patch written by Jiajun Huang. + +.. + +.. bpo: 29203 +.. date: 9845 +.. nonce: kN5S6v +.. section: Library + +functools.lru_cache() now respects PEP 468 and preserves the order of +keyword arguments. f(a=1, b=2) is now cached separately from f(b=2, a=1) +since both calls could potentially give different results. + +.. + +.. bpo: 15812 +.. date: 9844 +.. nonce: R1U-Ec +.. section: Library + +inspect.getframeinfo() now correctly shows the first line of a context. +Patch by Sam Breese. + +.. + +.. bpo: 29094 +.. date: 9843 +.. nonce: 460ZQo +.. section: Library + +Offsets in a ZIP file created with extern file object and modes "w" and "x" +now are relative to the start of the file. + +.. + +.. bpo: 29085 +.. date: 9842 +.. nonce: bm3gkx +.. section: Library + +Allow random.Random.seed() to use high quality OS randomness rather than the +pid and time. + +.. + +.. bpo: 29061 +.. date: 9841 +.. nonce: YKq0Ba +.. section: Library + +Fixed bug in secrets.randbelow() which would hang when given a negative +input. Patch by Brendan Donegan. + +.. + +.. bpo: 29079 +.. date: 9840 +.. nonce: g4YLix +.. section: Library + +Prevent infinite loop in pathlib.resolve() on Windows + +.. + +.. bpo: 13051 +.. date: 9839 +.. nonce: YzC1Te +.. section: Library + +Fixed recursion errors in large or resized curses.textpad.Textbox. Based on +patch by Tycho Andersen. + +.. + +.. bpo: 29119 +.. date: 9838 +.. nonce: Ov69fr +.. section: Library + +Fix weakrefs in the pure python version of collections.OrderedDict +move_to_end() method. Contributed by Andra Bogildea. + +.. + +.. bpo: 9770 +.. date: 9837 +.. nonce: WJJnwP +.. section: Library + +curses.ascii predicates now work correctly with negative integers. + +.. + +.. bpo: 28427 +.. date: 9836 +.. nonce: vUd-va +.. section: Library + +old keys should not remove new values from WeakValueDictionary when +collecting from another thread. + +.. + +.. bpo: 28923 +.. date: 9835 +.. nonce: naVULD +.. section: Library + +Remove editor artifacts from Tix.py. + +.. + +.. bpo: 29055 +.. date: 9834 +.. nonce: -r_9jc +.. section: Library + +Neaten-up empty population error on random.choice() by suppressing the +upstream exception. + +.. + +.. bpo: 28871 +.. date: 9833 +.. nonce: cPMXCJ +.. section: Library + +Fixed a crash when deallocate deep ElementTree. + +.. + +.. bpo: 19542 +.. date: 9832 +.. nonce: 5tCkaK +.. section: Library + +Fix bugs in WeakValueDictionary.setdefault() and WeakValueDictionary.pop() +when a GC collection happens in another thread. + +.. + +.. bpo: 20191 +.. date: 9831 +.. nonce: Q7uZCS +.. section: Library + +Fixed a crash in resource.prlimit() when passing a sequence that doesn't own +its elements as limits. + +.. + +.. bpo: 28779 +.. date: 9830 +.. nonce: t-mjED +.. section: Library + +multiprocessing.set_forkserver_preload() would crash the forkserver process +if a preloaded module instantiated some multiprocessing objects such as +locks. + +.. + +.. bpo: 28847 +.. date: 9829 +.. nonce: J7d3nG +.. section: Library + +dbm.dumb now supports reading read-only files and no longer writes the index +file when it is not changed. + +.. + +.. bpo: 26937 +.. date: 9828 +.. nonce: c9kgiA +.. section: Library + +The chown() method of the tarfile.TarFile class does not fail now when the +grp module cannot be imported, as for example on Android platforms. + +.. + +.. bpo: 29071 +.. date: 9827 +.. nonce: FCOpJn +.. section: IDLE + +IDLE colors f-string prefixes (but not invalid ur prefixes). + +.. + +.. bpo: 28572 +.. date: 9826 +.. nonce: 1_duKY +.. section: IDLE + +Add 10% to coverage of IDLE's test_configdialog. Update and augment +description of the configuration system. + +.. + +.. bpo: 29579 +.. date: 9825 +.. nonce: Ih-G2Q +.. section: Windows + +Removes readme.txt from the installer + +.. + +.. bpo: 29326 +.. date: 9824 +.. nonce: 4qDQzs +.. section: Windows + +Ignores blank lines in ._pth files (Patch by Alexey Izbyshev) + +.. + +.. bpo: 28164 +.. date: 9823 +.. nonce: h4CFX8 +.. section: Windows + +Correctly handle special console filenames (patch by Eryk Sun) + +.. + +.. bpo: 29409 +.. date: 9822 +.. nonce: bhvrJ2 +.. section: Windows + +Implement PEP 529 for io.FileIO (Patch by Eryk Sun) + +.. + +.. bpo: 29392 +.. date: 9821 +.. nonce: OtqS5t +.. section: Windows + +Prevent crash when passing invalid arguments into msvcrt module. + +.. + +.. bpo: 25778 +.. date: 9820 +.. nonce: 8uKJ82 +.. section: Windows + +winreg does not truncate string correctly (Patch by Eryk Sun) + +.. + +.. bpo: 28896 +.. date: 9819 +.. nonce: VMi9w0 +.. section: Windows + +Deprecate WindowsRegistryFinder and disable it by default. + +.. + +.. bpo: 27867 +.. date: 9818 +.. nonce: UC5ohc +.. section: C API + +Function PySlice_GetIndicesEx() is replaced with a macro if Py_LIMITED_API +is not set or set to the value between 0x03050400 and 0x03060000 (not +including) or 0x03060100 or higher. + +.. + +.. bpo: 29083 +.. date: 9817 +.. nonce: tGTjr_ +.. section: C API + +Fixed the declaration of some public API functions. PyArg_VaParse() and +PyArg_VaParseTupleAndKeywords() were not available in limited API. +PyArg_ValidateKeywordArguments(), PyArg_UnpackTuple() and Py_BuildValue() +were not available in limited API of version < 3.3 when PY_SSIZE_T_CLEAN is +defined. + +.. + +.. bpo: 29058 +.. date: 9816 +.. nonce: 0wNVP8 +.. section: C API + +All stable API extensions added after Python 3.2 are now available only when +Py_LIMITED_API is set to the PY_VERSION_HEX value of the minimum Python +version supporting this API. + +.. + +.. bpo: 28929 +.. date: 9815 +.. nonce: Md7kb0 +.. section: Documentation + +Link the documentation to its source file on GitHub. + +.. + +.. bpo: 25008 +.. date: 9814 +.. nonce: CeIzyU +.. section: Documentation + +Document smtpd.py as effectively deprecated and add a pointer to aiosmtpd, a +third-party asyncio-based replacement. + +.. + +.. bpo: 26355 +.. date: 9813 +.. nonce: SDq_8Y +.. section: Documentation + +Add canonical header link on each page to corresponding major version of the +documentation. Patch by Matthias Bussonnier. + +.. + +.. bpo: 29349 +.. date: 9812 +.. nonce: PjSo-t +.. section: Documentation + +Fix Python 2 syntax in code for building the documentation. + +.. + +.. bpo: 28087 +.. date: 9811 +.. nonce: m8dc4R +.. section: Tests + +Skip test_asyncore and test_eintr poll failures on macOS. Skip some tests of +select.poll when running on macOS due to unresolved issues with the +underlying system poll function on some macOS versions. + +.. + +.. bpo: 29571 +.. date: 9810 +.. nonce: r6Dixr +.. section: Tests + +to match the behaviour of the ``re.LOCALE`` flag, test_re.test_locale_flag +now uses ``locale.getpreferredencoding(False)`` to determine the candidate +encoding for the test regex (allowing it to correctly skip the test when the +default locale encoding is a multi-byte encoding) + +.. + +.. bpo: 28950 +.. date: 9809 +.. nonce: 1W8Glo +.. section: Tests + +Disallow -j0 to be combined with -T/-l in regrtest command line arguments. + +.. + +.. bpo: 28683 +.. date: 9808 +.. nonce: Fp-Hdq +.. section: Tests + +Fix the tests that bind() a unix socket and raise PermissionError on Android +for a non-root user. + +.. + +.. bpo: 26939 +.. date: 9807 +.. nonce: 7j_W5R +.. section: Tests + +Add the support.setswitchinterval() function to fix test_functools hanging +on the Android armv7 qemu emulator. + +.. + +.. bpo: 27593 +.. date: 9806 +.. nonce: v87xEr +.. section: Build + +sys.version and the platform module python_build(), python_branch(), and +python_revision() functions now use git information rather than hg when +building from a repo. + +.. + +.. bpo: 29572 +.. date: 9805 +.. nonce: iZ1XKK +.. section: Build + +Update Windows build and OS X installers to use OpenSSL 1.0.2k. + +.. + +.. bpo: 26851 +.. date: 9804 +.. nonce: R5243g +.. section: Build + +Set Android compilation and link flags. + +.. + +.. bpo: 28768 +.. date: 9803 +.. nonce: b9_a6E +.. section: Build + +Fix implicit declaration of function _setmode. Patch by Masayuki Yamamoto + +.. + +.. bpo: 29080 +.. date: 9802 +.. nonce: b3qLQT +.. section: Build + +Removes hard dependency on hg.exe from PCBuild/build.bat + +.. + +.. bpo: 23903 +.. date: 9801 +.. nonce: JXJ889 +.. section: Build + +Added missed names to PC/python3.def. + +.. + +.. bpo: 28762 +.. date: 9800 +.. nonce: Ru0YN_ +.. section: Build + +lockf() is available on Android API level 24, but the F_LOCK macro is not +defined in android-ndk-r13. + +.. + +.. bpo: 28538 +.. date: 9799 +.. nonce: FqtN7v +.. section: Build + +Fix the compilation error that occurs because if_nameindex() is available on +Android API level 24, but the if_nameindex structure is not defined. + +.. + +.. bpo: 20211 +.. date: 9798 +.. nonce: gpNptI +.. section: Build + +Do not add the directory for installing C header files and the directory for +installing object code libraries to the cross compilation search paths. +Original patch by Thomas Petazzoni. + +.. + +.. bpo: 28849 +.. date: 9797 +.. nonce: AzRRF5 +.. section: Build + +Do not define sys.implementation._multiarch on Android. diff --git a/Misc/NEWS.d/3.6.2.rst b/Misc/NEWS.d/3.6.2.rst new file mode 100644 index 00000000000000..dba43d146df954 --- /dev/null +++ b/Misc/NEWS.d/3.6.2.rst @@ -0,0 +1,7 @@ +.. bpo: 0 +.. date: 9993 +.. no changes: True +.. nonce: F9ENBV +.. release date: 2017-07-17 + +No changes since release candidate 2 diff --git a/Misc/NEWS.d/3.6.2rc1.rst b/Misc/NEWS.d/3.6.2rc1.rst new file mode 100644 index 00000000000000..4ec0f9789fea94 --- /dev/null +++ b/Misc/NEWS.d/3.6.2rc1.rst @@ -0,0 +1,942 @@ +.. bpo: 30682 +.. date: 9989 +.. nonce: zZm88E +.. release date: 2017-06-17 +.. section: Core and Builtins + +Removed a too-strict assertion that failed for certain f-strings, such as +eval("f'\\\n'") and eval("f'\\\r'"). + +.. + +.. bpo: 30604 +.. date: 9988 +.. nonce: W47hPY +.. section: Core and Builtins + +Move co_extra_freefuncs to not be per-thread to avoid crashes + +.. + +.. bpo: 29104 +.. date: 9987 +.. nonce: u26yCx +.. section: Core and Builtins + +Fixed parsing backslashes in f-strings. + +.. + +.. bpo: 27945 +.. date: 9986 +.. nonce: p29r3O +.. section: Core and Builtins + +Fixed various segfaults with dict when input collections are mutated during +searching, inserting or comparing. Based on patches by Duane Griffin and +Tim Mitchell. + +.. + +.. bpo: 25794 +.. date: 9985 +.. nonce: xfPwqm +.. section: Core and Builtins + +Fixed type.__setattr__() and type.__delattr__() for non-interned attribute +names. Based on patch by Eryk Sun. + +.. + +.. bpo: 30039 +.. date: 9984 +.. nonce: e0u4DG +.. section: Core and Builtins + +If a KeyboardInterrupt happens when the interpreter is in the middle of +resuming a chain of nested 'yield from' or 'await' calls, it's now correctly +delivered to the innermost frame. + +.. + +.. bpo: 12414 +.. date: 9983 +.. nonce: T9ix8O +.. section: Core and Builtins + +sys.getsizeof() on a code object now returns the sizes which includes the +code struct and sizes of objects which it references. Patch by Dong-hee Na. + +.. + +.. bpo: 29949 +.. date: 9982 +.. nonce: DevGPS +.. section: Core and Builtins + +Fix memory usage regression of set and frozenset object. + +.. + +.. bpo: 29935 +.. date: 9981 +.. nonce: vgjdJo +.. section: Core and Builtins + +Fixed error messages in the index() method of tuple, list and deque when +pass indices of wrong type. + +.. + +.. bpo: 29859 +.. date: 9980 +.. nonce: Z1MLcA +.. section: Core and Builtins + +Show correct error messages when any of the pthread_* calls in +thread_pthread.h fails. + +.. + +.. bpo: 28876 +.. date: 9979 +.. nonce: cU-sGT +.. section: Core and Builtins + +``bool(range)`` works even if ``len(range)`` raises :exc:`OverflowError`. + +.. + +.. bpo: 29600 +.. date: 9978 +.. nonce: 77wQ6C +.. section: Core and Builtins + +Fix wrapping coroutine return values in StopIteration. + +.. + +.. bpo: 28856 +.. date: 9977 +.. nonce: AFRmo4 +.. section: Core and Builtins + +Fix an oversight that %b format for bytes should support objects follow the +buffer protocol. + +.. + +.. bpo: 29714 +.. date: 9976 +.. nonce: z-BhVd +.. section: Core and Builtins + +Fix a regression that bytes format may fail when containing zero bytes +inside. + +.. + +.. bpo: 29478 +.. date: 9975 +.. nonce: rTQ-qy +.. section: Core and Builtins + +If max_line_length=None is specified while using the Compat32 policy, it is +no longer ignored. Patch by Mircea Cosbuc. + +.. + +.. bpo: 30616 +.. date: 9974 +.. nonce: I2mDTz +.. section: Library + +Functional API of enum allows to create empty enums. Patched by Dong-hee Na + +.. + +.. bpo: 30038 +.. date: 9973 +.. nonce: vb4DWk +.. section: Library + +Fix race condition between signal delivery and wakeup file descriptor. +Patch by Nathaniel Smith. + +.. + +.. bpo: 23894 +.. date: 9972 +.. nonce: k2pADV +.. section: Library + +lib2to3 now recognizes ``rb'...'`` and ``f'...'`` strings. + +.. + +.. bpo: 23890 +.. date: 9971 +.. nonce: GCFAAZ +.. section: Library + +unittest.TestCase.assertRaises() now manually breaks a reference cycle to +not keep objects alive longer than expected. + +.. + +.. bpo: 30149 +.. date: 9970 +.. nonce: hE649r +.. section: Library + +inspect.signature() now supports callables with variable-argument parameters +wrapped with partialmethod. Patch by Dong-hee Na. + +.. + +.. bpo: 30645 +.. date: 9969 +.. nonce: oYzbbW +.. section: Library + +Fix path calculation in imp.load_package(), fixing it for cases when a +package is only shipped with bytecodes. Patch by Alexandru Ardelean. + +.. + +.. bpo: 29931 +.. date: 9968 +.. nonce: tfcTwK +.. section: Library + +Fixed comparison check for ipaddress.ip_interface objects. Patch by Sanjay +Sundaresan. + +.. + +.. bpo: 30605 +.. date: 9967 +.. nonce: XqGz1r +.. section: Library + +re.compile() no longer raises a BytesWarning when compiling a bytes instance +with misplaced inline modifier. Patch by Roy Williams. + +.. + +.. bpo: 29591 +.. date: 9966 +.. nonce: ExKblw +.. original section: Library +.. section: Security + +Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and +CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +information. + +.. + +.. bpo: 24484 +.. date: 9965 +.. nonce: fNS32j +.. section: Library + +Avoid race condition in multiprocessing cleanup (#2159) + +.. + +.. bpo: 28994 +.. date: 9964 +.. nonce: 9vzun1 +.. section: Library + +The traceback no longer displayed for SystemExit raised in a callback +registered by atexit. + +.. + +.. bpo: 30508 +.. date: 9963 +.. nonce: wNWRS2 +.. section: Library + +Don't log exceptions if Task/Future "cancel()" method was called. + +.. + +.. bpo: 28556 +.. date: 9962 +.. nonce: mESP7G +.. section: Library + +Updates to typing module: Add generic AsyncContextManager, add support for +ContextManager on all versions. Original PRs by Jelle Zijlstra and Ivan +Levkivskyi + +.. + +.. bpo: 29870 +.. date: 9961 +.. nonce: p960Ih +.. section: Library + +Fix ssl sockets leaks when connection is aborted in asyncio/ssl +implementation. Patch by Michaël Sghaïer. + +.. + +.. bpo: 29743 +.. date: 9960 +.. nonce: en2P4s +.. section: Library + +Closing transport during handshake process leaks open socket. Patch by +Nikolay Kim + +.. + +.. bpo: 27585 +.. date: 9959 +.. nonce: 0Ugqqu +.. section: Library + +Fix waiter cancellation in asyncio.Lock. Patch by Mathieu Sornay. + +.. + +.. bpo: 30418 +.. date: 9958 +.. nonce: EwISQm +.. section: Library + +On Windows, subprocess.Popen.communicate() now also ignore EINVAL on +stdin.write() if the child process is still running but closed the pipe. + +.. + +.. bpo: 29822 +.. date: 9957 +.. nonce: G7dX13 +.. section: Library + +inspect.isabstract() now works during __init_subclass__. Patch by Nate +Soares. + +.. + +.. bpo: 29581 +.. date: 9956 +.. nonce: gHCrxP +.. section: Library + +ABCMeta.__new__ now accepts ``**kwargs``, allowing abstract base classes to +use keyword parameters in __init_subclass__. Patch by Nate Soares. + +.. + +.. bpo: 30557 +.. date: 9955 +.. nonce: uykrLf +.. section: Library + +faulthandler now correctly filters and displays exception codes on Windows + +.. + +.. bpo: 30378 +.. date: 9954 +.. nonce: R_19_5 +.. section: Library + +Fix the problem that logging.handlers.SysLogHandler cannot handle IPv6 +addresses. + +.. + +.. bpo: 29960 +.. date: 9953 +.. nonce: g0wr3r +.. section: Library + +Preserve generator state when _random.Random.setstate() raises an exception. +Patch by Bryan Olson. + +.. + +.. bpo: 30414 +.. date: 9952 +.. nonce: jGl1Lb +.. section: Library + +multiprocessing.Queue._feed background running thread do not break from main +loop on exception. + +.. + +.. bpo: 30003 +.. date: 9951 +.. nonce: BOl9HE +.. section: Library + +Fix handling escape characters in HZ codec. Based on patch by Ma Lin. + +.. + +.. bpo: 30301 +.. date: 9950 +.. nonce: ywOkjN +.. section: Library + +Fix AttributeError when using SimpleQueue.empty() under *spawn* and +*forkserver* start methods. + +.. + +.. bpo: 30329 +.. date: 9949 +.. nonce: EuT36N +.. section: Library + +imaplib and poplib now catch the Windows socket WSAEINVAL error (code 10022) +on shutdown(SHUT_RDWR): An invalid operation was attempted. This error +occurs sometimes on SSL connections. + +.. + +.. bpo: 30375 +.. date: 9948 +.. nonce: 9c8qM7 +.. section: Library + +Warnings emitted when compile a regular expression now always point to the +line in the user code. Previously they could point into inners of the re +module if emitted from inside of groups or conditionals. + +.. + +.. bpo: 30048 +.. date: 9947 +.. nonce: ELRx8R +.. section: Library + +Fixed ``Task.cancel()`` can be ignored when the task is running coroutine +and the coroutine returned without any more ``await``. + +.. + +.. bpo: 30266 +.. date: 9946 +.. nonce: YJzHAH +.. section: Library + +contextlib.AbstractContextManager now supports anti-registration by setting +__enter__ = None or __exit__ = None, following the pattern introduced in +bpo-25958. Patch by Jelle Zijlstra. + +.. + +.. bpo: 30298 +.. date: 9945 +.. nonce: ZN-bWo +.. section: Library + +Weaken the condition of deprecation warnings for inline modifiers. Now +allowed several subsequential inline modifiers at the start of the pattern +(e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments now are +allowed before and between inline modifiers (e.g. ``'(?x) (?i) (?s)...'``). + +.. + +.. bpo: 29990 +.. date: 9944 +.. nonce: HWV6KE +.. section: Library + +Fix range checking in GB18030 decoder. Original patch by Ma Lin. + +.. + +.. bpo: 26293 +.. date: 9943 +.. nonce: wig0YG +.. section: Library + +Change resulted because of zipfile breakage. (See also: bpo-29094) + +.. + +.. bpo: 30243 +.. date: 9942 +.. nonce: RHQt0v +.. section: Library + +Removed the __init__ methods of _json's scanner and encoder. Misusing them +could cause memory leaks or crashes. Now scanner and encoder objects are +completely initialized in the __new__ methods. + +.. + +.. bpo: 30185 +.. date: 9941 +.. nonce: Tiu1n8 +.. section: Library + +Avoid KeyboardInterrupt tracebacks in forkserver helper process when Ctrl-C +is received. + +.. + +.. bpo: 28556 +.. date: 9940 +.. nonce: 51gjbP +.. section: Library + +Various updates to typing module: add typing.NoReturn type, use +WrapperDescriptorType, minor bug-fixes. Original PRs by Jim Fasarakis- +Hilliard and Ivan Levkivskyi. + +.. + +.. bpo: 30205 +.. date: 9939 +.. nonce: BsxO34 +.. section: Library + +Fix getsockname() for unbound AF_UNIX sockets on Linux. + +.. + +.. bpo: 30070 +.. date: 9938 +.. nonce: XM_B41 +.. section: Library + +Fixed leaks and crashes in errors handling in the parser module. + +.. + +.. bpo: 30061 +.. date: 9937 +.. nonce: 2w_dX9 +.. section: Library + +Fixed crashes in IOBase methods __next__() and readlines() when readline() +or __next__() respectively return non-sizeable object. Fixed possible other +errors caused by not checking results of PyObject_Size(), PySequence_Size(), +or PyMapping_Size(). + +.. + +.. bpo: 30017 +.. date: 9936 +.. nonce: cKBuhU +.. section: Library + +Allowed calling the close() method of the zip entry writer object multiple +times. Writing to a closed writer now always produces a ValueError. + +.. + +.. bpo: 30068 +.. date: 9935 +.. nonce: n4q47r +.. section: Library + +_io._IOBase.readlines will check if it's closed first when hint is present. + +.. + +.. bpo: 29694 +.. date: 9934 +.. nonce: LWKxb1 +.. section: Library + +Fixed race condition in pathlib mkdir with flags parents=True. Patch by +Armin Rigo. + +.. + +.. bpo: 29692 +.. date: 9933 +.. nonce: oyWrAE +.. section: Library + +Fixed arbitrary unchaining of RuntimeError exceptions in +contextlib.contextmanager. Patch by Siddharth Velankar. + +.. + +.. bpo: 29998 +.. date: 9932 +.. nonce: poeIKD +.. section: Library + +Pickling and copying ImportError now preserves name and path attributes. + +.. + +.. bpo: 29953 +.. date: 9931 +.. nonce: Q1hSt- +.. section: Library + +Fixed memory leaks in the replace() method of datetime and time objects when +pass out of bound fold argument. + +.. + +.. bpo: 29942 +.. date: 9930 +.. nonce: CsGNuT +.. section: Library + +Fix a crash in itertools.chain.from_iterable when encountering long runs of +empty iterables. + +.. + +.. bpo: 27863 +.. date: 9929 +.. nonce: pPYHHI +.. section: Library + +Fixed multiple crashes in ElementTree caused by race conditions and wrong +types. + +.. + +.. bpo: 28699 +.. date: 9928 +.. nonce: wZztZP +.. section: Library + +Fixed a bug in pools in multiprocessing.pool that raising an exception at +the very first of an iterable may swallow the exception or make the program +hang. Patch by Davin Potts and Xiang Zhang. + +.. + +.. bpo: 25803 +.. date: 9927 +.. nonce: CPDR0W +.. section: Library + +Avoid incorrect errors raised by Path.mkdir(exist_ok=True) when the OS gives +priority to errors such as EACCES over EEXIST. + +.. + +.. bpo: 29861 +.. date: 9926 +.. nonce: t2ZoRK +.. section: Library + +Release references to tasks, their arguments and their results as soon as +they are finished in multiprocessing.Pool. + +.. + +.. bpo: 29884 +.. date: 9925 +.. nonce: kWXR8W +.. section: Library + +faulthandler: Restore the old sigaltstack during teardown. Patch by +Christophe Zeitouny. + +.. + +.. bpo: 25455 +.. date: 9924 +.. nonce: ZsahHN +.. section: Library + +Fixed crashes in repr of recursive buffered file-like objects. + +.. + +.. bpo: 29800 +.. date: 9923 +.. nonce: d2xASa +.. section: Library + +Fix crashes in partial.__repr__ if the keys of partial.keywords are not +strings. Patch by Michael Seifert. + +.. + +.. bpo: 29742 +.. date: 9922 +.. nonce: 8hqfEO +.. section: Library + +get_extra_info() raises exception if get called on closed ssl transport. +Patch by Nikolay Kim. + +.. + +.. bpo: 8256 +.. date: 9921 +.. nonce: jAwGQH +.. section: Library + +Fixed possible failing or crashing input() if attributes "encoding" or +"errors" of sys.stdin or sys.stdout are not set or are not strings. + +.. + +.. bpo: 28298 +.. date: 9920 +.. nonce: xfm84U +.. section: Library + +Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big intables +(objects that have __int__) as elements. Patch by Oren Milman. + +.. + +.. bpo: 28231 +.. date: 9919 +.. nonce: MG1X09 +.. section: Library + +The zipfile module now accepts path-like objects for external paths. + +.. + +.. bpo: 26915 +.. date: 9918 +.. nonce: qShJZO +.. section: Library + +index() and count() methods of collections.abc.Sequence now check identity +before checking equality when do comparisons. + +.. + +.. bpo: 29615 +.. date: 9917 +.. nonce: OpFKzg +.. section: Library + +SimpleXMLRPCDispatcher no longer chains KeyError (or any other exception) to +exception(s) raised in the dispatched methods. Patch by Petr Motejlek. + +.. + +.. bpo: 30177 +.. date: 9916 +.. nonce: JGIJNL +.. section: Library + +path.resolve(strict=False) no longer cuts the path after the first element +not present in the filesystem. Patch by Antoine Pietri. + +.. + +.. bpo: 15786 +.. date: 9915 +.. nonce: _XRbaR +.. section: IDLE + +Fix several problems with IDLE's autocompletion box. The following should +now work: clicking on selection box items; using the scrollbar; selecting an +item by hitting Return. Hangs on MacOSX should no longer happen. Patch by +Louie Lu. + +.. + +.. bpo: 25514 +.. date: 9914 +.. nonce: 882pXa +.. section: IDLE + +Add doc subsubsection about IDLE failure to start. Popup no-connection +message directs users to this section. + +.. + +.. bpo: 30642 +.. date: 9913 +.. nonce: 3Zujzt +.. section: IDLE + +Fix reference leaks in IDLE tests. Patches by Louie Lu and Terry Jan Reedy. + +.. + +.. bpo: 30495 +.. date: 9912 +.. nonce: I3i5vL +.. section: IDLE + +Add docstrings for textview.py and use PEP8 names. Patches by Cheryl Sabella +and Terry Jan Reedy. + +.. + +.. bpo: 30290 +.. date: 9911 +.. nonce: fZ3kod +.. section: IDLE + +Help-about: use pep8 names and add tests. Increase coverage to 100%. Patches +by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. + +.. + +.. bpo: 30303 +.. date: 9910 +.. nonce: 2L2F-4 +.. section: IDLE + +Add _utest option to textview; add new tests. Increase coverage to 100%. +Patches by Louie Lu and Terry Jan Reedy. + +.. + +.. bpo: 27867 +.. date: 9909 +.. nonce: B46BRE +.. section: C API + +Function PySlice_GetIndicesEx() no longer replaced with a macro if +Py_LIMITED_API is not set. + +.. + +.. bpo: 29941 +.. date: 9908 +.. nonce: ylh45A +.. section: Build + +Add ``--with-assertions`` configure flag to explicitly enable C ``assert()`` +checks. Defaults to off. ``--with-pydebug`` implies ``--with-assertions``. + +.. + +.. bpo: 28787 +.. date: 9907 +.. nonce: vhH_6a +.. section: Build + +Fix out-of-tree builds of Python when configured with ``--with--dtrace``. + +.. + +.. bpo: 29243 +.. date: 9906 +.. nonce: WDK4hT +.. section: Build + +Prevent unnecessary rebuilding of Python during ``make test``, ``make +install`` and some other make targets when configured with ``--enable- +optimizations``. + +.. + +.. bpo: 23404 +.. date: 9905 +.. nonce: PdYVWg +.. section: Build + +Don't regenerate generated files based on file modification time anymore: +the action is now explicit. Replace ``make touch`` with ``make regen-all``. + +.. + +.. bpo: 29643 +.. date: 9904 +.. nonce: 4WLIJQ +.. section: Build + +Fix ``--enable-optimization`` didn't work. + +.. + +.. bpo: 30176 +.. date: 9903 +.. nonce: VivmCg +.. section: Documentation + +Add missing attribute related constants in curses documentation. + +.. + +.. bpo: 30052 +.. date: 9902 +.. nonce: TpmpaF +.. section: Documentation + +the link targets for :func:`bytes` and :func:`bytearray` are now their +respective type definitions, rather than the corresponding builtin function +entries. Use :ref:`bytes ` and :ref:`bytearray ` +to reference the latter. + +In order to ensure this and future cross-reference updates are applied +automatically, the daily documentation builds now disable the default output +caching features in Sphinx. + +.. + +.. bpo: 26985 +.. date: 9901 +.. nonce: NB5_9S +.. section: Documentation + +Add missing info of code object in inspect documentation. + +.. + +.. bpo: 29367 +.. date: 9900 +.. nonce: 4dOKL0 +.. section: Tools/Demos + +python-gdb.py now supports also ``method-wrapper`` (``wrapperobject``) +objects. + +.. + +.. bpo: 30357 +.. date: 9899 +.. nonce: n4CPEa +.. section: Tests + +test_thread: setUp() now uses support.threading_setup() and +support.threading_cleanup() to wait until threads complete to avoid random +side effects on following tests. Initial patch written by Grzegorz Grzywacz. + +.. + +.. bpo: 30197 +.. date: 9898 +.. nonce: c5wRfu +.. section: Tests + +Enhanced functions swap_attr() and swap_item() in the test.support module. +They now work when delete replaced attribute or item inside the with +statement. The old value of the attribute or item (or None if it doesn't +exist) now will be assigned to the target of the "as" clause, if there is +one. + +.. + +.. bpo: 30687 +.. date: 9897 +.. nonce: 8mqHnu +.. section: Windows + +Locate msbuild.exe on Windows when building rather than vcvarsall.bat + +.. + +.. bpo: 30450 +.. date: 9896 +.. nonce: qsaK8y +.. section: Windows + +The build process on Windows no longer depends on Subversion, instead +pulling external code from GitHub via a Python script. If Python 3.6 is not +found on the system (via ``py -3.6``), NuGet is used to download a copy of +32-bit Python. diff --git a/Misc/NEWS.d/3.6.2rc2.rst b/Misc/NEWS.d/3.6.2rc2.rst new file mode 100644 index 00000000000000..4fdfabd0e31167 --- /dev/null +++ b/Misc/NEWS.d/3.6.2rc2.rst @@ -0,0 +1,39 @@ +.. bpo: 30730 +.. date: 9992 +.. nonce: rJsyTH +.. original section: Library +.. release date: 2017-07-07 +.. section: Security + +Prevent environment variables injection in subprocess on Windows. Prevent +passing other environment variables and command arguments. + +.. + +.. bpo: 30694 +.. date: 9991 +.. nonce: WkMWM_ +.. original section: Library +.. section: Security + +Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security +vulnerabilities including: CVE-2017-9233 (External entity infinite loop +DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix +regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 +(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use os- +specific entropy sources like getrandom) doesn't impact Python, since Python +already gets entropy from the OS to set the expat secret using +``XML_SetHashSalt()``. + +.. + +.. bpo: 30500 +.. date: 9990 +.. nonce: 1VG7R- +.. original section: Library +.. section: Security + +Fix urllib.parse.splithost() to correctly parse fragments. For example, +``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the +``127.0.0.1`` host, instead of treating ``@evil.com`` as the host in an +authentification (``login@host``). diff --git a/Misc/NEWS.d/3.6.3.rst b/Misc/NEWS.d/3.6.3.rst new file mode 100644 index 00000000000000..4d591d77ffe545 --- /dev/null +++ b/Misc/NEWS.d/3.6.3.rst @@ -0,0 +1,27 @@ +.. bpo: 31641 +.. date: 2017-10-03-01-05-11 +.. nonce: vlQEq5 +.. release date: 2017-10-03 +.. section: Library + +Re-allow arbitrary iterables in `concurrent.futures.as_completed()`. Fixes +regression in 3.6.3rc1. + +.. + +.. bpo: 31662 +.. date: 2017-10-03-01-06-24 +.. nonce: 8l2jEz +.. section: Build + +Fix typos in Windows ``uploadrelease.bat`` script. Fix Windows Doc build +issues in ``Doc/make.bat``. + +.. + +.. bpo: 31423 +.. date: 2017-10-03-01-01-52 +.. nonce: uKvPYA +.. section: Build + +Fix building the PDF documentation with newer versions of Sphinx. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst new file mode 100644 index 00000000000000..b16592bf1a5d3b --- /dev/null +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -0,0 +1,1259 @@ +.. bpo: 29781 +.. date: 2017-09-05-15-26-30 +.. nonce: LwYtBP +.. release date: 2017-09-18 +.. section: Security + +SSLObject.version() now correctly returns None when handshake over BIO has +not been performed yet. + +.. + +.. bpo: 30947 +.. date: 2017-08-16-16-35-59 +.. nonce: iNMmm4 +.. section: Security + +Upgrade libexpat embedded copy from version 2.2.1 to 2.2.3 to get security +fixes. + +.. + +.. bpo: 31471 +.. date: 2017-09-14-19-47-57 +.. nonce: 0yiA5Q +.. section: Core and Builtins + +Fix an assertion failure in `subprocess.Popen()` on Windows, in case the env +argument has a bad keys() method. Patch by Oren Milman. + +.. + +.. bpo: 31418 +.. date: 2017-09-13-13-03-52 +.. nonce: rS-FlC +.. section: Core and Builtins + +Fix an assertion failure in `PyErr_WriteUnraisable()` in case of an +exception with a bad ``__module__`` attribute. Patch by Oren Milman. + +.. + +.. bpo: 31416 +.. date: 2017-09-11-12-54-35 +.. nonce: 2hlQFd +.. section: Core and Builtins + +Fix assertion failures in case of a bad warnings.filters or +warnings.defaultaction. Patch by Oren Milman. + +.. + +.. bpo: 31411 +.. date: 2017-09-11-08-50-41 +.. nonce: HZz82I +.. section: Core and Builtins + +Raise a TypeError instead of SystemError in case warnings.onceregistry is +not a dictionary. Patch by Oren Milman. + +.. + +.. bpo: 31373 +.. date: 2017-09-06-15-25-59 +.. nonce: dC4jd4 +.. section: Core and Builtins + +Fix several possible instances of undefined behavior due to floating-point +demotions. + +.. + +.. bpo: 30465 +.. date: 2017-09-06-10-47-29 +.. nonce: oe-3GD +.. section: Core and Builtins + +Location information (``lineno`` and ``col_offset``) in f-strings is now +(mostly) correct. This fixes tools like flake8 from showing warnings on the +wrong line (typically the first line of the file). + +.. + +.. bpo: 31343 +.. date: 2017-09-04-14-57-27 +.. nonce: Kl_fS5 +.. section: Core and Builtins + +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +plans to remove the functions from sys/types.h. + +.. + +.. bpo: 31291 +.. date: 2017-08-28-11-51-29 +.. nonce: t8QggK +.. section: Core and Builtins + +Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +when the return value of ``pathname.replace('/','\\')`` isn't a string. +Patch by Oren Milman. + +.. + +.. bpo: 31271 +.. date: 2017-08-25-20-43-22 +.. nonce: YMduKF +.. section: Core and Builtins + +Fix an assertion failure in the write() method of `io.TextIOWrapper`, when +the encoder doesn't return a bytes object. Patch by Oren Milman. + +.. + +.. bpo: 31243 +.. date: 2017-08-24-13-34-49 +.. nonce: dRJzqR +.. section: Core and Builtins + +Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +is invalid. Patch by Oren Milman. + +.. + +.. bpo: 30721 +.. date: 2017-08-18-15-15-20 +.. nonce: Hmc56z +.. section: Core and Builtins + +``print`` now shows correct usage hint for using Python 2 redirection +syntax. Patch by Sanyam Khurana. + +.. + +.. bpo: 31070 +.. date: 2017-08-09-09-40-54 +.. nonce: oDyLiI +.. section: Core and Builtins + +Fix a race condition in importlib _get_module_lock(). + +.. + +.. bpo: 31095 +.. date: 2017-08-01-18-48-30 +.. nonce: bXWZDb +.. section: Core and Builtins + +Fix potential crash during GC caused by ``tp_dealloc`` which doesn't call +``PyObject_GC_UnTrack()``. + +.. + +.. bpo: 31071 +.. date: 2017-07-31-13-28-53 +.. nonce: P9UBDy +.. section: Core and Builtins + +Avoid masking original TypeError in call with * unpacking when other +arguments are passed. + +.. + +.. bpo: 30978 +.. date: 2017-07-21-07-39-05 +.. nonce: f0jODc +.. section: Core and Builtins + +str.format_map() now passes key lookup exceptions through. Previously any +exception was replaced with a KeyError exception. + +.. + +.. bpo: 30808 +.. date: 2017-07-17-12-12-59 +.. nonce: bA3zOv +.. section: Core and Builtins + +Use _Py_atomic API for concurrency-sensitive signal state. + +.. + +.. bpo: 30876 +.. date: 2017-07-11-06-31-32 +.. nonce: x35jZX +.. section: Core and Builtins + +Relative import from unloaded package now reimports the package instead of +failing with SystemError. Relative import from non-package now fails with +ImportError rather than SystemError. + +.. + +.. bpo: 30703 +.. date: 2017-06-28-21-07-32 +.. nonce: ULCdFp +.. section: Core and Builtins + +Improve signal delivery. + +Avoid using Py_AddPendingCall from signal handler, to avoid calling signal- +unsafe functions. The tests I'm adding here fail without the rest of the +patch, on Linux and OS X. This means our signal delivery logic had defects +(some signals could be lost). + +.. + +.. bpo: 30765 +.. date: 2017-06-26-14-29-50 +.. nonce: Q5iBmf +.. section: Core and Builtins + +Avoid blocking in pthread_mutex_lock() when PyThread_acquire_lock() is asked +not to block. + +.. + +.. bpo: 31161 +.. date: 06 +.. nonce: FcUAA0 +.. section: Core and Builtins + +Make sure the 'Missing parentheses' syntax error message is only applied to +SyntaxError, not to subclasses. Patch by Martijn Pieters. + +.. + +.. bpo: 30814 +.. date: 05 +.. nonce: HcYsfM +.. section: Core and Builtins + +Fixed a race condition when import a submodule from a package. + +.. + +.. bpo: 30597 +.. date: 04 +.. nonce: 7erHiP +.. section: Core and Builtins + +``print`` now shows expected input in custom error message when used as a +Python 2 statement. Patch by Sanyam Khurana. + +.. + +.. bpo: 31499 +.. date: 2017-09-18-10-57-04 +.. nonce: BydYhf +.. section: Library + +xml.etree: Fix a crash when a parser is part of a reference cycle. + +.. + +.. bpo: 28556 +.. date: 2017-09-14-11-02-56 +.. nonce: EUOiYs +.. section: Library + +typing.get_type_hints now finds the right globalns for classes and modules +by default (when no ``globalns`` was specified by the caller). + +.. + +.. bpo: 28556 +.. date: 2017-09-13-23-27-39 +.. nonce: UmTQvv +.. section: Library + +Speed improvements to the ``typing`` module. Original PRs by Ivan +Levkivskyi and Mitar. + +.. + +.. bpo: 31544 +.. date: 2017-09-13-19-55-35 +.. nonce: beTh6t +.. section: Library + +The C accelerator module of ElementTree ignored exceptions raised when +looking up TreeBuilder target methods in XMLParser(). + +.. + +.. bpo: 31234 +.. date: 2017-09-13-18-05-56 +.. nonce: lGkcPg +.. section: Library + +socket.create_connection() now fixes manually a reference cycle: clear the +variable storing the last exception on success. + +.. + +.. bpo: 31457 +.. date: 2017-09-13-13-33-39 +.. nonce: bIVBtI +.. section: Library + +LoggerAdapter objects can now be nested. + +.. + +.. bpo: 31400 +.. date: 2017-09-08-14-19-57 +.. nonce: YOTPKi +.. section: Library + +Improves SSL error handling to avoid losing error numbers. + +.. + +.. bpo: 28958 +.. date: 2017-09-06-19-41-01 +.. nonce: x4-K5F +.. section: Library + +ssl.SSLContext() now uses OpenSSL error information when a context cannot be +instantiated. + +.. + +.. bpo: 27340 +.. date: 2017-09-06-06-50-41 +.. nonce: GgekV5 +.. section: Library + +SSLSocket.sendall() now uses memoryview to create slices of data. This fixes +support for all bytes-like object. It is also more efficient and avoids +costly copies. + +.. + +.. bpo: 31178 +.. date: 2017-09-05-14-55-28 +.. nonce: JrSFo7 +.. section: Library + +Fix string concatenation bug in rare error path in the subprocess module + +.. + +.. bpo: 31350 +.. date: 2017-09-05-10-30-48 +.. nonce: dXJ-7N +.. section: Library + +Micro-optimize :func:`asyncio._get_running_loop` to become up to 10% faster. + +.. + +.. bpo: 31170 +.. date: 2017-09-04-23-41-35 +.. nonce: QGmJ1t +.. section: Library + +expat: Update libexpat from 2.2.3 to 2.2.4. Fix copying of partial +characters for UTF-8 input (libexpat bug 115): +https://github.com/libexpat/libexpat/issues/115 + +.. + +.. bpo: 29136 +.. date: 2017-09-04-16-39-49 +.. nonce: vSn1oR +.. section: Library + +Add TLS 1.3 cipher suites and OP_NO_TLSv1_3. + +.. + +.. bpo: 29212 +.. date: 2017-09-03-14-31-00 +.. nonce: bicycl +.. section: Library + +Fix concurrent.futures.thread.ThreadPoolExecutor threads to have a non +repr() based thread name by default when no thread_name_prefix is supplied. +They will now identify themselves as "ThreadPoolExecutor-y_n". + +.. + +.. bpo: 9146 +.. date: 2017-09-03-14-10-00 +.. nonce: _-oo-_ +.. section: Library + +Fix a segmentation fault in _hashopenssl when standard hash functions such +as md5 are not available in the linked OpenSSL library. As in some special +FIPS-140 build environments. + +.. + +.. bpo: 27144 +.. date: 2017-08-30-11-26-14 +.. nonce: PEDJsE +.. section: Library + +The ``map()`` and ``as_completed()`` iterators in ``concurrent.futures`` now +avoid keeping a reference to yielded objects. + +.. + +.. bpo: 10746 +.. date: 2017-08-28-13-01-05 +.. nonce: nmAvfu +.. section: Library + +Fix ctypes producing wrong PEP 3118 type codes for integer types. + +.. + +.. bpo: 22536 +.. date: 2017-08-23 +.. nonce: _narf_ +.. section: Library + +The subprocess module now sets the filename when FileNotFoundError is raised +on POSIX systems due to the executable or cwd not being found. + +.. + +.. bpo: 31249 +.. date: 2017-08-22-12-44-48 +.. nonce: STPbb9 +.. section: Library + +concurrent.futures: WorkItem.run() used by ThreadPoolExecutor now breaks a +reference cycle between an exception object and the WorkItem object. + +.. + +.. bpo: 31247 +.. date: 2017-08-21-17-50-27 +.. nonce: 8S3zJp +.. section: Library + +xmlrpc.server now explicitly breaks reference cycles when using +sys.exc_info() in code handling exceptions. + +.. + +.. bpo: 30102 +.. date: 2017-08-16-21-14-31 +.. nonce: 1sPqmc +.. section: Library + +The ssl and hashlib modules now call OPENSSL_add_all_algorithms_noconf() on +OpenSSL < 1.1.0. The function detects CPU features and enables optimizations +on some CPU architectures such as POWER8. Patch is based on research from +Gustavo Serra Scalet. + +.. + +.. bpo: 31185 +.. date: 2017-08-11-19-30-00 +.. nonce: i6TPgL +.. section: Library + +Fixed miscellaneous errors in asyncio speedup module. + +.. + +.. bpo: 31135 +.. date: 2017-08-08-14-44-37 +.. nonce: HH94xR +.. section: Library + +ttk: fix the destroy() method of LabeledScale and OptionMenu classes. Call +the parent destroy() method even if the used attribute doesn't exist. The +LabeledScale.destroy() method now also explicitly clears label and scale +attributes to help the garbage collector to destroy all widgets. + +.. + +.. bpo: 31107 +.. date: 2017-08-02-12-48-15 +.. nonce: 1t2hn5 +.. section: Library + +Fix `copyreg._slotnames()` mangled attribute calculation for classes whose +name begins with an underscore. Patch by Shane Harvey. + +.. + +.. bpo: 31061 +.. date: 2017-08-01-09-32-58 +.. nonce: husAYX +.. section: Library + +Fixed a crash when using asyncio and threads. + +.. + +.. bpo: 30502 +.. date: 2017-07-27-11-33-58 +.. nonce: GJlfU8 +.. section: Library + +Fix handling of long oids in ssl. Based on patch by Christian Heimes. + +.. + +.. bpo: 30119 +.. date: 2017-07-26-15-15-00 +.. nonce: DZ6C_S +.. section: Library + +ftplib.FTP.putline() now throws ValueError on commands that contains CR or +LF. Patch by Dong-hee Na. + +.. + +.. bpo: 30595 +.. date: 2017-07-26-04-46-12 +.. nonce: -zJ7d8 +.. section: Library + +multiprocessing.Queue.get() with a timeout now polls its reader in non- +blocking mode if it succeeded to aquire the lock but the acquire took longer +than the timeout. + +.. + +.. bpo: 29403 +.. date: 2017-07-20-02-29-49 +.. nonce: 3RinCV +.. section: Library + +Fix ``unittest.mock``'s autospec to not fail on method-bound builtin +functions. Patch by Aaron Gallagher. + +.. + +.. bpo: 30961 +.. date: 2017-07-18-23-47-51 +.. nonce: 064jz0 +.. section: Library + +Fix decrementing a borrowed reference in tracemalloc. + +.. + +.. bpo: 25684 +.. date: 2017-07-17-11-35-00 +.. nonce: usELVx +.. section: Library + +Change ``ttk.OptionMenu`` radiobuttons to be unique across instances of +``OptionMenu``. + +.. + +.. bpo: 30886 +.. date: 2017-07-10-12-14-22 +.. nonce: nqQj34 +.. section: Library + +Fix multiprocessing.Queue.join_thread(): it now waits until the thread +completes, even if the thread was started by the same process which created +the queue. + +.. + +.. bpo: 29854 +.. date: 2017-07-07-02-18-57 +.. nonce: J8wKb_ +.. section: Library + +Fix segfault in readline when using readline's history-size option. Patch +by Nir Soffer. + +.. + +.. bpo: 30319 +.. date: 2017-07-04-13-48-21 +.. nonce: hg_3TX +.. section: Library + +socket.close() now ignores ECONNRESET error. + +.. + +.. bpo: 30828 +.. date: 2017-07-04-13-10-52 +.. nonce: CLvEvV +.. section: Library + +Fix out of bounds write in `asyncio.CFuture.remove_done_callback()`. + +.. + +.. bpo: 30807 +.. date: 2017-06-29-22-04-44 +.. nonce: sLtjY- +.. section: Library + +signal.setitimer() may disable the timer when passed a tiny value. + +Tiny values (such as 1e-6) are valid non-zero values for setitimer(), which +is specified as taking microsecond-resolution intervals. However, on some +platform, our conversion routine could convert 1e-6 into a zero interval, +therefore disabling the timer instead of (re-)scheduling it. + +.. + +.. bpo: 30441 +.. date: 2017-06-29-14-25-14 +.. nonce: 3Wh9kc +.. section: Library + +Fix bug when modifying os.environ while iterating over it + +.. + +.. bpo: 30532 +.. date: 2017-06-26-11-01-59 +.. nonce: qTeL1o +.. section: Library + +Fix email header value parser dropping folding white space in certain cases. + +.. + +.. bpo: 30879 +.. date: 03 +.. nonce: N3KI-o +.. section: Library + +os.listdir() and os.scandir() now emit bytes names when called with bytes- +like argument. + +.. + +.. bpo: 30746 +.. date: 02 +.. nonce: 7drQI0 +.. section: Library + +Prohibited the '=' character in environment variable names in +``os.putenv()`` and ``os.spawn*()``. + +.. + +.. bpo: 29755 +.. date: 01 +.. nonce: diQcY_ +.. section: Library + +Fixed the lgettext() family of functions in the gettext module. They now +always return bytes. + +.. + +.. bpo: 31294 +.. date: 2017-09-07-20-49-09 +.. nonce: WgI18w +.. section: Documentation + +Fix incomplete code snippet in the ZeroMQSocketListener and +ZeroMQSocketHandler examples and adapt them to Python 3. + +.. + +.. bpo: 21649 +.. date: 2017-09-06-10-11-57 +.. nonce: EUvqA9 +.. section: Documentation + +Add RFC 7525 and Mozilla server side TLS links to SSL documentation. + +.. + +.. bpo: 30803 +.. date: 2017-07-29-14-55-50 +.. nonce: 6hutqQ +.. section: Documentation + +Clarify doc on truth value testing. Original patch by Peter Thomassen. + +.. + +.. bpo: 31320 +.. date: 2017-09-05-14-23-35 +.. nonce: JRDHx7 +.. section: Tests + +Silence traceback in test_ssl + +.. + +.. bpo: 25674 +.. date: 2017-09-04-13-03-55 +.. nonce: whVTXh +.. section: Tests + +Remove sha256.tbs-internet.com ssl test + +.. + +.. bpo: 30715 +.. date: 2017-07-25-15-27-44 +.. nonce: Sp7bTF +.. section: Tests + +Address ALPN callback changes for OpenSSL 1.1.0f. The latest version behaves +like OpenSSL 1.0.2 and no longer aborts handshake. + +.. + +.. bpo: 30822 +.. date: 2017-07-20-14-29-54 +.. nonce: X0wREo +.. section: Tests + +regrtest: Exclude tzdata from regrtest --all. When running the test suite +using --use=all / -u all, exclude tzdata since it makes test_datetime too +slow (15-20 min on some buildbots) which then times out on some buildbots. +Fix also regrtest command line parser to allow passing -u extralargefile to +run test_zipfile64. + +.. + +.. bpo: 30854 +.. date: 2017-07-05-16-54-59 +.. nonce: sPADRI +.. section: Build + +Fix compile error when compiling --without-threads. Patch by Masayuki +Yamamoto. + +.. + +.. bpo: 30389 +.. date: 2017-09-06-17-14-54 +.. nonce: 9Dizrx +.. section: Windows + +Adds detection of Visual Studio 2017 to distutils on Windows. + +.. + +.. bpo: 31340 +.. date: 2017-09-04-13-19-05 +.. nonce: MbkzLi +.. section: Windows + +Change to building with MSVC v141 (included with Visual Studio 2017) + +.. + +.. bpo: 30581 +.. date: 2017-08-04-10-05-19 +.. nonce: OQhR7l +.. section: Windows + +os.cpu_count() now returns the correct number of processors on Windows when +the number of logical processors is greater than 64. + +.. + +.. bpo: 30731 +.. date: 2017-07-13-11-22-53 +.. nonce: nmMDwI +.. section: Windows + +Add a missing xmlns to python.manifest so that it matches the schema. + +.. + +.. bpo: 31493 +.. date: 2017-09-16-23-43-39 +.. nonce: nmHMCR +.. section: IDLE + +IDLE code context -- fix code update and font update timers. + +Canceling timers prevents a warning message when test_idle completes. + +.. + +.. bpo: 31488 +.. date: 2017-09-16-01-21-20 +.. nonce: 0rtXIT +.. section: IDLE + +IDLE - Update non-key options in former extension classes. When applying +configdialog changes, call .reload for each feature class. Change ParenMatch +so updated options affect existing instances attached to existing editor +windows. + +.. + +.. bpo: 31477 +.. date: 2017-09-15-12-38-47 +.. nonce: n__6sa +.. section: IDLE + +IDLE - Improve rstrip entry in doc. Strip trailing whitespace strips more +than blank spaces. Multiline string literals are not skipped. + +.. + +.. bpo: 31480 +.. date: 2017-09-14-17-53-53 +.. nonce: 4WJ0pl +.. section: IDLE + +IDLE - make tests pass with zzdummy extension disabled by default. + +.. + +.. bpo: 31421 +.. date: 2017-09-12-08-38-27 +.. nonce: mYfQNq +.. section: IDLE + +Document how IDLE runs tkinter programs. IDLE calls tcl/tk update in the +background in order to make live + +interaction and experimentatin with tkinter applications much easier. + +.. + +.. bpo: 31414 +.. date: 2017-09-11-15-46-05 +.. nonce: wiepgK +.. section: IDLE + +IDLE -- fix tk entry box tests by deleting first. Adding to an int entry is +not the same as deleting and inserting because int('') will fail. + +.. + +.. bpo: 31051 +.. date: 2017-08-30-00-06-58 +.. nonce: 50Jp_Q +.. section: IDLE + +Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections. + +.. + +.. bpo: 30617 +.. date: 2017-08-27-16-49-36 +.. nonce: UHnswr +.. section: IDLE + +IDLE - Add docstrings and tests for outwin subclass of editor. + +Move some data and functions from the class to module level. Patch by Cheryl +Sabella. + +.. + +.. bpo: 31287 +.. date: 2017-08-27-15-31-33 +.. nonce: aZERfI +.. section: IDLE + +IDLE - Do not modify tkinter.message in test_configdialog. + +.. + +.. bpo: 27099 +.. date: 2017-08-24-13-48-16 +.. nonce: rENefC +.. section: IDLE + +Convert IDLE's built-in 'extensions' to regular features. + +About 10 IDLE features were implemented as supposedly optional extensions. +Their different behavior could be confusing or worse for users and not good +for maintenance. Hence the conversion. + +The main difference for users is that user configurable key bindings for +builtin features are now handled uniformly. Now, editing a binding in a +keyset only affects its value in the keyset. All bindings are defined +together in the system-specific default keysets in config- extensions.def. +All custom keysets are saved as a whole in config- extension.cfg. All take +effect as soon as one clicks Apply or Ok. + +The affected events are '<>', '<>', +'<>', '<>', '<>', '<>', '<>', and '<>'. Any (global) +customizations made before 3.6.3 will not affect their keyset- specific +customization after 3.6.3. and vice versa. + +Inital patch by Charles Wohlganger. + +.. + +.. bpo: 31206 +.. date: 2017-08-18-14-13-42 +.. nonce: F1-tKK +.. section: IDLE + +IDLE: Factor HighPage(Frame) class from ConfigDialog. Patch by Cheryl +Sabella. + +.. + +.. bpo: 31001 +.. date: 2017-08-17-15-00-20 +.. nonce: KLxYHC +.. section: IDLE + +Add tests for configdialog highlight tab. Patch by Cheryl Sabella. + +.. + +.. bpo: 31205 +.. date: 2017-08-15-12-58-23 +.. nonce: iuziZ5 +.. section: IDLE + +IDLE: Factor KeysPage(Frame) class from ConfigDialog. The slightly modified +tests continue to pass. Patch by Cheryl Sabella. + +.. + +.. bpo: 31130 +.. date: 2017-08-07-14-02-56 +.. nonce: FbsC7f +.. section: IDLE + +IDLE -- stop leaks in test_configdialog. Initial patch by Victor Stinner. + +.. + +.. bpo: 31002 +.. date: 2017-08-03-17-54-02 +.. nonce: kUSgTE +.. section: IDLE + +Add tests for configdialog keys tab. Patch by Cheryl Sabella. + +.. + +.. bpo: 19903 +.. date: 2017-08-03-14-08-42 +.. nonce: sqE1FS +.. section: IDLE + +IDLE: Calltips use `inspect.signature` instead of `inspect.getfullargspec`. +This improves calltips for builtins converted to use Argument Clinic. Patch +by Louie Lu. + +.. + +.. bpo: 31083 +.. date: 2017-07-31-23-20-51 +.. nonce: 991FXm +.. section: IDLE + +IDLE - Add an outline of a TabPage class in configdialog. Update existing +classes to match outline. Initial patch by Cheryl Sabella. + +.. + +.. bpo: 31050 +.. date: 2017-07-30-17-39-59 +.. nonce: AXR3kP +.. section: IDLE + +Factor GenPage(Frame) class from ConfigDialog. The slightly modified tests +continue to pass. Patch by Cheryl Sabella. + +.. + +.. bpo: 31004 +.. date: 2017-07-30-01-00-58 +.. nonce: m8cc1t +.. section: IDLE + +IDLE - Factor FontPage(Frame) class from ConfigDialog. + +Slightly modified tests continue to pass. Fix General tests. Patch mostly by +Cheryl Sabella. + +.. + +.. bpo: 30781 +.. date: 2017-07-28-18-59-06 +.. nonce: ud5m18 +.. section: IDLE + +IDLE - Use ttk widgets in ConfigDialog. Patches by Terry Jan Reedy and +Cheryl Sabella. + +.. + +.. bpo: 31060 +.. date: 2017-07-27-14-48-42 +.. nonce: GdY_VY +.. section: IDLE + +IDLE - Finish rearranging methods of ConfigDialog Grouping methods +pertaining to each tab and the buttons will aid writing tests and improving +the tabs and will enable splitting the groups into classes. + +.. + +.. bpo: 30853 +.. date: 2017-07-27-10-01-14 +.. nonce: enPvvc +.. section: IDLE + +IDLE -- Factor a VarTrace class out of ConfigDialog. + +Instance tracers manages pairs consisting of a tk variable and a callback +function. When tracing is turned on, setting the variable calls the +function. Test coverage for the new class is 100%. + +.. + +.. bpo: 31003 +.. date: 2017-07-25-01-28-35 +.. nonce: bYINVH +.. section: IDLE + +IDLE: Add more tests for General tab. + +.. + +.. bpo: 30993 +.. date: 2017-07-22-18-08-41 +.. nonce: 34vJkB +.. section: IDLE + +IDLE - Improve configdialog font page and tests. + +In configdialog: Document causal pathways in create_font_tab docstring. +Simplify some attribute names. Move set_samples calls to var_changed_font +(idea from Cheryl Sabella). Move related functions to positions after the +create widgets function. + +In test_configdialog: Fix test_font_set so not order dependent. Fix renamed +test_indent_scale so it tests the widget. Adjust tests for movement of +set_samples call. Add tests for load functions. Put all font tests in one +class and tab indent tests in another. Except for two lines, these tests +completely cover the related functions. + +.. + +.. bpo: 30981 +.. date: 2017-07-21-01-55-14 +.. nonce: ZFvQPt +.. section: IDLE + +IDLE -- Add more configdialog font page tests. + +.. + +.. bpo: 28523 +.. date: 2017-07-21-00-54-52 +.. nonce: OPcqYJ +.. section: IDLE + +IDLE: replace 'colour' with 'color' in configdialog. + +.. + +.. bpo: 30917 +.. date: 2017-07-17-23-35-57 +.. nonce: hSiuuO +.. section: IDLE + +Add tests for idlelib.config.IdleConf. Increase coverage from 46% to 96%. +Patch by Louie Lu. + +.. + +.. bpo: 30934 +.. date: 2017-07-15-22-26-57 +.. nonce: BanuSB +.. section: IDLE + +Document coverage details for idlelib tests. + +* Add section to idlelib/idle-test/README.txt. + +* Include check that branches are taken both ways. + +* Exclude IDLE-specific code that does not run during unit tests. + +.. + +.. bpo: 30913 +.. date: 2017-07-13-23-07-33 +.. nonce: aezn_e +.. section: IDLE + +IDLE: Document ConfigDialog tk Vars, methods, and widgets in docstrings This +will facilitate improving the dialog and splitting up the class. Original +patch by Cheryl Sabella. + +.. + +.. bpo: 30899 +.. date: 2017-07-11-02-26-17 +.. nonce: SQmVO8 +.. section: IDLE + +IDLE: Add tests for ConfigParser subclasses in config. Patch by Louie Lu. + +.. + +.. bpo: 30881 +.. date: 2017-07-11-02-21-42 +.. nonce: 4KAq_9 +.. section: IDLE + +IDLE: Add docstrings to browser.py. Patch by Cheryl Sabella. + +.. + +.. bpo: 30851 +.. date: 2017-07-09-23-53-00 +.. nonce: AHXBYa +.. section: IDLE + +IDLE: Remove unused variables in configdialog. One is a duplicate, one is +set but cannot be altered by users. Patch by Cheryl Sabella. + +.. + +.. bpo: 30870 +.. date: 2017-07-08-17-57-04 +.. nonce: IcR2pf +.. section: IDLE + +IDLE: In Settings dialog, select font with Up, Down keys as well as mouse. +Initial patch by Louie Lu. + +.. + +.. bpo: 8231 +.. date: 2017-07-07-21-10-55 +.. nonce: yEge3L +.. section: IDLE + +IDLE: call config.IdleConf.GetUserCfgDir only once. + +.. + +.. bpo: 30779 +.. date: 2017-07-07-20-26-37 +.. nonce: 8KXEXN +.. section: IDLE + +IDLE: Factor ConfigChanges class from configdialog, put in config; test. * +In config, put dump test code in a function; run it and unittest in 'if +__name__ == '__main__'. * Add class config.ConfigChanges based on +changes_class_v4.py on bpo issue. * Add class test_config.ChangesTest, +partly using configdialog_tests_v1.py. * Revise configdialog to use +ConfigChanges; see tracker msg297804. * Revise test_configdialog to match +configdialog changes. * Remove configdialog functions unused or moved to +ConfigChanges. Cheryl Sabella contributed parts of the patch. + +.. + +.. bpo: 30777 +.. date: 2017-07-04-22-45-46 +.. nonce: uxzlMB +.. section: IDLE + +IDLE: configdialog - Add docstrings and fix comments. Patch by Cheryl +Sabella. + +.. + +.. bpo: 30495 +.. date: 2017-06-29-18-23-06 +.. nonce: qIWgc4 +.. section: IDLE + +IDLE: Improve textview with docstrings, PEP8 names, and more tests. Patch by +Cheryl Sabella. + +.. + +.. bpo: 30723 +.. date: 2017-06-27-19-05-40 +.. nonce: rQh06y +.. section: IDLE + +IDLE: Make several improvements to parenmatch. Add 'parens' style to +highlight both opener and closer. Make 'default' style, which is not +default, a synonym for 'opener'. Make time-delay work the same with all +styles. Add help for config dialog extensions tab, including help for +parenmatch. Add new tests. Original patch by Charles Wohlganger. + +.. + +.. bpo: 30674 +.. date: 2017-06-27-01-40-34 +.. nonce: ppK_q8 +.. section: IDLE + +IDLE: add docstrings to grep module. Patch by Cheryl Sabella + +.. + +.. bpo: 21519 +.. date: 2017-06-27-00-29-56 +.. nonce: fTj9T0 +.. section: IDLE + +IDLE's basic custom key entry dialog now detects duplicates properly. +Original patch by Saimadhav Heblikar. + +.. + +.. bpo: 29910 +.. date: 2017-06-26-22-45-27 +.. nonce: mqHh7u +.. section: IDLE + +IDLE no longer deletes a character after commenting out a region by a key +shortcut. Add ``return 'break'`` for this and other potential conflicts +between IDLE and default key bindings. + +.. + +.. bpo: 30728 +.. date: 2017-06-26-15-47-13 +.. nonce: qH4TGL +.. section: IDLE + +Review and change idlelib.configdialog names. Lowercase method and attribute +names. Replace 'colour' with 'color', expand overly cryptic names, delete +unneeded underscores. Replace ``import *`` with specific imports. Patches by +Cheryl Sabella. + +.. + +.. bpo: 6739 +.. date: 2017-06-26-00-28-59 +.. nonce: x5MfhB +.. section: IDLE + +IDLE: Verify user-entered key sequences by trying to bind them with tk. Add +tests for all 3 validation functions. Original patch by G Polo. Tests added +by Cheryl Sabella. + +.. + +.. bpo: 30983 +.. date: 2017-08-18-17-19-23 +.. nonce: ggGz9z +.. section: Tools/Demos + +gdb integration commands (py-bt, etc.) work on optimized shared builds now, +too. PEP 523 introduced _PyEval_EvalFrameDefault which inlines +PyEval_EvalFrameEx on non-debug shared builds. This broke the ability to +use py-bt, py-up, and a few other Python-specific gdb integrations. The +problem is fixed by only looking for _PyEval_EvalFrameDefault frames in +python-gdb.py. Original patch by Bruno "Polaco" Penteado. diff --git a/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst b/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst new file mode 100644 index 00000000000000..414f1a45eab8b3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst @@ -0,0 +1 @@ +Avoid wholesale rebuild after `make regen-all` if nothing changed. diff --git a/Misc/NEWS.d/next/Build/2017-09-26-22-39-58.bpo-22140.ZRf7Wn.rst b/Misc/NEWS.d/next/Build/2017-09-26-22-39-58.bpo-22140.ZRf7Wn.rst new file mode 100644 index 00000000000000..e20c0942f6c94e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-09-26-22-39-58.bpo-22140.ZRf7Wn.rst @@ -0,0 +1 @@ +Prevent double substitution of prefix in python-config.sh. diff --git a/Misc/NEWS.d/next/Build/README.rst b/Misc/NEWS.d/next/Build/README.rst new file mode 100644 index 00000000000000..0d2d2c1fc38be9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Build* section in this directory. diff --git a/Misc/NEWS.d/next/C API/2017-09-20-21-59-52.bpo-31532.s9Cw9_.rst b/Misc/NEWS.d/next/C API/2017-09-20-21-59-52.bpo-31532.s9Cw9_.rst new file mode 100644 index 00000000000000..7451986846e118 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-09-20-21-59-52.bpo-31532.s9Cw9_.rst @@ -0,0 +1,2 @@ +Fix memory corruption due to allocator mix in getpath.c between Py_GetPath() +and Py_SetPath() diff --git a/Misc/NEWS.d/next/C API/README.rst b/Misc/NEWS.d/next/C API/README.rst new file mode 100644 index 00000000000000..5a04f76f47b67e --- /dev/null +++ b/Misc/NEWS.d/next/C API/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *C API* section in this directory. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-27-21-18-30.bpo-31285.7lzaKV.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-27-21-18-30.bpo-31285.7lzaKV.rst new file mode 100644 index 00000000000000..61f2c4eff65a71 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-27-21-18-30.bpo-31285.7lzaKV.rst @@ -0,0 +1,3 @@ +Fix an assertion failure in `warnings.warn_explicit`, when the return value +of the received loader's get_source() has a bad splitlines() method. Patch +by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-28-17-51-42.bpo-31293.eMYZXj.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-28-17-51-42.bpo-31293.eMYZXj.rst new file mode 100644 index 00000000000000..28b7bfb2ef3722 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-28-17-51-42.bpo-31293.eMYZXj.rst @@ -0,0 +1,2 @@ +Fix crashes in true division and multiplication of a timedelta object by a +float with a bad as_integer_ratio() method. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-31-17-52-56.bpo-31311.bNE2l-.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-31-17-52-56.bpo-31311.bNE2l-.rst new file mode 100644 index 00000000000000..db51cd2d586e0f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-31-17-52-56.bpo-31311.bNE2l-.rst @@ -0,0 +1,2 @@ +Fix a crash in the ``__setstate__()`` method of `ctypes._CData`, in case of +a bad ``__dict__``. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-01-00-40-58.bpo-31315.ZX20bl.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-01-00-40-58.bpo-31315.ZX20bl.rst new file mode 100644 index 00000000000000..d13badbb358ff1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-01-00-40-58.bpo-31315.ZX20bl.rst @@ -0,0 +1,2 @@ +Fix an assertion failure in imp.create_dynamic(), when spec.name is not a +string. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-15-09-13-07.bpo-31478.o06iKD.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-15-09-13-07.bpo-31478.o06iKD.rst new file mode 100644 index 00000000000000..bbeb810ebaa248 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-15-09-13-07.bpo-31478.o06iKD.rst @@ -0,0 +1,2 @@ +Fix an assertion failure in `_random.Random.seed()` in case the argument has a +bad ``__abs__()`` method. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-16-13-32-35.bpo-31490.r7m2sj.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-16-13-32-35.bpo-31490.r7m2sj.rst new file mode 100644 index 00000000000000..d95e825f132a94 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-16-13-32-35.bpo-31490.r7m2sj.rst @@ -0,0 +1,3 @@ +Fix an assertion failure in `ctypes` class definition, in case the class has +an attribute whose name is specified in ``_anonymous_`` but not in +``_fields_``. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-16-22-49-16.bpo-31492.RtyteL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-16-22-49-16.bpo-31492.RtyteL.rst new file mode 100644 index 00000000000000..a8704738d1a512 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-16-22-49-16.bpo-31492.RtyteL.rst @@ -0,0 +1,3 @@ +Fix assertion failures in case of failing to import from a module with a bad +``__name__`` attribute, and in case of failing to access an attribute of such +a module. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-18-12-07-39.bpo-31505.VomaFa.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-18-12-07-39.bpo-31505.VomaFa.rst new file mode 100644 index 00000000000000..bad9e51aaf6108 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-18-12-07-39.bpo-31505.VomaFa.rst @@ -0,0 +1,2 @@ +Fix an assertion failure in `json`, in case `_json.make_encoder()` received +a bad `encoder()` argument. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-24-09-57-04.bpo-31566.OxwINs.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-24-09-57-04.bpo-31566.OxwINs.rst new file mode 100644 index 00000000000000..d3ccfd7a242943 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-24-09-57-04.bpo-31566.OxwINs.rst @@ -0,0 +1,2 @@ +Fix an assertion failure in `_warnings.warn()` in case of a bad +``__name__`` global. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-26-13-03-16.bpo-31588.wT9Iy7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-13-03-16.bpo-31588.wT9Iy7.rst new file mode 100644 index 00000000000000..44a7aba9d7f94a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-13-03-16.bpo-31588.wT9Iy7.rst @@ -0,0 +1,2 @@ +Raise a `TypeError` with a helpful error message when class creation fails +due to a metaclass with a bad ``__prepare__()`` method. Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst new file mode 100644 index 00000000000000..29f3461921ed35 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-26-16-05-04.bpo-31592.IFBZj9.rst @@ -0,0 +1,2 @@ +Fixed an assertion failure in Python parser in case of a bad `unicodedata.normalize()`. +Patch by Oren Milman. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-29-20-32-24.bpo-31619.6gQ1kv.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-29-20-32-24.bpo-31619.6gQ1kv.rst new file mode 100644 index 00000000000000..3efcc9d134d31b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-29-20-32-24.bpo-31619.6gQ1kv.rst @@ -0,0 +1,2 @@ +Fixed a ValueError when convert a string with large number of underscores +to integer with binary base. diff --git a/Misc/NEWS.d/next/Core and Builtins/README.rst b/Misc/NEWS.d/next/Core and Builtins/README.rst new file mode 100644 index 00000000000000..52b8c3e6263163 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Core and Builtins* section in this directory. diff --git a/Misc/NEWS.d/next/Documentation/2017-09-14-18-44-50.bpo-30085.0J9w-u.rst b/Misc/NEWS.d/next/Documentation/2017-09-14-18-44-50.bpo-30085.0J9w-u.rst new file mode 100644 index 00000000000000..c8be050009c274 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2017-09-14-18-44-50.bpo-30085.0J9w-u.rst @@ -0,0 +1,2 @@ +The operator functions without double underscores are preferred for clarity. +The one with underscores are only kept for back-compatibility. diff --git a/Misc/NEWS.d/next/Documentation/README.rst b/Misc/NEWS.d/next/Documentation/README.rst new file mode 100644 index 00000000000000..405f0ac01a7202 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Documentation* section in this directory. diff --git a/Misc/NEWS.d/next/IDLE/2017-08-14-15-13-50.bpo-1612262.-x_Oyq.rst b/Misc/NEWS.d/next/IDLE/2017-08-14-15-13-50.bpo-1612262.-x_Oyq.rst new file mode 100644 index 00000000000000..0d4494c16a7a01 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-14-15-13-50.bpo-1612262.-x_Oyq.rst @@ -0,0 +1,3 @@ +IDLE module browser now shows nested classes and functions. +Original patches for code and tests by Guilherme Polo and +Cheryl Sabella, respectively. diff --git a/Misc/NEWS.d/next/IDLE/2017-09-18-10-43-03.bpo-31500.Y_YDxA.rst b/Misc/NEWS.d/next/IDLE/2017-09-18-10-43-03.bpo-31500.Y_YDxA.rst new file mode 100644 index 00000000000000..68d68cb1c86032 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-18-10-43-03.bpo-31500.Y_YDxA.rst @@ -0,0 +1 @@ +Default fonts now are scaled on HiDPI displays. diff --git a/Misc/NEWS.d/next/IDLE/2017-09-22-20-26-23.bpo-31459.L0pnH9.rst b/Misc/NEWS.d/next/IDLE/2017-09-22-20-26-23.bpo-31459.L0pnH9.rst new file mode 100644 index 00000000000000..b53c009dfcd3a8 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-22-20-26-23.bpo-31459.L0pnH9.rst @@ -0,0 +1,7 @@ +Rename IDLE's module browser from Class Browser to Module Browser. +The original module-level class and method browser became a module +browser, with the addition of module-level functions, years ago. +Nested classes and functions were added yesterday. For back- +compatibility, the virtual event <>, which +appears on the Keys tab of the Settings dialog, is not changed. +Patch by Cheryl Sabella. diff --git a/Misc/NEWS.d/next/IDLE/2017-09-23-12-52-24.bpo-31559.ydckYX.rst b/Misc/NEWS.d/next/IDLE/2017-09-23-12-52-24.bpo-31559.ydckYX.rst new file mode 100644 index 00000000000000..424da7a3f17a7e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-23-12-52-24.bpo-31559.ydckYX.rst @@ -0,0 +1 @@ +Remove test order dependence in idle_test.test_browser. diff --git a/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst new file mode 100644 index 00000000000000..cc995862372d05 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-30-13-59-18.bpo-31649.LxN4Vb.rst @@ -0,0 +1 @@ +IDLE - Make _htest, _utest parameters keyword only. diff --git a/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst b/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst new file mode 100644 index 00000000000000..6492115260ad3f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-09-30-19-03-26.bpo-31460.HpveI6.rst @@ -0,0 +1,6 @@ +Simplify the API of IDLE's Module Browser. + +Passing a widget instead of an flist with a root widget opens the option of +creating a browser frame that is only part of a window. Passing a full file +name instead of pieces assumed to come from a .py file opens the possibility +of browsing python files that do not end in .py. diff --git a/Misc/NEWS.d/next/IDLE/README.rst b/Misc/NEWS.d/next/IDLE/README.rst new file mode 100644 index 00000000000000..5475f7b42050cd --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *IDLE* section in this directory. diff --git a/Misc/NEWS.d/next/Library/2017-09-16-01-53-11.bpo-31482.39s5dS.rst b/Misc/NEWS.d/next/Library/2017-09-16-01-53-11.bpo-31482.39s5dS.rst new file mode 100644 index 00000000000000..ed7a4175cd70ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-16-01-53-11.bpo-31482.39s5dS.rst @@ -0,0 +1 @@ +``random.seed()`` now works with bytes in version=1 diff --git a/Misc/NEWS.d/next/Library/2017-09-17-15-24-25.bpo-31351.yQdKv-.rst b/Misc/NEWS.d/next/Library/2017-09-17-15-24-25.bpo-31351.yQdKv-.rst new file mode 100644 index 00000000000000..20f2c1bdc117f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-17-15-24-25.bpo-31351.yQdKv-.rst @@ -0,0 +1,2 @@ +python -m ensurepip now exits with non-zero exit code if pip bootstrapping +has failed. diff --git a/Misc/NEWS.d/next/Library/2017-09-20-18-43-01.bpo-31516.23Yuq3.rst b/Misc/NEWS.d/next/Library/2017-09-20-18-43-01.bpo-31516.23Yuq3.rst new file mode 100644 index 00000000000000..af48d159fef43c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-20-18-43-01.bpo-31516.23Yuq3.rst @@ -0,0 +1 @@ +``threading.current_thread()`` should not return a dummy thread at shutdown. diff --git a/Misc/NEWS.d/next/Library/2017-09-25-14-04-30.bpo-30347.B4--_D.rst b/Misc/NEWS.d/next/Library/2017-09-25-14-04-30.bpo-30347.B4--_D.rst new file mode 100644 index 00000000000000..859c64155297bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-25-14-04-30.bpo-30347.B4--_D.rst @@ -0,0 +1 @@ +Stop crashes when concurrently iterate over itertools.groupby() iterators. diff --git a/Misc/NEWS.d/next/Library/2017-09-27-08-11-38.bpo-25532.ey4Yez.rst b/Misc/NEWS.d/next/Library/2017-09-27-08-11-38.bpo-25532.ey4Yez.rst new file mode 100644 index 00000000000000..8146dcdc6d57ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-27-08-11-38.bpo-25532.ey4Yez.rst @@ -0,0 +1,3 @@ +inspect.unwrap() will now only try to unwrap an object +sys.getrecursionlimit() times, to protect against objects which create a new +object on every attribute access. diff --git a/Misc/NEWS.d/next/Library/2017-09-28-23-10-51.bpo-25351.2JmFpF.rst b/Misc/NEWS.d/next/Library/2017-09-28-23-10-51.bpo-25351.2JmFpF.rst new file mode 100644 index 00000000000000..f7afc2570aa873 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-28-23-10-51.bpo-25351.2JmFpF.rst @@ -0,0 +1 @@ +Avoid venv activate failures with undefined variables diff --git a/Misc/NEWS.d/next/Library/2017-10-03-14-37-46.bpo-31673.RFCrka.rst b/Misc/NEWS.d/next/Library/2017-10-03-14-37-46.bpo-31673.RFCrka.rst new file mode 100644 index 00000000000000..8fce050c041021 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-03-14-37-46.bpo-31673.RFCrka.rst @@ -0,0 +1 @@ +Fixed typo in the name of Tkinter's method adderrorinfo(). diff --git a/Misc/NEWS.d/next/Library/2017-10-03-15-06-24.bpo-31675.Nh7jJ3.rst b/Misc/NEWS.d/next/Library/2017-10-03-15-06-24.bpo-31675.Nh7jJ3.rst new file mode 100644 index 00000000000000..4e4430773af894 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-03-15-06-24.bpo-31675.Nh7jJ3.rst @@ -0,0 +1,2 @@ +Fixed memory leaks in Tkinter's methods splitlist() and split() when pass a +string larger than 2 GiB. diff --git a/Misc/NEWS.d/next/Library/README.rst b/Misc/NEWS.d/next/Library/README.rst new file mode 100644 index 00000000000000..6d2d30eca0fd05 --- /dev/null +++ b/Misc/NEWS.d/next/Library/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Library* section in this directory. diff --git a/Misc/NEWS.d/next/Security/README.rst b/Misc/NEWS.d/next/Security/README.rst new file mode 100644 index 00000000000000..84c1a3a6ed7a07 --- /dev/null +++ b/Misc/NEWS.d/next/Security/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Security* section in this directory. diff --git a/Misc/NEWS.d/next/Tests/README.rst b/Misc/NEWS.d/next/Tests/README.rst new file mode 100644 index 00000000000000..d2e50e43d84790 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Tests* section in this directory. diff --git a/Misc/NEWS.d/next/Tools-Demos/README.rst b/Misc/NEWS.d/next/Tools-Demos/README.rst new file mode 100644 index 00000000000000..357f82862cb51a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Tools/Demos* section in this directory. diff --git a/Misc/NEWS.d/next/Windows/README.rst b/Misc/NEWS.d/next/Windows/README.rst new file mode 100644 index 00000000000000..1e65de35f0ad97 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *Windows* section in this directory. diff --git a/Misc/NEWS.d/next/macOS/README.rst b/Misc/NEWS.d/next/macOS/README.rst new file mode 100644 index 00000000000000..a3adb59b62631e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/README.rst @@ -0,0 +1 @@ +Put news entry ``blurb`` files for the *macOS* section in this directory. diff --git a/Misc/Porting b/Misc/Porting index c43b1129788454..f16c460052151c 100644 --- a/Misc/Porting +++ b/Misc/Porting @@ -1 +1 @@ -This document is moved to https://docs.python.org/devguide/faq.html#how-do-i-port-python-to-a-new-platform +This document is moved to https://devguide.python.org/porting/ diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 30c6927871001d..d1d3275fa2758d 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -24,17 +24,18 @@ installed_prefix () echo $RESULT } -prefix_build="@prefix@" prefix_real=$(installed_prefix "$0") # Use sed to fix paths from their built-to locations to their installed-to -# locations. -prefix=$(echo "$prefix_build" | sed "s#$prefix_build#$prefix_real#") -exec_prefix_build="@exec_prefix@" -exec_prefix=$(echo "$exec_prefix_build" | sed "s#$exec_prefix_build#$prefix_real#") -includedir=$(echo "@includedir@" | sed "s#$prefix_build#$prefix_real#") -libdir=$(echo "@libdir@" | sed "s#$prefix_build#$prefix_real#") -CFLAGS=$(echo "@CFLAGS@" | sed "s#$prefix_build#$prefix_real#") +# locations. Keep prefix & exec_prefix using their original values in case +# they are referenced in other configure variables, to prevent double +# substitution, issue #22140. +prefix="@prefix@" +exec_prefix="@exec_prefix@" +exec_prefix_real=${prefix_real} +includedir=$(echo "@includedir@" | sed "s#$prefix#$prefix_real#") +libdir=$(echo "@libdir@" | sed "s#$prefix#$prefix_real#") +CFLAGS=$(echo "@CFLAGS@" | sed "s#$prefix#$prefix_real#") VERSION="@VERSION@" LIBM="@LIBM@" LIBC="@LIBC@" @@ -47,8 +48,8 @@ LINKFORSHARED="@LINKFORSHARED@" OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" -LIBDEST=${prefix}/lib/python${VERSION} -LIBPL=$(echo "@LIBPL@" | sed "s#$prefix_build#$prefix_real#") +LIBDEST=${prefix_real}/lib/python${VERSION} +LIBPL=$(echo "@LIBPL@" | sed "s#$prefix#$prefix_real#") SO="@EXT_SUFFIX@" PYTHONFRAMEWORK="@PYTHONFRAMEWORK@" INCDIR="-I$includedir/python${VERSION}${ABIFLAGS}" @@ -73,10 +74,10 @@ for ARG in "$@" do case "$ARG" in --prefix) - echo "$prefix" + echo "$prefix_real" ;; --exec-prefix) - echo "$exec_prefix" + echo "$exec_prefix_real" ;; --includes) echo "$INCDIR $PLATINCDIR" diff --git a/Misc/python.man b/Misc/python.man index 385b6546c8a52b..075b974e904955 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -438,7 +438,7 @@ Main website: https://www.python.org/ .br Documentation: https://docs.python.org/ .br -Developer resources: https://docs.python.org/devguide/ +Developer resources: https://devguide.python.org/ .br Downloads: https://www.python.org/downloads/ .br diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 8b87fc8143f089..735bacb1b4dfbf 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -250,6 +250,7 @@ _symtable symtablemodule.c #_sha1 sha1module.c #_sha256 sha256module.c #_sha512 sha512module.c +#_sha3 _sha3/sha3module.c # _blake module #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fff90468a5927c..35632a6e6faacb 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -68,7 +68,7 @@ typedef struct { PyObject_HEAD TaskObj *sw_task; PyObject *sw_arg; -} TaskSendMethWrapper; +} TaskStepMethWrapper; typedef struct { PyObject_HEAD @@ -92,11 +92,11 @@ static int future_schedule_callbacks(FutureObj *fut) { Py_ssize_t len; - PyObject* iters; + PyObject *callbacks; int i; if (fut->fut_callbacks == NULL) { - PyErr_SetString(PyExc_RuntimeError, "NULL callbacks"); + PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); return -1; } @@ -105,43 +105,42 @@ future_schedule_callbacks(FutureObj *fut) return 0; } - iters = PyList_GetSlice(fut->fut_callbacks, 0, len); - if (iters == NULL) { + callbacks = PyList_GetSlice(fut->fut_callbacks, 0, len); + if (callbacks == NULL) { return -1; } if (PyList_SetSlice(fut->fut_callbacks, 0, len, NULL) < 0) { - Py_DECREF(iters); + Py_DECREF(callbacks); return -1; } for (i = 0; i < len; i++) { - PyObject *handle = NULL; - PyObject *cb = PyList_GET_ITEM(iters, i); + PyObject *handle; + PyObject *cb = PyList_GET_ITEM(callbacks, i); - handle = _PyObject_CallMethodId( - fut->fut_loop, &PyId_call_soon, "OO", cb, fut, NULL); + handle = _PyObject_CallMethodIdObjArgs(fut->fut_loop, &PyId_call_soon, + cb, fut, NULL); if (handle == NULL) { - Py_DECREF(iters); + Py_DECREF(callbacks); return -1; } - else { - Py_DECREF(handle); - } + Py_DECREF(handle); } - Py_DECREF(iters); + Py_DECREF(callbacks); return 0; } static int future_init(FutureObj *fut, PyObject *loop) { - PyObject *res = NULL; + PyObject *res; + int is_true; _Py_IDENTIFIER(get_debug); - if (loop == NULL || loop == Py_None) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return -1; } @@ -149,25 +148,25 @@ future_init(FutureObj *fut, PyObject *loop) else { Py_INCREF(loop); } - Py_CLEAR(fut->fut_loop); - fut->fut_loop = loop; + Py_XSETREF(fut->fut_loop, loop); res = _PyObject_CallMethodId(fut->fut_loop, &PyId_get_debug, NULL); if (res == NULL) { return -1; } - if (PyObject_IsTrue(res)) { - Py_CLEAR(res); - fut->fut_source_tb = PyObject_CallObject(traceback_extract_stack, NULL); + is_true = PyObject_IsTrue(res); + Py_DECREF(res); + if (is_true < 0) { + return -1; + } + if (is_true) { + Py_XSETREF(fut->fut_source_tb, _PyObject_CallNoArg(traceback_extract_stack)); if (fut->fut_source_tb == NULL) { return -1; } } - else { - Py_CLEAR(res); - } - fut->fut_callbacks = PyList_New(0); + Py_XSETREF(fut->fut_callbacks, PyList_New(0)); if (fut->fut_callbacks == NULL) { return -1; } @@ -183,6 +182,7 @@ future_set_result(FutureObj *fut, PyObject *res) return NULL; } + assert(!fut->fut_result); Py_INCREF(res); fut->fut_result = res; fut->fut_state = STATE_FINISHED; @@ -208,6 +208,11 @@ future_set_exception(FutureObj *fut, PyObject *exc) if (exc_val == NULL) { return NULL; } + if (fut->fut_state != STATE_PENDING) { + Py_DECREF(exc_val); + PyErr_SetString(asyncio_InvalidStateError, "invalid state"); + return NULL; + } } else { exc_val = exc; @@ -226,6 +231,7 @@ future_set_exception(FutureObj *fut, PyObject *exc) return NULL; } + assert(!fut->fut_exception); fut->fut_exception = exc_val; fut->fut_state = STATE_FINISHED; @@ -240,31 +246,14 @@ future_set_exception(FutureObj *fut, PyObject *exc) static int future_get_result(FutureObj *fut, PyObject **result) { - PyObject *exc; - if (fut->fut_state == STATE_CANCELLED) { - exc = _PyObject_CallNoArg(asyncio_CancelledError); - if (exc == NULL) { - return -1; - } - *result = exc; - return 1; + PyErr_SetNone(asyncio_CancelledError); + return -1; } if (fut->fut_state != STATE_FINISHED) { - PyObject *msg = PyUnicode_FromString("Result is not ready."); - if (msg == NULL) { - return -1; - } - - exc = _PyObject_CallArg1(asyncio_InvalidStateError, msg); - Py_DECREF(msg); - if (exc == NULL) { - return -1; - } - - *result = exc; - return 1; + PyErr_SetString(asyncio_InvalidStateError, "Result is not set."); + return -1; } fut->fut_log_tb = 0; @@ -283,17 +272,19 @@ static PyObject * future_add_done_callback(FutureObj *fut, PyObject *arg) { if (fut->fut_state != STATE_PENDING) { - PyObject *handle = _PyObject_CallMethodId( - fut->fut_loop, &PyId_call_soon, "OO", arg, fut, NULL); - + PyObject *handle = _PyObject_CallMethodIdObjArgs(fut->fut_loop, + &PyId_call_soon, + arg, fut, NULL); if (handle == NULL) { return NULL; } - else { - Py_DECREF(handle); - } + Py_DECREF(handle); } else { + if (fut->fut_callbacks == NULL) { + PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); + return NULL; + } int err = PyList_Append(fut->fut_callbacks, arg); if (err != 0) { return NULL; @@ -305,6 +296,8 @@ future_add_done_callback(FutureObj *fut, PyObject *arg) static PyObject * future_cancel(FutureObj *fut) { + fut->fut_log_tb = 0; + if (fut->fut_state != STATE_PENDING) { Py_RETURN_FALSE; } @@ -321,7 +314,7 @@ future_cancel(FutureObj *fut) _asyncio.Future.__init__ * - loop: 'O' = NULL + loop: object = None This class is *almost* compatible with concurrent.futures.Future. @@ -339,7 +332,7 @@ This class is *almost* compatible with concurrent.futures.Future. static int _asyncio_Future___init___impl(FutureObj *self, PyObject *loop) -/*[clinic end generated code: output=9ed75799eaccb5d6 input=8e1681f23605be2d]*/ +/*[clinic end generated code: output=9ed75799eaccb5d6 input=89af317082bc0bf8]*/ { return future_init(self, loop); @@ -417,12 +410,12 @@ _asyncio_Future_exception_impl(FutureObj *self) /*[clinic end generated code: output=88b20d4f855e0710 input=733547a70c841c68]*/ { if (self->fut_state == STATE_CANCELLED) { - PyErr_SetString(asyncio_CancelledError, ""); + PyErr_SetNone(asyncio_CancelledError); return NULL; } if (self->fut_state != STATE_FINISHED) { - PyErr_SetString(asyncio_InvalidStateError, "Result is not ready."); + PyErr_SetString(asyncio_InvalidStateError, "Exception is not set."); return NULL; } @@ -438,7 +431,7 @@ _asyncio_Future_exception_impl(FutureObj *self) /*[clinic input] _asyncio.Future.set_result - res: 'O' + res: object / Mark the future done and set its result. @@ -449,7 +442,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_result(FutureObj *self, PyObject *res) -/*[clinic end generated code: output=a620abfc2796bfb6 input=8619565e0503357e]*/ +/*[clinic end generated code: output=a620abfc2796bfb6 input=5b9dc180f1baa56d]*/ { return future_set_result(self, res); } @@ -457,7 +450,7 @@ _asyncio_Future_set_result(FutureObj *self, PyObject *res) /*[clinic input] _asyncio.Future.set_exception - exception: 'O' + exception: object / Mark the future done and set an exception. @@ -468,7 +461,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_exception(FutureObj *self, PyObject *exception) -/*[clinic end generated code: output=f1c1b0cd321be360 input=1377dbe15e6ea186]*/ +/*[clinic end generated code: output=f1c1b0cd321be360 input=e45b7d7aa71cc66d]*/ { return future_set_exception(self, exception); } @@ -476,7 +469,7 @@ _asyncio_Future_set_exception(FutureObj *self, PyObject *exception) /*[clinic input] _asyncio.Future.add_done_callback - fn: 'O' + fn: object / Add a callback to be run when the future becomes done. @@ -488,7 +481,7 @@ scheduled with call_soon. static PyObject * _asyncio_Future_add_done_callback(FutureObj *self, PyObject *fn) -/*[clinic end generated code: output=819e09629b2ec2b5 input=8cce187e32cec6a8]*/ +/*[clinic end generated code: output=819e09629b2ec2b5 input=8f818b39990b027d]*/ { return future_add_done_callback(self, fn); } @@ -496,7 +489,7 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyObject *fn) /*[clinic input] _asyncio.Future.remove_done_callback - fn: 'O' + fn: object / Remove all instances of a callback from the "call when done" list. @@ -506,11 +499,16 @@ Returns the number of callbacks removed. static PyObject * _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) -/*[clinic end generated code: output=5ab1fb52b24ef31f input=3fedb73e1409c31c]*/ +/*[clinic end generated code: output=5ab1fb52b24ef31f input=0a43280a149d505b]*/ { PyObject *newlist; Py_ssize_t len, i, j=0; + if (self->fut_callbacks == NULL) { + PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); + return NULL; + } + len = PyList_GET_SIZE(self->fut_callbacks); if (len == 0) { return PyLong_FromSsize_t(0); @@ -521,25 +519,34 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) return NULL; } - for (i = 0; i < len; i++) { + for (i = 0; i < PyList_GET_SIZE(self->fut_callbacks); i++) { int ret; PyObject *item = PyList_GET_ITEM(self->fut_callbacks, i); - - if ((ret = PyObject_RichCompareBool(fn, item, Py_EQ)) < 0) { - goto fail; - } + Py_INCREF(item); + ret = PyObject_RichCompareBool(fn, item, Py_EQ); if (ret == 0) { - Py_INCREF(item); - PyList_SET_ITEM(newlist, j, item); - j++; + if (j < len) { + PyList_SET_ITEM(newlist, j, item); + j++; + continue; + } + ret = PyList_Append(newlist, item); + } + Py_DECREF(item); + if (ret < 0) { + goto fail; } } - if (PyList_SetSlice(newlist, j, len, NULL) < 0) { - goto fail; + if (j < len) { + Py_SIZE(newlist) = j; } - if (PyList_SetSlice(self->fut_callbacks, 0, len, newlist) < 0) { - goto fail; + j = PyList_GET_SIZE(newlist); + len = PyList_GET_SIZE(self->fut_callbacks); + if (j != len) { + if (PyList_SetSlice(self->fut_callbacks, 0, len, newlist) < 0) { + goto fail; + } } Py_DECREF(newlist); return PyLong_FromSsize_t(len - j); @@ -638,6 +645,17 @@ FutureObj_get_log_traceback(FutureObj *fut) } } +static int +FutureObj_set_log_traceback(FutureObj *fut, PyObject *val) +{ + int is_true = PyObject_IsTrue(val); + if (is_true < 0) { + return -1; + } + fut->fut_log_tb = is_true; + return 0; +} + static PyObject * FutureObj_get_loop(FutureObj *fut) { @@ -709,7 +727,7 @@ FutureObj_get_state(FutureObj *fut) default: assert (0); } - Py_INCREF(ret); + Py_XINCREF(ret); return ret; } @@ -745,25 +763,14 @@ FutureObj_repr(FutureObj *fut) { _Py_IDENTIFIER(_repr_info); - PyObject *_repr_info = _PyUnicode_FromId(&PyId__repr_info); // borrowed - if (_repr_info == NULL) { - return NULL; - } - - PyObject *rinfo = PyObject_CallMethodObjArgs((PyObject*)fut, _repr_info, - NULL); + PyObject *rinfo = _PyObject_CallMethodIdObjArgs((PyObject*)fut, + &PyId__repr_info, + NULL); if (rinfo == NULL) { return NULL; } - PyObject *sp = PyUnicode_FromString(" "); - if (sp == NULL) { - Py_DECREF(rinfo); - return NULL; - } - - PyObject *rinfo_s = PyUnicode_Join(sp, rinfo); - Py_DECREF(sp); + PyObject *rinfo_s = PyUnicode_Join(NULL, rinfo); Py_DECREF(rinfo); if (rinfo_s == NULL) { return NULL; @@ -773,7 +780,7 @@ FutureObj_repr(FutureObj *fut) PyObject *type_name = PyObject_GetAttrString((PyObject*)Py_TYPE(fut), "__name__"); if (type_name != NULL) { - rstr = PyUnicode_FromFormat("<%S %S>", type_name, rinfo_s); + rstr = PyUnicode_FromFormat("<%S %U>", type_name, rinfo_s); Py_DECREF(type_name); } Py_DECREF(rinfo_s); @@ -789,22 +796,21 @@ FutureObj_finalize(FutureObj *fut) _Py_IDENTIFIER(future); _Py_IDENTIFIER(source_traceback); + PyObject *error_type, *error_value, *error_traceback; + PyObject *context; + PyObject *type_name; + PyObject *message = NULL; + PyObject *func; + if (!fut->fut_log_tb) { return; } assert(fut->fut_exception != NULL); - fut->fut_log_tb = 0;; + fut->fut_log_tb = 0; - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ PyErr_Fetch(&error_type, &error_value, &error_traceback); - PyObject *context = NULL; - PyObject *type_name = NULL; - PyObject *message = NULL; - PyObject *func = NULL; - PyObject *res = NULL; - context = PyDict_New(); if (context == NULL) { goto finally; @@ -817,6 +823,7 @@ FutureObj_finalize(FutureObj *fut) message = PyUnicode_FromFormat( "%S exception was never retrieved", type_name); + Py_DECREF(type_name); if (message == NULL) { goto finally; } @@ -835,18 +842,19 @@ FutureObj_finalize(FutureObj *fut) func = _PyObject_GetAttrId(fut->fut_loop, &PyId_call_exception_handler); if (func != NULL) { - res = _PyObject_CallArg1(func, context); + PyObject *res = _PyObject_CallArg1(func, context); if (res == NULL) { PyErr_WriteUnraisable(func); } + else { + Py_DECREF(res); + } + Py_DECREF(func); } finally: - Py_CLEAR(context); - Py_CLEAR(type_name); - Py_CLEAR(message); - Py_CLEAR(func); - Py_CLEAR(res); + Py_XDECREF(context); + Py_XDECREF(message); /* Restore the saved exception. */ PyErr_Restore(error_type, error_value, error_traceback); @@ -882,7 +890,8 @@ static PyMethodDef FutureType_methods[] = { {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, NULL, NULL}, \ + {"_log_traceback", (getter)FutureObj_get_log_traceback, \ + (setter)FutureObj_set_log_traceback, NULL}, \ {"_source_traceback", (getter)FutureObj_get_source_traceback, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { @@ -950,6 +959,8 @@ FutureObj_dealloc(PyObject *self) } } + PyObject_GC_UnTrack(self); + if (fut->fut_weakreflist != NULL) { PyObject_ClearWeakRefs(self); } @@ -990,22 +1001,19 @@ FutureIter_iternext(futureiterobject *it) Py_INCREF(fut); return (PyObject *)fut; } - PyErr_Format(PyExc_AssertionError, - "yield from wasn't used with future"); + PyErr_SetString(PyExc_AssertionError, + "yield from wasn't used with future"); return NULL; } + it->future = NULL; res = _asyncio_Future_result_impl(fut); if (res != NULL) { /* The result of the Future is not an exception. */ - if (_PyGen_SetStopIterationValue(res) < 0) { - Py_DECREF(res); - return NULL; - } + (void)_PyGen_SetStopIterationValue(res); Py_DECREF(res); } - it->future = NULL; Py_DECREF(fut); return NULL; } @@ -1022,7 +1030,7 @@ FutureIter_send(futureiterobject *self, PyObject *unused) static PyObject * FutureIter_throw(futureiterobject *self, PyObject *args) { - PyObject *type=NULL, *val=NULL, *tb=NULL; + PyObject *type, *val = NULL, *tb = NULL; if (!PyArg_ParseTuple(args, "O|OO", &type, &val, &tb)) return NULL; @@ -1066,7 +1074,7 @@ FutureIter_throw(futureiterobject *self, PyObject *args) PyErr_Restore(type, val, tb); - return FutureIter_iternext(self); + return NULL; fail: Py_DECREF(type); @@ -1147,7 +1155,7 @@ static PyObject * task_step(TaskObj *, PyObject *); /* ----- Task._step wrapper */ static int -TaskSendMethWrapper_clear(TaskSendMethWrapper *o) +TaskStepMethWrapper_clear(TaskStepMethWrapper *o) { Py_CLEAR(o->sw_task); Py_CLEAR(o->sw_arg); @@ -1155,22 +1163,30 @@ TaskSendMethWrapper_clear(TaskSendMethWrapper *o) } static void -TaskSendMethWrapper_dealloc(TaskSendMethWrapper *o) +TaskStepMethWrapper_dealloc(TaskStepMethWrapper *o) { PyObject_GC_UnTrack(o); - (void)TaskSendMethWrapper_clear(o); + (void)TaskStepMethWrapper_clear(o); Py_TYPE(o)->tp_free(o); } static PyObject * -TaskSendMethWrapper_call(TaskSendMethWrapper *o, +TaskStepMethWrapper_call(TaskStepMethWrapper *o, PyObject *args, PyObject *kwds) { + if (kwds != NULL && PyDict_Size(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, "function takes no keyword arguments"); + return NULL; + } + if (args != NULL && PyTuple_GET_SIZE(args) != 0) { + PyErr_SetString(PyExc_TypeError, "function takes no positional arguments"); + return NULL; + } return task_call_step(o->sw_task, o->sw_arg); } static int -TaskSendMethWrapper_traverse(TaskSendMethWrapper *o, +TaskStepMethWrapper_traverse(TaskStepMethWrapper *o, visitproc visit, void *arg) { Py_VISIT(o->sw_task); @@ -1179,7 +1195,7 @@ TaskSendMethWrapper_traverse(TaskSendMethWrapper *o, } static PyObject * -TaskSendMethWrapper_get___self__(TaskSendMethWrapper *o) +TaskStepMethWrapper_get___self__(TaskStepMethWrapper *o) { if (o->sw_task) { Py_INCREF(o->sw_task); @@ -1188,30 +1204,30 @@ TaskSendMethWrapper_get___self__(TaskSendMethWrapper *o) Py_RETURN_NONE; } -static PyGetSetDef TaskSendMethWrapper_getsetlist[] = { - {"__self__", (getter)TaskSendMethWrapper_get___self__, NULL, NULL}, +static PyGetSetDef TaskStepMethWrapper_getsetlist[] = { + {"__self__", (getter)TaskStepMethWrapper_get___self__, NULL, NULL}, {NULL} /* Sentinel */ }; -PyTypeObject TaskSendMethWrapper_Type = { +PyTypeObject TaskStepMethWrapper_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "TaskSendMethWrapper", - .tp_basicsize = sizeof(TaskSendMethWrapper), + "TaskStepMethWrapper", + .tp_basicsize = sizeof(TaskStepMethWrapper), .tp_itemsize = 0, - .tp_getset = TaskSendMethWrapper_getsetlist, - .tp_dealloc = (destructor)TaskSendMethWrapper_dealloc, - .tp_call = (ternaryfunc)TaskSendMethWrapper_call, + .tp_getset = TaskStepMethWrapper_getsetlist, + .tp_dealloc = (destructor)TaskStepMethWrapper_dealloc, + .tp_call = (ternaryfunc)TaskStepMethWrapper_call, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)TaskSendMethWrapper_traverse, - .tp_clear = (inquiry)TaskSendMethWrapper_clear, + .tp_traverse = (traverseproc)TaskStepMethWrapper_traverse, + .tp_clear = (inquiry)TaskStepMethWrapper_clear, }; static PyObject * -TaskSendMethWrapper_new(TaskObj *task, PyObject *arg) +TaskStepMethWrapper_new(TaskObj *task, PyObject *arg) { - TaskSendMethWrapper *o; - o = PyObject_GC_New(TaskSendMethWrapper, &TaskSendMethWrapper_Type); + TaskStepMethWrapper *o; + o = PyObject_GC_New(TaskStepMethWrapper, &TaskStepMethWrapper_Type); if (o == NULL) { return NULL; } @@ -1234,7 +1250,11 @@ TaskWakeupMethWrapper_call(TaskWakeupMethWrapper *o, { PyObject *fut; - if (!PyArg_ParseTuple(args, "O|", &fut)) { + if (kwds != NULL && PyDict_Size(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, "function takes no keyword arguments"); + return NULL; + } + if (!PyArg_ParseTuple(args, "O", &fut)) { return NULL; } @@ -1298,16 +1318,16 @@ TaskWakeupMethWrapper_new(TaskObj *task) /*[clinic input] _asyncio.Task.__init__ - coro: 'O' + coro: object * - loop: 'O' = NULL + loop: object = None A coroutine wrapped in a Future. [clinic start generated code]*/ static int _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) -/*[clinic end generated code: output=9f24774c2287fc2f input=71d8d28c201a18cd]*/ +/*[clinic end generated code: output=9f24774c2287fc2f input=8d132974b049593e]*/ { PyObject *res; _Py_IDENTIFIER(add); @@ -1413,7 +1433,7 @@ TaskObj_get_fut_waiter(TaskObj *task) @classmethod _asyncio.Task.current_task - loop: 'O' = NULL + loop: object = None Return the currently running task in an event loop or None. @@ -1424,12 +1444,12 @@ None is returned when called not in the context of a Task. static PyObject * _asyncio_Task_current_task_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=99fbe7332c516e03 input=cd784537f02cf833]*/ +/*[clinic end generated code: output=99fbe7332c516e03 input=cd14770c5b79c7eb]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1486,12 +1506,14 @@ task_all_tasks(PyObject *loop) Py_DECREF(task_loop); Py_DECREF(task); } - + if (PyErr_Occurred()) { + goto fail; + } Py_DECREF(iter); return set; fail: - Py_XDECREF(set); + Py_DECREF(set); Py_XDECREF(iter); return NULL; } @@ -1500,7 +1522,7 @@ task_all_tasks(PyObject *loop) @classmethod _asyncio.Task.all_tasks - loop: 'O' = NULL + loop: object = None Return a set of all tasks for an event loop. @@ -1509,12 +1531,12 @@ By default all tasks for the current event loop are returned. static PyObject * _asyncio_Task_all_tasks_impl(PyTypeObject *type, PyObject *loop) -/*[clinic end generated code: output=11f9b20749ccca5d input=cd64aa5f88bd5c49]*/ +/*[clinic end generated code: output=11f9b20749ccca5d input=497f80bc9ce726b5]*/ { PyObject *res; - if (loop == NULL) { - loop = PyObject_CallObject(asyncio_get_event_loop, NULL); + if (loop == Py_None) { + loop = _PyObject_CallNoArg(asyncio_get_event_loop); if (loop == NULL) { return NULL; } @@ -1568,6 +1590,8 @@ static PyObject * _asyncio_Task_cancel_impl(TaskObj *self) /*[clinic end generated code: output=6bfc0479da9d5757 input=13f9bf496695cb52]*/ { + self->task_log_tb = 0; + if (self->task_state != STATE_PENDING) { Py_RETURN_FALSE; } @@ -1601,7 +1625,7 @@ _asyncio_Task_cancel_impl(TaskObj *self) _asyncio.Task.get_stack * - limit: 'O' = None + limit: object = None Return the list of stack frames for this task's coroutine. @@ -1626,7 +1650,7 @@ returned for a suspended coroutine. static PyObject * _asyncio_Task_get_stack_impl(TaskObj *self, PyObject *limit) -/*[clinic end generated code: output=c9aeeeebd1e18118 input=b1920230a766d17a]*/ +/*[clinic end generated code: output=c9aeeeebd1e18118 input=05b323d42b809b90]*/ { return PyObject_CallFunctionObjArgs( asyncio_task_get_stack_func, self, limit, NULL); @@ -1636,8 +1660,8 @@ _asyncio_Task_get_stack_impl(TaskObj *self, PyObject *limit) _asyncio.Task.print_stack * - limit: 'O' = None - file: 'O' = None + limit: object = None + file: object = None Print the stack or traceback for this task's coroutine. @@ -1651,7 +1675,7 @@ to sys.stderr. static PyObject * _asyncio_Task_print_stack_impl(TaskObj *self, PyObject *limit, PyObject *file) -/*[clinic end generated code: output=7339e10314cd3f4d input=19f1e99ab5400bc3]*/ +/*[clinic end generated code: output=7339e10314cd3f4d input=1a0352913b7fcd92]*/ { return PyObject_CallFunctionObjArgs( asyncio_task_print_stack_func, self, limit, file, NULL); @@ -1660,12 +1684,12 @@ _asyncio_Task_print_stack_impl(TaskObj *self, PyObject *limit, /*[clinic input] _asyncio.Task._step - exc: 'O' = NULL + exc: object = None [clinic start generated code]*/ static PyObject * _asyncio_Task__step_impl(TaskObj *self, PyObject *exc) -/*[clinic end generated code: output=7ed23f0cefd5ae42 input=ada4b2324e5370af]*/ +/*[clinic end generated code: output=7ed23f0cefd5ae42 input=1e19a985ace87ca4]*/ { return task_step(self, exc == Py_None ? NULL : exc); } @@ -1673,12 +1697,12 @@ _asyncio_Task__step_impl(TaskObj *self, PyObject *exc) /*[clinic input] _asyncio.Task._wakeup - fut: 'O' + fut: object [clinic start generated code]*/ static PyObject * _asyncio_Task__wakeup_impl(TaskObj *self, PyObject *fut) -/*[clinic end generated code: output=75cb341c760fd071 input=11ee4918a5bdbf21]*/ +/*[clinic end generated code: output=75cb341c760fd071 input=6a0616406f829a7b]*/ { return task_wakeup(self, fut); } @@ -1691,11 +1715,9 @@ TaskObj_finalize(TaskObj *task) _Py_IDENTIFIER(message); _Py_IDENTIFIER(source_traceback); + PyObject *context; PyObject *message = NULL; - PyObject *context = NULL; - PyObject *func = NULL; - PyObject *res = NULL; - + PyObject *func; PyObject *error_type, *error_value, *error_traceback; if (task->task_state != STATE_PENDING || !task->task_log_destroy_pending) { @@ -1731,17 +1753,19 @@ TaskObj_finalize(TaskObj *task) func = _PyObject_GetAttrId(task->task_loop, &PyId_call_exception_handler); if (func != NULL) { - res = _PyObject_CallArg1(func, context); + PyObject *res = _PyObject_CallArg1(func, context); if (res == NULL) { PyErr_WriteUnraisable(func); } + else { + Py_DECREF(res); + } + Py_DECREF(func); } finally: - Py_CLEAR(context); - Py_CLEAR(message); - Py_CLEAR(func); - Py_CLEAR(res); + Py_XDECREF(context); + Py_XDECREF(message); /* Restore the saved exception. */ PyErr_Restore(error_type, error_value, error_traceback); @@ -1822,6 +1846,8 @@ TaskObj_dealloc(PyObject *self) } } + PyObject_GC_UnTrack(self); + if (task->task_weakreflist != NULL) { PyObject_ClearWeakRefs(self); } @@ -1851,9 +1877,6 @@ task_call_step(TaskObj *task, PyObject *arg) } else { /* `task` is a subclass of Task */ - if (arg == NULL) { - arg = Py_None; - } return _PyObject_CallMethodIdObjArgs((PyObject*)task, &PyId__step, arg, NULL); } @@ -1864,7 +1887,7 @@ task_call_step_soon(TaskObj *task, PyObject *arg) { PyObject *handle; - PyObject *cb = TaskSendMethWrapper_new(task, arg); + PyObject *cb = TaskStepMethWrapper_new(task, arg); if (cb == NULL) { return -1; } @@ -1919,7 +1942,7 @@ task_step_impl(TaskObj *task, PyObject *exc) int res; int clear_exc = 0; PyObject *result = NULL; - PyObject *coro = task->task_coro; + PyObject *coro; PyObject *o; if (task->task_state != STATE_PENDING) { @@ -1960,6 +1983,12 @@ task_step_impl(TaskObj *task, PyObject *exc) Py_CLEAR(task->task_fut_waiter); + coro = task->task_coro; + if (coro == NULL) { + PyErr_SetString(PyExc_RuntimeError, "uninitialized Task object"); + return NULL; + } + if (exc == NULL) { if (PyGen_CheckExact(coro) || PyCoro_CheckExact(coro)) { result = _PyGen_Send((PyGenObject*)coro, Py_None); @@ -1974,7 +2003,7 @@ task_step_impl(TaskObj *task, PyObject *exc) coro, &PyId_throw, exc, NULL); if (clear_exc) { /* We created 'exc' during this call */ - Py_CLEAR(exc); + Py_DECREF(exc); } } @@ -1984,6 +2013,16 @@ task_step_impl(TaskObj *task, PyObject *exc) if (_PyGen_FetchStopIterationValue(&o) == 0) { /* The error is StopIteration and that means that the underlying coroutine has resolved */ + if (task->task_must_cancel) { + // Task is cancelled right before coro stops. + Py_DECREF(o); + task->task_must_cancel = 0; + et = asyncio_CancelledError; + Py_INCREF(et); + ev = NULL; + tb = NULL; + goto set_exception; + } PyObject *res = future_set_result((FutureObj*)task, o); Py_DECREF(o); if (res == NULL) { @@ -2001,6 +2040,8 @@ task_step_impl(TaskObj *task, PyObject *exc) /* Some other exception; pop it and call Task.set_exception() */ PyErr_Fetch(&et, &ev, &tb); + +set_exception: assert(et); if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { PyErr_NormalizeException(&et, &ev, &tb); @@ -2011,13 +2052,13 @@ task_step_impl(TaskObj *task, PyObject *exc) o = future_set_exception((FutureObj*)task, ev); if (!o) { /* An exception in Task.set_exception() */ - Py_XDECREF(et); + Py_DECREF(et); Py_XDECREF(tb); Py_XDECREF(ev); goto fail; } assert(o == Py_None); - Py_CLEAR(o); + Py_DECREF(o); if (!PyErr_GivenExceptionMatches(et, PyExc_Exception)) { /* We've got a BaseException; re-raise it */ @@ -2025,7 +2066,7 @@ task_step_impl(TaskObj *task, PyObject *exc) goto fail; } - Py_XDECREF(et); + Py_DECREF(et); Py_XDECREF(tb); Py_XDECREF(ev); @@ -2097,7 +2138,7 @@ task_step_impl(TaskObj *task, PyObject *exc) } else { if (o == Py_None) { - Py_CLEAR(o); + Py_DECREF(o); } else { /* `result` is a Future-compatible object */ @@ -2105,7 +2146,7 @@ task_step_impl(TaskObj *task, PyObject *exc) PyObject *res; int blocking = PyObject_IsTrue(o); - Py_CLEAR(o); + Py_DECREF(o); if (blocking < 0) { goto fail; } @@ -2188,7 +2229,7 @@ task_step_impl(TaskObj *task, PyObject *exc) goto fail; } res = PyObject_IsTrue(o); - Py_CLEAR(o); + Py_DECREF(o); if (res == -1) { /* An exception while checking if 'val' is True */ goto fail; @@ -2256,14 +2297,8 @@ task_step(TaskObj *task, PyObject *exc) PyObject *et, *ev, *tb; PyErr_Fetch(&et, &ev, &tb); ot = _PyDict_Pop(current_tasks, task->task_loop, NULL); - if (ot == NULL) { - Py_XDECREF(et); - Py_XDECREF(tb); - Py_XDECREF(ev); - return NULL; - } - Py_DECREF(ot); - PyErr_Restore(et, ev, tb); + Py_XDECREF(ot); + _PyErr_ChainExceptions(et, ev, tb); return NULL; } else { @@ -2282,17 +2317,18 @@ task_step(TaskObj *task, PyObject *exc) static PyObject * task_wakeup(TaskObj *task, PyObject *o) { + PyObject *et, *ev, *tb; + PyObject *result; assert(o); if (Future_CheckExact(o) || Task_CheckExact(o)) { PyObject *fut_result = NULL; int res = future_get_result((FutureObj*)o, &fut_result); - PyObject *result; switch(res) { case -1: assert(fut_result == NULL); - return NULL; + break; /* exception raised */ case 0: Py_DECREF(fut_result); return task_call_step(task, NULL); @@ -2303,29 +2339,32 @@ task_wakeup(TaskObj *task, PyObject *o) return result; } } - - PyObject *fut_result = PyObject_CallMethod(o, "result", NULL); - if (fut_result == NULL) { - PyObject *et, *ev, *tb; - PyObject *res; - - PyErr_Fetch(&et, &ev, &tb); - if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { - PyErr_NormalizeException(&et, &ev, &tb); + else { + PyObject *fut_result = PyObject_CallMethod(o, "result", NULL); + if (fut_result != NULL) { + Py_DECREF(fut_result); + return task_call_step(task, NULL); } + /* exception raised */ + } - res = task_call_step(task, ev); - - Py_XDECREF(et); - Py_XDECREF(tb); - Py_XDECREF(ev); - - return res; + PyErr_Fetch(&et, &ev, &tb); + if (!PyErr_GivenExceptionMatches(et, PyExc_Exception)) { + /* We've got a BaseException; re-raise it */ + PyErr_Restore(et, ev, tb); + return NULL; } - else { - Py_DECREF(fut_result); - return task_call_step(task, NULL); + if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { + PyErr_NormalizeException(&et, &ev, &tb); } + + result = task_call_step(task, ev); + + Py_DECREF(et); + Py_XDECREF(tb); + Py_XDECREF(ev); + + return result; } @@ -2358,7 +2397,7 @@ module_init(void) Py_CLEAR(module); \ module = PyImport_ImportModule(NAME); \ if (module == NULL) { \ - return -1; \ + goto fail; \ } #define GET_MOD_ATTR(VAR, NAME) \ @@ -2388,8 +2427,8 @@ module_init(void) WITH_MOD("weakref") GET_MOD_ATTR(cls, "WeakSet") - all_tasks = PyObject_CallObject(cls, NULL); - Py_CLEAR(cls); + all_tasks = _PyObject_CallNoArg(cls); + Py_DECREF(cls); if (all_tasks == NULL) { goto fail; } @@ -2399,7 +2438,7 @@ module_init(void) goto fail; } - Py_CLEAR(module); + Py_DECREF(module); return 0; fail: @@ -2438,7 +2477,7 @@ PyInit__asyncio(void) if (PyType_Ready(&FutureIterType) < 0) { return NULL; } - if (PyType_Ready(&TaskSendMethWrapper_Type) < 0) { + if (PyType_Ready(&TaskStepMethWrapper_Type) < 0) { return NULL; } if(PyType_Ready(&TaskWakeupMethWrapper_Type) < 0) { diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c64e77de2..af20d6edd91859 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); @@ -1706,6 +1706,8 @@ dequeiter_traverse(dequeiterobject *dio, visitproc visit, void *arg) static void dequeiter_dealloc(dequeiterobject *dio) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(dio); Py_XDECREF(dio->deque); PyObject_GC_Del(dio); } @@ -2086,6 +2088,8 @@ static PyMemberDef defdict_members[] = { static void defdict_dealloc(defdictobject *dd) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(dd); Py_CLEAR(dd->default_factory); PyDict_Type.tp_dealloc((PyObject *)dd); } @@ -2272,7 +2276,9 @@ _count_elements(PyObject *self, PyObject *args) dict_setitem = _PyType_LookupId(&PyDict_Type, &PyId___setitem__); if (mapping_get != NULL && mapping_get == dict_get && - mapping_setitem != NULL && mapping_setitem == dict_setitem) { + mapping_setitem != NULL && mapping_setitem == dict_setitem && + PyDict_Check(mapping)) + { while (1) { /* Fast path advantages: 1. Eliminate double hashing diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aedec6eeab5..258e805870b11d 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -250,6 +250,71 @@ PyDict_GetItemProxy(PyObject *dict, PyObject *key) } /******************************************************************/ + +/* + Allocate a memory block for a pep3118 format string, filled with + a suitable PEP 3118 type code corresponding to the given ctypes + type. Returns NULL on failure, with the error indicator set. + + This produces type codes in the standard size mode (cf. struct module), + since the endianness may need to be swapped to a non-native one + later on. + */ +static char * +_ctypes_alloc_format_string_for_type(char code, int big_endian) +{ + char *result; + char pep_code = '\0'; + + switch (code) { +#if SIZEOF_INT == 2 + case 'i': pep_code = 'h'; break; + case 'I': pep_code = 'H'; break; +#elif SIZEOF_INT == 4 + case 'i': pep_code = 'i'; break; + case 'I': pep_code = 'I'; break; +#elif SIZEOF_INT == 8 + case 'i': pep_code = 'q'; break; + case 'I': pep_code = 'Q'; break; +#else +# error SIZEOF_INT has an unexpected value +#endif /* SIZEOF_INT */ +#if SIZEOF_LONG == 4 + case 'l': pep_code = 'l'; break; + case 'L': pep_code = 'L'; break; +#elif SIZEOF_LONG == 8 + case 'l': pep_code = 'q'; break; + case 'L': pep_code = 'Q'; break; +#else +# error SIZEOF_LONG has an unexpected value +#endif /* SIZEOF_LONG */ +#if SIZEOF__BOOL == 1 + case '?': pep_code = '?'; break; +#elif SIZEOF__BOOL == 2 + case '?': pep_code = 'H'; break; +#elif SIZEOF__BOOL == 4 + case '?': pep_code = 'L'; break; +#elif SIZEOF__BOOL == 8 + case '?': pep_code = 'Q'; break; +#else +# error SIZEOF__BOOL has an unexpected value +#endif /* SIZEOF__BOOL */ + default: + /* The standard-size code is the same as the ctypes one */ + pep_code = code; + break; + } + + result = PyMem_Malloc(3); + if (result == NULL) + return NULL; + + result[0] = big_endian ? '>' : '<'; + result[1] = pep_code; + result[2] = '\0'; + return result; +} + /* Allocate a memory block for a pep3118 format string, copy prefix (if non-null) and suffix into it. Returns NULL on failure, with the error @@ -925,6 +990,7 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (proto) { StgDictObject *itemdict = PyType_stgdict(proto); const char *current_format; + /* PyCPointerType_SetProto has verified proto has a stgdict. */ assert(itemdict); /* If itemdict->format is NULL, then this is a pointer to an incomplete type. We create a generic format string @@ -971,7 +1037,11 @@ PyCPointerType_set_type(PyTypeObject *self, PyObject *type) StgDictObject *dict; dict = PyType_stgdict((PyObject *)self); - assert(dict); + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "abstract class"); + return NULL; + } if (-1 == PyCPointerType_SetProto(dict, type)) return NULL; @@ -997,7 +1067,11 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) } typedict = PyType_stgdict(type); - assert(typedict); /* Cannot be NULL for pointer types */ + if (!typedict) { + PyErr_SetString(PyExc_TypeError, + "abstract class"); + return NULL; + } /* If we expect POINTER(), but receive a instance, accept it by calling byref(). @@ -1926,9 +2000,9 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) stgdict->setfunc = fmt->setfunc; stgdict->getfunc = fmt->getfunc; #ifdef WORDS_BIGENDIAN - stgdict->format = _ctypes_alloc_format_string(">", proto_str); + stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); #else - stgdict->format = _ctypes_alloc_format_string("<", proto_str); + stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); #endif if (stgdict->format == NULL) { Py_DECREF(result); @@ -2062,7 +2136,11 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) } dict = PyType_stgdict(type); - assert(dict); + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "abstract class"); + return NULL; + } /* I think we can rely on this being a one-character string */ fmt = PyUnicode_AsUTF8(dict->proto); @@ -2598,6 +2676,16 @@ PyCData_setstate(PyObject *myself, PyObject *args) len = self->b_size; memmove(self->b_ptr, data, len); mydict = PyObject_GetAttrString(myself, "__dict__"); + if (mydict == NULL) { + return NULL; + } + if (!PyDict_Check(mydict)) { + PyErr_Format(PyExc_TypeError, + "%.200s.__dict__ must be a dictionary, not %.200s", + Py_TYPE(myself)->tp_name, Py_TYPE(mydict)->tp_name); + Py_DECREF(mydict); + return NULL; + } res = PyDict_Update(mydict, dict); Py_DECREF(mydict); if (res == -1) @@ -3156,7 +3244,11 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) PyObject *argtypes; dict = PyType_stgdict((PyObject *)type); - assert(dict); /* Cannot be NULL. 'type' is a PyCFuncPtr type. */ + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "abstract class"); + return 0; + } argtypes = dict->argtypes; if (paramflags == NULL || dict->argtypes == NULL) @@ -3610,7 +3702,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, case (PARAMFLAG_FIN | PARAMFLAG_FOUT): *pinoutmask |= (1 << i); /* mark as inout arg */ (*pnumretvals)++; - /* fall through to PARAMFLAG_FIN... */ + /* fall through */ case 0: case PARAMFLAG_FIN: /* 'in' parameter. Copy it from inargs. */ @@ -4273,11 +4365,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4505,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; @@ -4789,7 +4879,7 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) } stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL fr pointer instances */ + assert(stgdict); /* Cannot be NULL for pointer instances */ proto = stgdict->proto; assert(proto); @@ -4817,7 +4907,7 @@ Pointer_get_contents(CDataObject *self, void *closure) } stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL fr pointer instances */ + assert(stgdict); /* Cannot be NULL for pointer instances */ return PyCData_FromBaseObj(stgdict->proto, (PyObject *)self, 0, *(void **)self->b_ptr); @@ -4836,7 +4926,7 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) return -1; } stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL fr pointer instances */ + assert(stgdict); /* Cannot be NULL for pointer instances */ assert(stgdict->proto); if (!CDataObject_Check(value)) { int res = PyObject_IsInstance(value, stgdict->proto); diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 629ddf66bc4212..6119ecdaf90dc6 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -44,6 +44,19 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) func(in); } +/* + * See issue 29565. Update a structure passed by value; + * the caller should not see any change. + */ + +EXPORT(void) +_testfunc_large_struct_update_value(Test in) +{ + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 7d542fb50d8767..ff95dffb82c244 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -668,7 +668,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) #ifdef CTYPES_UNICODE if (PyUnicode_Check(obj)) { pa->ffi_type = &ffi_type_pointer; - pa->value.p = PyUnicode_AsWideCharString(obj, NULL); + pa->value.p = _PyUnicode_AsWideCharString(obj); if (pa->value.p == NULL) return -1; pa->keep = PyCapsule_New(pa->value.p, CTYPES_CAPSULE_NAME_PYMEM, pymem_destructor); @@ -1231,14 +1231,15 @@ The handle may be used to locate exported functions in this\n\ module.\n"; static PyObject *load_library(PyObject *self, PyObject *args) { - WCHAR *name; + const WCHAR *name; PyObject *nameobj; PyObject *ignored; HMODULE hMod; - if (!PyArg_ParseTuple(args, "O|O:LoadLibrary", &nameobj, &ignored)) + + if (!PyArg_ParseTuple(args, "U|O:LoadLibrary", &nameobj, &ignored)) return NULL; - name = PyUnicode_AsUnicode(nameobj); + name = _PyUnicode_AsUnicode(nameobj); if (!name) return NULL; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a43585f1b83f13..e39065fa9393b0 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -205,7 +205,11 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - assert(CDataObject_Check(inst)); + if (!CDataObject_Check(inst)) { + PyErr_SetString(PyExc_TypeError, + "not a ctype instance"); + return -1; + } dst = (CDataObject *)inst; ptr = dst->b_ptr + self->offset; if (value == NULL) { @@ -225,7 +229,11 @@ PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) Py_INCREF(self); return (PyObject *)self; } - assert(CDataObject_Check(inst)); + if (!CDataObject_Check(inst)) { + PyErr_SetString(PyExc_TypeError, + "not a ctype instance"); + return NULL; + } src = (CDataObject *)inst; return PyCData_get(self->proto, self->getfunc, inst, self->index, self->size, src->b_ptr + self->offset); @@ -1372,7 +1380,7 @@ Z_set(void *ptr, PyObject *value, Py_ssize_t size) /* We must create a wchar_t* buffer from the unicode object, and keep it alive */ - buffer = PyUnicode_AsWideCharString(value, NULL); + buffer = _PyUnicode_AsWideCharString(value); if (!buffer) return NULL; keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); diff --git a/Modules/_ctypes/libffi_msvc/ffi.c b/Modules/_ctypes/libffi_msvc/ffi.c index 1d82929f530220..91a27dce3f2575 100644 --- a/Modules/_ctypes/libffi_msvc/ffi.c +++ b/Modules/_ctypes/libffi_msvc/ffi.c @@ -239,6 +239,16 @@ ffi_call(/*@dependent@*/ ffi_cif *cif, break; #else case FFI_SYSV: + /* If a single argument takes more than 8 bytes, + then a copy is passed by reference. */ + for (unsigned i = 0; i < cif->nargs; i++) { + size_t z = cif->arg_types[i]->size; + if (z > 8) { + void *temp = alloca(z); + memcpy(temp, avalue[i], z); + avalue[i] = temp; + } + } /*@-usedef@*/ return ffi_call_AMD64(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, fn); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 716d1e9e8eb5df..8278b2fbefbfc0 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -282,7 +282,15 @@ MakeAnonFields(PyObject *type) Py_DECREF(anon_names); return -1; } - assert(Py_TYPE(descr) == &PyCField_Type); + if (Py_TYPE(descr) != &PyCField_Type) { + PyErr_Format(PyExc_AttributeError, + "'%U' is specified in _anonymous_ but not in " + "_fields_", + fname); + Py_DECREF(anon_names); + Py_DECREF(descr); + return -1; + } descr->anonymous = 1; /* descr is in the field descriptor. */ diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7ce15392..7a7095118a0baf 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -342,10 +342,11 @@ static int PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, PyObject **bytes, wchar_t **wstr) { + char *str; if (PyUnicode_Check(obj)) { #ifdef HAVE_NCURSESW assert (wstr != NULL); - *wstr = PyUnicode_AsWideCharString(obj, NULL); + *wstr = _PyUnicode_AsWideCharString(obj); if (*wstr == NULL) return 0; return 2; @@ -354,12 +355,20 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, *bytes = PyUnicode_AsEncodedString(obj, win->encoding, NULL); if (*bytes == NULL) return 0; + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(*bytes, &str, NULL) < 0) { + return 0; + } return 1; #endif } else if (PyBytes_Check(obj)) { Py_INCREF(obj); *bytes = obj; + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(*bytes, &str, NULL) < 0) { + return 0; + } return 1; } @@ -2082,7 +2091,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f4a90999..d44bb3e047f5f7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1646,6 +1646,33 @@ multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) return result; } +static PyObject * +get_float_as_integer_ratio(PyObject *floatobj) +{ + PyObject *ratio; + + assert(floatobj && PyFloat_Check(floatobj)); + ratio = _PyObject_CallMethodId(floatobj, &PyId_as_integer_ratio, NULL); + if (ratio == NULL) { + return NULL; + } + if (!PyTuple_Check(ratio)) { + PyErr_Format(PyExc_TypeError, + "unexpected return type from as_integer_ratio(): " + "expected tuple, got '%.200s'", + Py_TYPE(ratio)->tp_name); + Py_DECREF(ratio); + return NULL; + } + if (PyTuple_Size(ratio) != 2) { + PyErr_SetString(PyExc_ValueError, + "as_integer_ratio() must return a 2-tuple"); + Py_DECREF(ratio); + return NULL; + } + return ratio; +} + static PyObject * multiply_float_timedelta(PyObject *floatobj, PyDateTime_Delta *delta) { @@ -1656,9 +1683,10 @@ multiply_float_timedelta(PyObject *floatobj, PyDateTime_Delta *delta) pyus_in = delta_to_microseconds(delta); if (pyus_in == NULL) return NULL; - ratio = _PyObject_CallMethodId(floatobj, &PyId_as_integer_ratio, NULL); - if (ratio == NULL) + ratio = get_float_as_integer_ratio(floatobj); + if (ratio == NULL) { goto error; + } temp = PyNumber_Multiply(pyus_in, PyTuple_GET_ITEM(ratio, 0)); Py_DECREF(pyus_in); pyus_in = NULL; @@ -1754,9 +1782,10 @@ truedivide_timedelta_float(PyDateTime_Delta *delta, PyObject *f) pyus_in = delta_to_microseconds(delta); if (pyus_in == NULL) return NULL; - ratio = _PyObject_CallMethodId(f, &PyId_as_integer_ratio, NULL); - if (ratio == NULL) + ratio = get_float_as_integer_ratio(f); + if (ratio == NULL) { goto error; + } temp = PyNumber_Multiply(pyus_in, PyTuple_GET_ITEM(ratio, 1)); Py_DECREF(pyus_in); pyus_in = NULL; @@ -3926,16 +3955,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5048,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index fcc1f151cf51e0..13418c905b2eba 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -2163,13 +2163,17 @@ dec_from_long(PyTypeObject *type, const PyObject *v, /* Return a new PyDecObject from a PyLongObject. Use the context for conversion. */ static PyObject * -PyDecType_FromLong(PyTypeObject *type, const PyObject *pylong, - PyObject *context) +PyDecType_FromLong(PyTypeObject *type, const PyObject *v, PyObject *context) { PyObject *dec; uint32_t status = 0; - dec = dec_from_long(type, pylong, CTX(context), &status); + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, "argument must be an integer"); + return NULL; + } + + dec = dec_from_long(type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2185,15 +2189,20 @@ PyDecType_FromLong(PyTypeObject *type, const PyObject *pylong, /* Return a new PyDecObject from a PyLongObject. Use a maximum context for conversion. If the conversion is not exact, set InvalidOperation. */ static PyObject * -PyDecType_FromLongExact(PyTypeObject *type, const PyObject *pylong, +PyDecType_FromLongExact(PyTypeObject *type, const PyObject *v, PyObject *context) { PyObject *dec; uint32_t status = 0; mpd_context_t maxctx; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, "argument must be an integer"); + return NULL; + } + mpd_maxcontext(&maxctx); - dec = dec_from_long(type, pylong, &maxctx, &status); + dec = dec_from_long(type, v, &maxctx, &status); if (dec == NULL) { return NULL; } diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index 3aadfb0437979c..f45e558f1a9573 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -45,6 +45,12 @@ PEP-3101 formatting for numeric types. */ +/* Disable warning that is part of -Wextra since gcc 7.0. */ +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 + #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + + /* * Work around the behavior of tolower() and strcasecmp() in certain * locales. For example, in tr_TR.utf8: diff --git a/Modules/_decimal/tests/runall.bat b/Modules/_decimal/tests/runall.bat index 568f92f6ddf392..5bc872a63f83aa 100755 --- a/Modules/_decimal/tests/runall.bat +++ b/Modules/_decimal/tests/runall.bat @@ -1,111 +1,111 @@ -@ECHO OFF - -rem Test all machine configurations, pydebug, refleaks, release build. - -cd ..\..\..\ - - -echo. -echo # ====================================================================== -echo # Building Python -echo # ====================================================================== -echo. - -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x64 -msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 -msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=x64 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 - -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 -msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 -echo. -echo. - -echo. -echo # ====================================================================== -echo # test_decimal: platform=x64 -echo # ====================================================================== -echo. - -cd PCbuild\amd64 - -echo # ==================== refleak tests ======================= -echo. -python_d.exe -m test -uall -R 2:2 test_decimal -echo. -echo. - -echo # ==================== regular tests ======================= -echo. -python.exe -m test -uall test_decimal -echo. -echo. - -cd .. - -echo. -echo # ====================================================================== -echo # test_decimal: platform=x86 -echo # ====================================================================== -echo. - -echo # ==================== refleak tests ======================= -echo. -python_d.exe -m test -uall -R 2:2 test_decimal -echo. -echo. - -echo # ==================== regular tests ======================= -echo. -python.exe -m test -uall test_decimal -echo. -echo. - -cd amd64 - -echo. -echo # ====================================================================== -echo # deccheck: platform=x64 -echo # ====================================================================== -echo. - -echo # ==================== debug build ======================= -echo. -python_d.exe ..\..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -echo # =================== release build ====================== -echo. -python.exe ..\..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -cd .. - -echo. -echo # ====================================================================== -echo # deccheck: platform=x86 -echo # ====================================================================== -echo. -echo. - -echo # ==================== debug build ======================= -echo. -python_d.exe ..\Modules\_decimal\tests\deccheck.py -echo. -echo. - -echo # =================== release build ====================== -echo. -python.exe ..\Modules\_decimal\tests\deccheck.py -echo. -echo. - - -cd ..\Modules\_decimal\tests - - - +@ECHO OFF + +rem Test all machine configurations, pydebug, refleaks, release build. + +cd ..\..\..\ + + +echo. +echo # ====================================================================== +echo # Building Python +echo # ====================================================================== +echo. + +call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x64 +msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 +msbuild /noconsolelogger /target:clean PCbuild\pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=x64 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 + +call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 +msbuild /noconsolelogger PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 +echo. +echo. + +echo. +echo # ====================================================================== +echo # test_decimal: platform=x64 +echo # ====================================================================== +echo. + +cd PCbuild\amd64 + +echo # ==================== refleak tests ======================= +echo. +python_d.exe -m test -uall -R 2:2 test_decimal +echo. +echo. + +echo # ==================== regular tests ======================= +echo. +python.exe -m test -uall test_decimal +echo. +echo. + +cd .. + +echo. +echo # ====================================================================== +echo # test_decimal: platform=x86 +echo # ====================================================================== +echo. + +echo # ==================== refleak tests ======================= +echo. +python_d.exe -m test -uall -R 2:2 test_decimal +echo. +echo. + +echo # ==================== regular tests ======================= +echo. +python.exe -m test -uall test_decimal +echo. +echo. + +cd amd64 + +echo. +echo # ====================================================================== +echo # deccheck: platform=x64 +echo # ====================================================================== +echo. + +echo # ==================== debug build ======================= +echo. +python_d.exe ..\..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +echo # =================== release build ====================== +echo. +python.exe ..\..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +cd .. + +echo. +echo # ====================================================================== +echo # deccheck: platform=x86 +echo # ====================================================================== +echo. +echo. + +echo # ==================== debug build ======================= +echo. +python_d.exe ..\Modules\_decimal\tests\deccheck.py +echo. +echo. + +echo # =================== release build ====================== +echo. +python.exe ..\Modules\_decimal\tests\deccheck.py +echo. +echo. + + +cd ..\Modules\_decimal\tests + + + diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e61127d6..a6d6c61dd24e08 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -625,6 +627,7 @@ element_gc_clear(ElementObject *self) static void element_dealloc(ElementObject* self) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); Py_TRASHCAN_SAFE_BEGIN(self) @@ -1710,11 +1713,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1766,11 +1769,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ @@ -2046,6 +2049,8 @@ elementiter_dealloc(ElementIterObject *it) { Py_ssize_t i = it->parent_stack_used; it->parent_stack_used = 0; + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(it); while (i--) Py_XDECREF(it->parent_stack[i].parent); PyMem_Free(it->parent_stack); @@ -2053,7 +2058,6 @@ elementiter_dealloc(ElementIterObject *it) Py_XDECREF(it->sought_tag); Py_XDECREF(it->root_element); - PyObject_GC_UnTrack(it); PyObject_GC_Del(it); } @@ -2147,6 +2151,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2406,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2500,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2604,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { @@ -3218,6 +3224,18 @@ xmlparser_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)self; } +static int +ignore_attribute_error(PyObject *value) +{ + if (value == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return -1; + } + PyErr_Clear(); + } + return 0; +} + /*[clinic input] _elementtree.XMLParser.__init__ @@ -3264,14 +3282,33 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *html, self->target = target; self->handle_start = PyObject_GetAttrString(target, "start"); + if (ignore_attribute_error(self->handle_start)) { + return -1; + } self->handle_data = PyObject_GetAttrString(target, "data"); + if (ignore_attribute_error(self->handle_data)) { + return -1; + } self->handle_end = PyObject_GetAttrString(target, "end"); + if (ignore_attribute_error(self->handle_end)) { + return -1; + } self->handle_comment = PyObject_GetAttrString(target, "comment"); + if (ignore_attribute_error(self->handle_comment)) { + return -1; + } self->handle_pi = PyObject_GetAttrString(target, "pi"); + if (ignore_attribute_error(self->handle_pi)) { + return -1; + } self->handle_close = PyObject_GetAttrString(target, "close"); + if (ignore_attribute_error(self->handle_close)) { + return -1; + } self->handle_doctype = PyObject_GetAttrString(target, "doctype"); - - PyErr_Clear(); + if (ignore_attribute_error(self->handle_doctype)) { + return -1; + } /* configure parser */ EXPAT(SetUserData)(self->parser, self); @@ -3330,7 +3367,11 @@ xmlparser_gc_traverse(XMLParserObject *self, visitproc visit, void *arg) static int xmlparser_gc_clear(XMLParserObject *self) { - EXPAT(ParserFree)(self->parser); + if (self->parser != NULL) { + XML_Parser parser = self->parser; + self->parser = NULL; + EXPAT(ParserFree)(parser); + } Py_CLEAR(self->handle_close); Py_CLEAR(self->handle_pi); @@ -3943,6 +3984,11 @@ PyInit__elementtree(void) st->deepcopy_obj = PyObject_GetAttrString(temp, "deepcopy"); Py_XDECREF(temp); + if (st->deepcopy_obj == NULL) { + return NULL; + } + + assert(!PyErr_Occurred()); if (!(st->elementpath_obj = PyImport_ImportModule("xml.etree.ElementPath"))) return NULL; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f785a7260e6a2e..5622e2a6c3b8f9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -88,10 +88,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else { + else if (Py_REFCNT(kw) == 1) { Py_INCREF(kw); pto->kw = kw; } + else { + pto->kw = PyDict_Copy(kw); + } } else { pto->kw = PyDict_Copy(pkw); @@ -113,6 +116,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) static void partial_dealloc(partialobject *pto) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(pto); if (pto->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) pto); @@ -247,8 +251,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } @@ -1032,7 +1039,11 @@ lru_cache_clear_list(lru_list_elem *link) static void lru_cache_dealloc(lru_cache_object *obj) { - lru_list_elem *list = lru_cache_unlink_list(obj); + lru_list_elem *list; + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(obj); + + list = lru_cache_unlink_list(obj); Py_XDECREF(obj->maxsize_O); Py_XDECREF(obj->func); Py_XDECREF(obj->cache); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index daa4f3db2e8bfc..5a86376aa8bcff 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -139,7 +139,10 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) process = MUNCH_SIZE; else process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int); - EVP_DigestUpdate(self->ctx, (const void*)cp, process); + if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) { + _setException(PyExc_ValueError); + break; + } len -= process; cp += process; } @@ -209,7 +212,10 @@ EVP_digest(EVPobject *self, PyObject *unused) return _setException(PyExc_ValueError); } digest_size = EVP_MD_CTX_size(temp_ctx); - EVP_DigestFinal(temp_ctx, digest, NULL); + if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { + _setException(PyExc_ValueError); + return NULL; + } retval = PyBytes_FromStringAndSize((const char *)digest, digest_size); EVP_MD_CTX_free(temp_ctx); @@ -237,7 +243,10 @@ EVP_hexdigest(EVPobject *self, PyObject *unused) return _setException(PyExc_ValueError); } digest_size = EVP_MD_CTX_size(temp_ctx); - EVP_DigestFinal(temp_ctx, digest, NULL); + if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { + _setException(PyExc_ValueError); + return NULL; + } EVP_MD_CTX_free(temp_ctx); @@ -362,7 +371,12 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds) PyBuffer_Release(&view); return -1; } - EVP_DigestInit(self->ctx, digest); + if (!EVP_DigestInit(self->ctx, digest)) { + _setException(PyExc_ValueError); + if (data_obj) + PyBuffer_Release(&view); + return -1; + } self->name = name_obj; Py_INCREF(self->name); @@ -461,7 +475,11 @@ EVPnew(PyObject *name_obj, if (initial_ctx) { EVP_MD_CTX_copy(self->ctx, initial_ctx); } else { - EVP_DigestInit(self->ctx, digest); + if (!EVP_DigestInit(self->ctx, digest)) { + _setException(PyExc_ValueError); + Py_DECREF(self); + return NULL; + } } if (cp && len) { @@ -902,6 +920,8 @@ generate_hash_name_list(void) * the generic one passing it a python string and are noticeably * faster than calling a python new() wrapper. Thats important for * code that wants to make hashes of a bunch of small strings. + * The first call will lazy-initialize, which reports an exception + * if initialization fails. */ #define GEN_CONSTRUCTOR(NAME) \ static PyObject * \ @@ -914,6 +934,17 @@ generate_hash_name_list(void) if (!PyArg_ParseTuple(args, "|O:" #NAME , &data_obj)) { \ return NULL; \ } \ + \ + if (CONST_new_ ## NAME ## _ctx_p == NULL) { \ + EVP_MD_CTX *ctx_p = EVP_MD_CTX_new(); \ + if (!EVP_get_digestbyname(#NAME) || \ + !EVP_DigestInit(ctx_p, EVP_get_digestbyname(#NAME))) { \ + _setException(PyExc_ValueError); \ + EVP_MD_CTX_free(ctx_p); \ + return NULL; \ + } \ + CONST_new_ ## NAME ## _ctx_p = ctx_p; \ + } \ \ if (data_obj) \ GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); \ @@ -942,10 +973,6 @@ generate_hash_name_list(void) #define INIT_CONSTRUCTOR_CONSTANTS(NAME) do { \ if (CONST_ ## NAME ## _name_obj == NULL) { \ CONST_ ## NAME ## _name_obj = PyUnicode_FromString(#NAME); \ - if (EVP_get_digestbyname(#NAME)) { \ - CONST_new_ ## NAME ## _ctx_p = EVP_MD_CTX_new(); \ - EVP_DigestInit(CONST_new_ ## NAME ## _ctx_p, EVP_get_digestbyname(#NAME)); \ - } \ } \ } while (0); @@ -995,8 +1022,11 @@ PyInit__hashlib(void) { PyObject *m, *openssl_md_meth_names; - OpenSSL_add_all_digests(); +#ifndef OPENSSL_VERSION_1_1 + /* Load all digest algorithms and initialize cpuid */ + OPENSSL_add_all_algorithms_noconf(); ERR_load_crypto_strings(); +#endif /* TODO build EVP_functions openssl_* entries dynamically based * on what hashes are supported rather than listing many diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425eaefa65..efc7d05b7d01b9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index a1ba121e2622ff..6c54de733b95be 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1131,6 +1131,8 @@ bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) static void bytesiobuf_dealloc(bytesiobuf *self) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(self); Py_CLEAR(self->source); Py_TYPE(self)->tp_free(self); } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab9625ef5..918fa577758d28 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -280,11 +280,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (fd < 0) { #ifdef MS_WINDOWS - Py_ssize_t length; if (!PyUnicode_FSDecoder(nameobj, &stringobj)) { return -1; } - widename = PyUnicode_AsUnicodeAndSize(stringobj, &length); + widename = PyUnicode_AsUnicode(stringobj); if (widename == NULL) return -1; #else @@ -1082,9 +1081,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 472ef3b97cd38e..c8642040ae426a 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -625,7 +625,8 @@ iobase_iternext(PyObject *self) if (line == NULL) return NULL; - if (PyObject_Size(line) == 0) { + if (PyObject_Size(line) <= 0) { + /* Error or empty */ Py_DECREF(line); return NULL; } @@ -650,7 +651,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) /*[clinic end generated code: output=2f50421677fa3dea input=1961c4a95e96e661]*/ { Py_ssize_t length = 0; - PyObject *result; + PyObject *result, *it = NULL; result = PyList_New(0); if (result == NULL) @@ -664,19 +665,23 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) PyObject *ret = _PyObject_CallMethodId(result, &PyId_extend, "O", self); if (ret == NULL) { - Py_DECREF(result); - return NULL; + goto error; } Py_DECREF(ret); return result; } + it = PyObject_GetIter(self); + if (it == NULL) { + goto error; + } + while (1) { - PyObject *line = PyIter_Next(self); + Py_ssize_t line_length; + PyObject *line = PyIter_Next(it); if (line == NULL) { if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + goto error; } else break; /* StopIteration raised */ @@ -684,16 +689,25 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) if (PyList_Append(result, line) < 0) { Py_DECREF(line); - Py_DECREF(result); - return NULL; + goto error; } - length += PyObject_Size(line); + line_length = PyObject_Size(line); Py_DECREF(line); - - if (length > hint) + if (line_length < 0) { + goto error; + } + if (line_length > hint - length) break; + length += line_length; } + + Py_DECREF(it); return result; + + error: + Py_XDECREF(it); + Py_DECREF(result); + return NULL; } /*[clinic input] diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df55626d4bc23..4638b4759701f8 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -538,6 +538,12 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) _PyIO_str_getstate, NULL); if (state == NULL) return NULL; + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, + "illegal decoder state"); + Py_DECREF(state); + return NULL; + } if (!PyArg_ParseTuple(state, "OK", &buffer, &flag)) { Py_DECREF(state); return NULL; @@ -569,6 +575,10 @@ _io_IncrementalNewlineDecoder_setstate(nldecoder_object *self, PyObject *buffer; unsigned long long flag; + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "state argument must be a tuple"); + return NULL; + } if (!PyArg_ParseTuple(state, "OK", &buffer, &flag)) return NULL; @@ -933,6 +943,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, else { PyErr_SetString(PyExc_IOError, "could not determine default encoding"); + goto error; } /* Check we have been asked for a real text encoding */ @@ -1321,6 +1332,13 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) Py_DECREF(text); if (b == NULL) return NULL; + if (!PyBytes_Check(b)) { + PyErr_Format(PyExc_TypeError, + "encoder should return a bytes object, not '%.200s'", + Py_TYPE(b)->tp_name); + Py_DECREF(b); + return NULL; + } if (self->pending_bytes == NULL) { self->pending_bytes = PyList_New(0); @@ -1440,15 +1458,23 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) /* Given this, we know there was a valid snapshot point * len(dec_buffer) bytes ago with decoder state (b'', dec_flags). */ - if (PyArg_ParseTuple(state, "OO", &dec_buffer, &dec_flags) < 0) { + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, + "illegal decoder state"); + Py_DECREF(state); + return -1; + } + if (!PyArg_ParseTuple(state, + "OO;illegal decoder state", &dec_buffer, &dec_flags)) + { Py_DECREF(state); return -1; } if (!PyBytes_Check(dec_buffer)) { PyErr_Format(PyExc_TypeError, - "decoder getstate() should have returned a bytes " - "object, not '%.200s'", + "illegal decoder state: the first item should be a " + "bytes object, not '%.200s'", Py_TYPE(dec_buffer)->tp_name); Py_DECREF(state); return -1; @@ -2320,14 +2346,20 @@ _io_TextIOWrapper_tell_impl(textio *self) _PyIO_str_getstate, NULL); \ if (_state == NULL) \ goto fail; \ + if (!PyTuple_Check(_state)) { \ + PyErr_SetString(PyExc_TypeError, \ + "illegal decoder state"); \ + Py_DECREF(_state); \ + goto fail; \ + } \ if (!PyArg_ParseTuple(_state, "Oi", &dec_buffer, &dec_flags)) { \ Py_DECREF(_state); \ goto fail; \ } \ if (!PyBytes_Check(dec_buffer)) { \ PyErr_Format(PyExc_TypeError, \ - "decoder getstate() should have returned a bytes " \ - "object, not '%.200s'", \ + "illegal decoder state: the first item should be a " \ + "bytes object, not '%.200s'", \ Py_TYPE(dec_buffer)->tp_name); \ Py_DECREF(_state); \ goto fail; \ @@ -2483,6 +2515,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2523,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2546,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2562,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 1d169e2764ed27..5cf3f070d82526 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -79,7 +79,7 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { PyErr_Clear(); return '\0'; } - decoded_wstr = PyUnicode_AsWideCharString(decoded, NULL); + decoded_wstr = _PyUnicode_AsWideCharString(decoded); Py_CLEAR(decoded); if (!decoded_wstr) { PyErr_Clear(); @@ -101,7 +101,7 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { DWORD length; wchar_t name_buf[MAX_PATH], *pname_buf = name_buf; - + length = GetFullPathNameW(decoded_wstr, MAX_PATH, pname_buf, NULL); if (length > MAX_PATH) { pname_buf = PyMem_New(wchar_t, length); @@ -305,30 +305,23 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = fd; if (fd < 0) { - PyObject *decodedname = Py_None; - Py_INCREF(decodedname); + PyObject *decodedname; int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname); if (!d) return -1; - Py_ssize_t length; - name = PyUnicode_AsWideCharString(decodedname, &length); + name = _PyUnicode_AsWideCharString(decodedname); console_type = _PyIO_get_console_type(decodedname); Py_CLEAR(decodedname); if (name == NULL) return -1; if (console_type == '\0') { + PyMem_Free(name); PyErr_SetString(PyExc_ValueError, "Cannot open non-console file"); return -1; } - - if (wcslen(name) != length) { - PyMem_Free(name); - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return -1; - } } s = mode; @@ -408,7 +401,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, PyErr_SetString(PyExc_ValueError, "Cannot open non-console file"); goto error; - } + } if (self->writable && console_type != 'w') { PyErr_SetString(PyExc_ValueError, "Cannot open console input buffer for writing"); @@ -436,8 +429,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, internal_close(self); done: - if (name) - PyMem_Free(name); + PyMem_Free(name); return ret; } @@ -1017,7 +1009,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); - if (n < wlen) { + if (res && n < wlen) { /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the * characters that were written. As this could potentially diff --git a/Modules/_json.c b/Modules/_json.c index faa213491b6387..8f1743bada8753 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -89,16 +89,12 @@ static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); static void scanner_dealloc(PyObject *self); static int scanner_clear(PyObject *self); static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); static void encoder_dealloc(PyObject *self); static int @@ -659,7 +655,8 @@ py_encode_basestring(PyObject* self UNUSED, PyObject *pystr) static void scanner_dealloc(PyObject *self) { - /* Deallocate scanner object */ + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(self); scanner_clear(self); Py_TYPE(self)->tp_free(self); } @@ -1203,38 +1200,21 @@ static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->strict = NULL; - s->object_hook = NULL; - s->object_pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ PyObject *ctx; static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; + return NULL; - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s == NULL) { + return NULL; } + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + /* All of these will fail "gracefully" so we don't need to verify them */ s->strict = PyObject_GetAttrString(ctx, "strict"); if (s->strict == NULL) @@ -1255,16 +1235,11 @@ scanner_init(PyObject *self, PyObject *args, PyObject *kwds) if (s->parse_constant == NULL) goto bail; - return 0; + return (PyObject *)s; bail: - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->object_pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; + Py_DECREF(s); + return NULL; } PyDoc_STRVAR(scanner_doc, "JSON scanner object"); @@ -1306,7 +1281,7 @@ PyTypeObject PyScannerType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ + 0, /* tp_init */ 0,/* PyType_GenericAlloc, */ /* tp_alloc */ scanner_new, /* tp_new */ 0,/* PyObject_GC_Del, */ /* tp_free */ @@ -1315,25 +1290,6 @@ PyTypeObject PyScannerType = { static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; @@ -1341,22 +1297,23 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) PyObject *item_separator, *sort_keys, *skipkeys; int allow_nan; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUOOp:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) - return -1; + return NULL; if (markers != Py_None && !PyDict_Check(markers)) { PyErr_Format(PyExc_TypeError, "make_encoder() argument 1 must be dict or None, " "not %.200s", Py_TYPE(markers)->tp_name); - return -1; + return NULL; } + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s == NULL) + return NULL; + s->markers = markers; s->defaultfn = defaultfn; s->encoder = encoder; @@ -1383,7 +1340,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) Py_INCREF(s->item_separator); Py_INCREF(s->sort_keys); Py_INCREF(s->skipkeys); - return 0; + return (PyObject *)s; } static PyObject * @@ -1474,10 +1431,20 @@ static PyObject * encoder_encode_string(PyEncoderObject *s, PyObject *obj) { /* Return the JSON representation of a string */ - if (s->fast_encode) + PyObject *encoded; + + if (s->fast_encode) { return s->fast_encode(NULL, obj); - else - return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); + } + encoded = PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); + if (encoded != NULL && !PyUnicode_Check(encoded)) { + PyErr_Format(PyExc_TypeError, + "encoder() must return a string, not %.80s", + Py_TYPE(encoded)->tp_name); + Py_DECREF(encoded); + return NULL; + } + return encoded; } static int @@ -1645,8 +1612,10 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, if (items == NULL) goto bail; sortkeys = PyObject_IsTrue(s->sort_keys); - if (sortkeys < 0 || (sortkeys && PyList_Sort(items) < 0)) + if (sortkeys < 0 || (sortkeys && PyList_Sort(items) < 0)) { + Py_DECREF(items); goto bail; + } it = PyObject_GetIter(items); Py_DECREF(items); if (it == NULL) @@ -1835,7 +1804,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyAccu *acc, static void encoder_dealloc(PyObject *self) { - /* Deallocate Encoder */ + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(self); encoder_clear(self); Py_TYPE(self)->tp_free(self); } @@ -1914,7 +1884,7 @@ PyTypeObject PyEncoderType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ + 0, /* tp_init */ 0, /* tp_alloc */ encoder_new, /* tp_new */ 0, /* tp_free */ @@ -1957,10 +1927,8 @@ PyInit__json(void) PyObject *m = PyModule_Create(&jsonmodule); if (!m) return NULL; - PyScannerType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyScannerType) < 0) goto fail; - PyEncoderType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyEncoderType) < 0) goto fail; Py_INCREF((PyObject*)&PyScannerType); diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index feb38029008e30..71c9146ccb8f47 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -215,10 +215,10 @@ PyLocale_strcoll(PyObject* self, PyObject* args) if (!PyArg_ParseTuple(args, "UU:strcoll", &os1, &os2)) return NULL; /* Convert the unicode strings to wchar[]. */ - ws1 = PyUnicode_AsWideCharString(os1, NULL); + ws1 = _PyUnicode_AsWideCharString(os1); if (ws1 == NULL) goto done; - ws2 = PyUnicode_AsWideCharString(os2, NULL); + ws2 = _PyUnicode_AsWideCharString(os2); if (ws2 == NULL) goto done; /* Collate the strings. */ @@ -252,6 +252,11 @@ PyLocale_strxfrm(PyObject* self, PyObject* args) s = PyUnicode_AsWideCharString(str, &n1); if (s == NULL) goto exit; + if (wcslen(s) != (size_t)n1) { + PyErr_SetString(PyExc_ValueError, + "embedded null character"); + goto exit; + } /* assume no change in size, first */ n1 = n1 + 1; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 920b46fc28595f..ef0a03b107939f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1705,7 +1705,7 @@ whichmodule(PyObject *global, PyObject *dotted_path) /* If no module is found, use __main__. */ module_name = _PyUnicode_FromId(&PyId___main__); - Py_INCREF(module_name); + Py_XINCREF(module_name); return module_name; } diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 5007a39bc2443d..d1434d59f818b4 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -111,13 +111,17 @@ _is_fdescfs_mounted_on_dev_fd(void) static int _sanity_check_python_fd_sequence(PyObject *fd_sequence) { - Py_ssize_t seq_idx, seq_len = PySequence_Length(fd_sequence); + Py_ssize_t seq_idx; long prev_fd = -1; - for (seq_idx = 0; seq_idx < seq_len; ++seq_idx) { - PyObject* py_fd = PySequence_Fast_GET_ITEM(fd_sequence, seq_idx); - long iter_fd = PyLong_AsLong(py_fd); + for (seq_idx = 0; seq_idx < PyTuple_GET_SIZE(fd_sequence); ++seq_idx) { + PyObject* py_fd = PyTuple_GET_ITEM(fd_sequence, seq_idx); + long iter_fd; + if (!PyLong_Check(py_fd)) { + return 1; + } + iter_fd = PyLong_AsLong(py_fd); if (iter_fd < 0 || iter_fd <= prev_fd || iter_fd > INT_MAX) { - /* Negative, overflow, not a Long, unsorted, too big for a fd. */ + /* Negative, overflow, unsorted, too big for a fd. */ return 1; } prev_fd = iter_fd; @@ -132,13 +136,12 @@ _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) { /* Binary search. */ Py_ssize_t search_min = 0; - Py_ssize_t search_max = PySequence_Length(fd_sequence) - 1; + Py_ssize_t search_max = PyTuple_GET_SIZE(fd_sequence) - 1; if (search_max < 0) return 0; do { long middle = (search_min + search_max) / 2; - long middle_fd = PyLong_AsLong( - PySequence_Fast_GET_ITEM(fd_sequence, middle)); + long middle_fd = PyLong_AsLong(PyTuple_GET_ITEM(fd_sequence, middle)); if (fd == middle_fd) return 1; if (fd > middle_fd) @@ -154,9 +157,9 @@ make_inheritable(PyObject *py_fds_to_keep, int errpipe_write) { Py_ssize_t i, len; - len = PySequence_Length(py_fds_to_keep); + len = PyTuple_GET_SIZE(py_fds_to_keep); for (i = 0; i < len; ++i) { - PyObject* fdobj = PySequence_Fast_GET_ITEM(py_fds_to_keep, i); + PyObject* fdobj = PyTuple_GET_ITEM(py_fds_to_keep, i); long fd = PyLong_AsLong(fdobj); assert(!PyErr_Occurred()); assert(0 <= fd && fd <= INT_MAX); @@ -213,14 +216,13 @@ static void _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) { long end_fd = safe_get_max_fd(); - Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); Py_ssize_t keep_seq_idx; int fd_num; /* As py_fds_to_keep is sorted we can loop through the list closing * fds inbetween any in the keep list falling within our range. */ for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { - PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep, - keep_seq_idx); + PyObject* py_keep_fd = PyTuple_GET_ITEM(py_fds_to_keep, keep_seq_idx); int keep_fd = PyLong_AsLong(py_keep_fd); if (keep_fd < start_fd) continue; @@ -306,7 +308,7 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) /* Close all open file descriptors from start_fd and higher. - * Do not close any in the sorted py_fds_to_keep list. + * Do not close any in the sorted py_fds_to_keep tuple. * * This function violates the strict use of async signal safe functions. :( * It calls opendir(), readdir() and closedir(). Of these, the one most @@ -562,8 +564,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) #endif if (!PyArg_ParseTuple( - args, "OOpOOOiiiiiiiiiiO:fork_exec", - &process_args, &executable_list, &close_fds, &py_fds_to_keep, + args, "OOpO!OOiiiiiiiiiiO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, @@ -574,10 +577,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); return NULL; } - if (PySequence_Length(py_fds_to_keep) < 0) { - PyErr_SetString(PyExc_ValueError, "cannot get length of fds_to_keep"); - return NULL; - } if (_sanity_check_python_fd_sequence(py_fds_to_keep)) { PyErr_SetString(PyExc_ValueError, "bad value(s) in fds_to_keep"); return NULL; @@ -631,6 +630,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) goto cleanup; for (arg_num = 0; arg_num < num_args; ++arg_num) { PyObject *borrowed_arg, *converted_arg; + if (PySequence_Fast_GET_SIZE(fast_args) != num_args) { + PyErr_SetString(PyExc_RuntimeError, "args changed during iteration"); + goto cleanup; + } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) goto cleanup; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0d3282db8f2810..769084fa88ddef 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -2,7 +2,7 @@ /* ------------------------------------------------------------------ The code in this module was based on a download from: - http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html It was modified in 2002 by Raymond Hettinger as follows: @@ -60,8 +60,8 @@ Any feedback is very welcome. - http://www.math.keio.ac.jp/matumoto/emt.html - email: matumoto@math.keio.ac.jp + http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html + email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ /* ---------------------------------------------------------------*/ @@ -259,8 +259,11 @@ random_seed(RandomObject *self, PyObject *args) * So: if the arg is a PyLong, use its absolute value. * Otherwise use its hash value, cast to unsigned. */ - if (PyLong_Check(arg)) - n = PyNumber_Absolute(arg); + if (PyLong_Check(arg)) { + /* Calling int.__abs__() prevents calling arg.__abs__(), which might + return an invalid value. See issue #31478. */ + n = PyLong_Type.tp_as_number->nb_absolute(arg); + } else { Py_hash_t hash = PyObject_Hash(arg); if (hash == -1) @@ -348,6 +351,7 @@ random_setstate(RandomObject *self, PyObject *state) int i; unsigned long element; long index; + uint32_t new_state[N]; if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, @@ -364,7 +368,7 @@ random_setstate(RandomObject *self, PyObject *state) element = PyLong_AsUnsignedLong(PyTuple_GET_ITEM(state, i)); if (element == (unsigned long)-1 && PyErr_Occurred()) return NULL; - self->state[i] = (uint32_t)element; + new_state[i] = (uint32_t)element; } index = PyLong_AsLong(PyTuple_GET_ITEM(state, i)); @@ -375,6 +379,8 @@ random_setstate(RandomObject *self, PyObject *state) return NULL; } self->index = (int)index; + for (i = 0; i < N; i++) + self->state[i] = new_state[i]; Py_INCREF(Py_None); return Py_None; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 39f7a6508c2aef..8341fb8480172c 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -511,10 +511,9 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - /* For backwards compatibility reasons, do not start a transaction if a - DDL statement is encountered. If anybody wants transactional DDL, - they can issue a BEGIN statement manually. */ - if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + /* We start a transaction implicitly before a DML statement. + SELECT is the only exception. See #9924. */ + if (self->connection->begin_statement && self->statement->is_dml) { if (sqlite3_get_autocommit(self->connection->db)) { result = _pysqlite_connection_begin(self->connection); if (!result) { @@ -609,7 +608,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } } - if (!sqlite3_stmt_readonly(self->statement->st)) { + if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } else { self->rowcount= -1L; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 0df661b9c7ad31..087375be9b63d8 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -73,8 +73,9 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con Py_INCREF(sql); self->sql = sql; - /* determine if the statement is a DDL statement */ - self->is_ddl = 0; + /* Determine if the statement is a DML statement. + SELECT is the only exception. See #9924. */ + self->is_dml = 0; for (p = sql_cstr; *p != 0; p++) { switch (*p) { case ' ': @@ -84,9 +85,10 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con continue; } - self->is_ddl = (PyOS_strnicmp(p, "create ", 7) == 0) - || (PyOS_strnicmp(p, "drop ", 5) == 0) - || (PyOS_strnicmp(p, "reindex ", 8) == 0); + self->is_dml = (PyOS_strnicmp(p, "insert ", 7) == 0) + || (PyOS_strnicmp(p, "update ", 7) == 0) + || (PyOS_strnicmp(p, "delete ", 7) == 0) + || (PyOS_strnicmp(p, "replace ", 8) == 0); break; } diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h index 6eef16857f7b85..8db10f6649ce80 100644 --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -38,7 +38,7 @@ typedef struct sqlite3_stmt* st; PyObject* sql; int in_use; - int is_ddl; + int is_dml; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b1988570604db1..a32fce47ef641a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -279,7 +279,7 @@ static unsigned int _ssl_locks_count = 0; typedef struct { PyObject_HEAD SSL_CTX *ctx; -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) unsigned char *npn_protocols; int npn_protocols_len; #endif @@ -298,12 +298,15 @@ typedef struct { PyObject *Socket; /* weakref to socket on which we're layered */ SSL *ssl; PySSLContext *ctx; /* weakref to SSL context */ - X509 *peer_cert; char shutdown_seen_zero; - char handshake_done; enum py_ssl_server_or_client socket_type; PyObject *owner; /* Python level "owner" passed to servername callback */ PyObject *server_hostname; + int ssl_errno; /* last seen error from SSL */ + int c_errno; /* last seen error from libc */ +#ifdef MS_WINDOWS + int ws_errno; /* last seen error from winsock */ +#endif } PySSLSocket; typedef struct { @@ -323,6 +326,20 @@ static PyTypeObject PySSLSocket_Type; static PyTypeObject PySSLMemoryBIO_Type; static PyTypeObject PySSLSession_Type; +#ifdef MS_WINDOWS +#define _PySSL_UPDATE_ERRNO_IF(cond, sock, retcode) if (cond) { \ + (sock)->ws_errno = WSAGetLastError(); \ + (sock)->c_errno = errno; \ + (sock)->ssl_errno = SSL_get_error((sock->ssl), (retcode)); \ + } else { sock->ws_errno = 0; sock->c_errno = 0; sock->ssl_errno = 0; } +#else +#define _PySSL_UPDATE_ERRNO_IF(cond, sock, retcode) if (cond) { \ + (sock)->c_errno = errno; \ + (sock)->ssl_errno = SSL_get_error((sock->ssl), (retcode)); \ + } else { (sock)->c_errno = 0; (sock)->ssl_errno = 0; } +#endif +#define _PySSL_UPDATE_ERRNO(sock, retcode) _PySSL_UPDATE_ERRNO_IF(1, (sock), (retcode)) + /*[clinic input] module _ssl class _ssl._SSLContext "PySSLContext *" "&PySSLContext_Type" @@ -496,7 +513,7 @@ PySSL_SetError(PySSLSocket *obj, int ret, const char *filename, int lineno) e = ERR_peek_last_error(); if (obj->ssl != NULL) { - err = SSL_get_error(obj->ssl, ret); + err = obj->ssl_errno; switch (err) { case SSL_ERROR_ZERO_RETURN: @@ -532,8 +549,16 @@ PySSL_SetError(PySSLSocket *obj, int ret, const char *filename, int lineno) errstr = "EOF occurred in violation of protocol"; } else if (s && ret == -1) { /* underlying BIO reported an I/O error */ - Py_INCREF(s); ERR_clear_error(); +#ifdef MS_WINDOWS + if (obj->ws_errno) + return PyErr_SetFromWindowsErr(obj->ws_errno); +#endif + if (obj->c_errno) { + errno = obj->c_errno; + return PyErr_SetFromErrno(PyExc_OSError); + } + Py_INCREF(s); s->errorhandler(); Py_DECREF(s); return NULL; @@ -595,12 +620,11 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, if (self == NULL) return NULL; - self->peer_cert = NULL; self->ssl = NULL; self->Socket = NULL; self->ctx = sslctx; + Py_INCREF(sslctx); self->shutdown_seen_zero = 0; - self->handshake_done = 0; self->owner = NULL; self->server_hostname = NULL; if (server_hostname != NULL) { @@ -612,8 +636,11 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } self->server_hostname = hostname; } - - Py_INCREF(sslctx); + self->ssl_errno = 0; + self->c_errno = 0; +#ifdef MS_WINDOWS + self->ws_errno = 0; +#endif /* Make sure the SSL error state is initialized */ (void) ERR_get_state(); @@ -711,8 +738,9 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) do { PySSL_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); - err = SSL_get_error(self->ssl, ret); + _PySSL_UPDATE_ERRNO_IF(ret < 1, self, ret); PySSL_END_ALLOW_THREADS + err = self->ssl_errno; if (PyErr_CheckSignals()) goto error; @@ -748,15 +776,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) if (ret < 1) return PySSL_SetError(self, ret, __FILE__, __LINE__); - if (self->peer_cert) - X509_free (self->peer_cert); - PySSL_BEGIN_ALLOW_THREADS - self->peer_cert = SSL_get_peer_certificate(self->ssl); - PySSL_END_ALLOW_THREADS - self->handshake_done = 1; - - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; error: Py_XDECREF(sock); @@ -764,49 +784,64 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) } static PyObject * -_create_tuple_for_attribute (ASN1_OBJECT *name, ASN1_STRING *value) { - - char namebuf[X509_NAME_MAXLEN]; +_asn1obj2py(const ASN1_OBJECT *name, int no_name) +{ + char buf[X509_NAME_MAXLEN]; + char *namebuf = buf; int buflen; - PyObject *name_obj; - PyObject *value_obj; - PyObject *attr; - unsigned char *valuebuf = NULL; + PyObject *name_obj = NULL; - buflen = OBJ_obj2txt(namebuf, sizeof(namebuf), name, 0); + buflen = OBJ_obj2txt(namebuf, X509_NAME_MAXLEN, name, no_name); if (buflen < 0) { _setSSLError(NULL, 0, __FILE__, __LINE__); - goto fail; + return NULL; } - name_obj = PyUnicode_FromStringAndSize(namebuf, buflen); - if (name_obj == NULL) - goto fail; + /* initial buffer is too small for oid + terminating null byte */ + if (buflen > X509_NAME_MAXLEN - 1) { + /* make OBJ_obj2txt() calculate the required buflen */ + buflen = OBJ_obj2txt(NULL, 0, name, no_name); + /* allocate len + 1 for terminating NULL byte */ + namebuf = PyMem_Malloc(buflen + 1); + if (namebuf == NULL) { + PyErr_NoMemory(); + return NULL; + } + buflen = OBJ_obj2txt(namebuf, buflen + 1, name, no_name); + if (buflen < 0) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + goto done; + } + } + if (!buflen && no_name) { + Py_INCREF(Py_None); + name_obj = Py_None; + } + else { + name_obj = PyUnicode_FromStringAndSize(namebuf, buflen); + } + + done: + if (buf != namebuf) { + PyMem_Free(namebuf); + } + return name_obj; +} + +static PyObject * +_create_tuple_for_attribute(ASN1_OBJECT *name, ASN1_STRING *value) +{ + Py_ssize_t buflen; + unsigned char *valuebuf = NULL; + PyObject *attr; buflen = ASN1_STRING_to_UTF8(&valuebuf, value); if (buflen < 0) { _setSSLError(NULL, 0, __FILE__, __LINE__); - Py_DECREF(name_obj); - goto fail; + return NULL; } - value_obj = PyUnicode_DecodeUTF8((char *) valuebuf, - buflen, "strict"); + attr = Py_BuildValue("Ns#", _asn1obj2py(name, 0), valuebuf, buflen); OPENSSL_free(valuebuf); - if (value_obj == NULL) { - Py_DECREF(name_obj); - goto fail; - } - attr = PyTuple_New(2); - if (attr == NULL) { - Py_DECREF(name_obj); - Py_DECREF(value_obj); - goto fail; - } - PyTuple_SET_ITEM(attr, 0, name_obj); - PyTuple_SET_ITEM(attr, 1, value_obj); return attr; - - fail: - return NULL; } static PyObject * @@ -1210,10 +1245,6 @@ _get_crl_dp(X509 *certificate) { int i, j; PyObject *lst, *res = NULL; -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - /* Calls x509v3_cache_extensions and sets up crldp */ - X509_check_ca(certificate); -#endif dps = X509_get_ext_d2i(certificate, NID_crl_distribution_points, NULL, NULL); if (dps == NULL) @@ -1258,9 +1289,7 @@ _get_crl_dp(X509 *certificate) { done: Py_XDECREF(lst); -#if OPENSSL_VERSION_NUMBER < 0x10001000L - sk_DIST_POINT_free(dps); -#endif + CRL_DIST_POINTS_free(dps); return res; } @@ -1513,25 +1542,30 @@ _ssl__SSLSocket_peer_certificate_impl(PySSLSocket *self, int binary_mode) /*[clinic end generated code: output=f0dc3e4d1d818a1d input=8281bd1d193db843]*/ { int verification; + X509 *peer_cert; + PyObject *result; - if (!self->handshake_done) { + if (!SSL_is_init_finished(self->ssl)) { PyErr_SetString(PyExc_ValueError, "handshake not done yet"); return NULL; } - if (!self->peer_cert) + peer_cert = SSL_get_peer_certificate(self->ssl); + if (peer_cert == NULL) Py_RETURN_NONE; if (binary_mode) { /* return cert in DER-encoded format */ - return _certificate_to_der(self->peer_cert); + result = _certificate_to_der(peer_cert); } else { verification = SSL_CTX_get_verify_mode(SSL_get_SSL_CTX(self->ssl)); if ((verification & SSL_VERIFY_PEER) == 0) - return PyDict_New(); + result = PyDict_New(); else - return _decode_certificate(self->peer_cert); + result = _decode_certificate(peer_cert); } + X509_free(peer_cert); + return result; } static PyObject * @@ -1694,13 +1728,17 @@ _ssl__SSLSocket_version_impl(PySSLSocket *self) if (self->ssl == NULL) Py_RETURN_NONE; + if (!SSL_is_init_finished(self->ssl)) { + /* handshake not finished */ + Py_RETURN_NONE; + } version = SSL_get_version(self->ssl); if (!strcmp(version, "unknown")) Py_RETURN_NONE; return PyUnicode_FromString(version); } -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) /*[clinic input] _ssl._SSLSocket.selected_npn_protocol [clinic start generated code]*/ @@ -1852,8 +1890,6 @@ Passed as \"self\" in servername callback."); static void PySSL_dealloc(PySSLSocket *self) { - if (self->peer_cert) /* Possible not to have one? */ - X509_free (self->peer_cert); if (self->ssl) SSL_free(self->ssl); Py_XDECREF(self->Socket); @@ -2000,8 +2036,9 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) do { PySSL_BEGIN_ALLOW_THREADS len = SSL_write(self->ssl, b->buf, (int)b->len); - err = SSL_get_error(self->ssl, len); + _PySSL_UPDATE_ERRNO_IF(len <= 0, self, len); PySSL_END_ALLOW_THREADS + err = self->ssl_errno; if (PyErr_CheckSignals()) goto error; @@ -2055,6 +2092,7 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) PySSL_BEGIN_ALLOW_THREADS count = SSL_pending(self->ssl); + _PySSL_UPDATE_ERRNO_IF(count < 0, self, count); PySSL_END_ALLOW_THREADS if (count < 0) return PySSL_SetError(self, count, __FILE__, __LINE__); @@ -2143,7 +2181,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, int len, int group_right_1, do { PySSL_BEGIN_ALLOW_THREADS count = SSL_read(self->ssl, mem, len); - err = SSL_get_error(self->ssl, count); + _PySSL_UPDATE_ERRNO_IF(count <= 0, self, count); PySSL_END_ALLOW_THREADS if (PyErr_CheckSignals()) @@ -2152,6 +2190,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, int len, int group_right_1, if (has_timeout) timeout = deadline - _PyTime_GetMonotonicClock(); + err = self->ssl_errno; if (err == SSL_ERROR_WANT_READ) { sockstate = PySSL_select(sock, 0, timeout); } else if (err == SSL_ERROR_WANT_WRITE) { @@ -2208,7 +2247,7 @@ static PyObject * _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) /*[clinic end generated code: output=ca1aa7ed9d25ca42 input=ede2cc1a2ddf0ee4]*/ { - int err, ssl_err, sockstate, nonblocking; + int err, sockstate, nonblocking; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); _PyTime_t timeout, deadline = 0; @@ -2247,6 +2286,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) if (self->shutdown_seen_zero) SSL_set_read_ahead(self->ssl, 0); err = SSL_shutdown(self->ssl); + _PySSL_UPDATE_ERRNO_IF(err < 0, self, err); PySSL_END_ALLOW_THREADS /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ @@ -2267,16 +2307,16 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) timeout = deadline - _PyTime_GetMonotonicClock(); /* Possibly retry shutdown until timeout or failure */ - ssl_err = SSL_get_error(self->ssl, err); - if (ssl_err == SSL_ERROR_WANT_READ) + _PySSL_UPDATE_ERRNO(self, err); + if (self->ssl_errno == SSL_ERROR_WANT_READ) sockstate = PySSL_select(sock, 0, timeout); - else if (ssl_err == SSL_ERROR_WANT_WRITE) + else if (self->ssl_errno == SSL_ERROR_WANT_WRITE) sockstate = PySSL_select(sock, 1, timeout); else break; if (sockstate == SOCKET_HAS_TIMED_OUT) { - if (ssl_err == SSL_ERROR_WANT_READ) + if (self->ssl_errno == SSL_ERROR_WANT_READ) PyErr_SetString(PySocketModule.timeout_error, "The read operation timed out"); else @@ -2449,7 +2489,7 @@ static int PySSL_set_session(PySSLSocket *self, PyObject *value, "Cannot set session for server-side SSLSocket."); return -1; } - if (self->handshake_done) { + if (SSL_is_init_finished(self->ssl)) { PyErr_SetString(PyExc_ValueError, "Cannot set session after handshake."); return -1; @@ -2640,8 +2680,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } if (ctx == NULL) { - PyErr_SetString(PySSLErrorObject, - "failed to allocate SSL context"); + _setSSLError(NULL, 0, __FILE__, __LINE__); return NULL; } @@ -2652,7 +2691,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } self->ctx = ctx; -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) self->npn_protocols = NULL; #endif #ifdef HAVE_ALPN @@ -2729,12 +2768,12 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif -#ifndef OPENSSL_NO_ECDH +#if !defined(OPENSSL_NO_ECDH) && !defined(OPENSSL_VERSION_1_1) /* Allow automatic ECDH curve selection (on OpenSSL 1.0.2+), or use prime256v1 by default. This is Apache mod_ssl's initialization policy, so we should be safe. OpenSSL 1.1 has it enabled by default. */ -#if defined(SSL_CTX_set_ecdh_auto) && !defined(OPENSSL_VERSION_1_1) +#if defined(SSL_CTX_set_ecdh_auto) SSL_CTX_set_ecdh_auto(self->ctx, 1); #else { @@ -2783,9 +2822,11 @@ context_clear(PySSLContext *self) static void context_dealloc(PySSLContext *self) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(self); context_clear(self); SSL_CTX_free(self->ctx); -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) PyMem_FREE(self->npn_protocols); #endif #ifdef HAVE_ALPN @@ -2863,7 +2904,7 @@ _ssl__SSLContext_get_ciphers_impl(PySSLContext *self) #endif -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) || defined(HAVE_ALPN) static int do_protocol_selection(int alpn, unsigned char **out, unsigned char *outlen, const unsigned char *server_protocols, unsigned int server_protocols_len, @@ -2887,7 +2928,9 @@ do_protocol_selection(int alpn, unsigned char **out, unsigned char *outlen, return SSL_TLSEXT_ERR_OK; } +#endif +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) /* this callback gets passed to SSL_CTX_set_next_protos_advertise_cb */ static int _advertiseNPN_cb(SSL *s, @@ -2930,7 +2973,7 @@ _ssl__SSLContext__set_npn_protocols_impl(PySSLContext *self, Py_buffer *protos) /*[clinic end generated code: output=72b002c3324390c6 input=319fcb66abf95bd7]*/ { -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) PyMem_Free(self->npn_protocols); self->npn_protocols = PyMem_Malloc(protos->len); if (self->npn_protocols == NULL) @@ -2981,6 +3024,12 @@ _ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, /*[clinic end generated code: output=87599a7f76651a9b input=9bba964595d519be]*/ { #ifdef HAVE_ALPN + if ((size_t)protos->len > UINT_MAX) { + PyErr_Format(PyExc_OverflowError, + "protocols longer than %d bytes", UINT_MAX); + return NULL; + } + PyMem_FREE(self->alpn_protocols); self->alpn_protocols = PyMem_Malloc(protos->len); if (!self->alpn_protocols) @@ -4291,6 +4340,7 @@ static PyTypeObject PySSLMemoryBIO_Type = { static void PySSLSession_dealloc(PySSLSession *self) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); Py_XDECREF(self->ctx); if (self->session != NULL) { @@ -4482,12 +4532,12 @@ _ssl.RAND_add Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in -string. See RFC 1750. +string. See RFC 4086. [clinic start generated code]*/ static PyObject * _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) -/*[clinic end generated code: output=e6dd48df9c9024e9 input=580c85e6a3a4fe29]*/ +/*[clinic end generated code: output=e6dd48df9c9024e9 input=5c33017422828f5c]*/ { const char *buf; Py_ssize_t len, written; @@ -4671,8 +4721,6 @@ asn1obj2py(ASN1_OBJECT *obj) { int nid; const char *ln, *sn; - char buf[100]; - Py_ssize_t buflen; nid = OBJ_obj2nid(obj); if (nid == NID_undef) { @@ -4681,16 +4729,7 @@ asn1obj2py(ASN1_OBJECT *obj) } sn = OBJ_nid2sn(nid); ln = OBJ_nid2ln(nid); - buflen = OBJ_obj2txt(buf, sizeof(buf), obj, 1); - if (buflen < 0) { - _setSSLError(NULL, 0, __FILE__, __LINE__); - return NULL; - } - if (buflen) { - return Py_BuildValue("isss#", nid, sn, ln, buf, buflen); - } else { - return Py_BuildValue("issO", nid, sn, ln, Py_None); - } + return Py_BuildValue("issN", nid, sn, ln, _asn1obj2py(obj, 1)); } /*[clinic input] @@ -5173,9 +5212,14 @@ PyInit__ssl(void) return NULL; PySocketModule = *socket_api; +#ifndef OPENSSL_VERSION_1_1 + /* Load all algorithms and initialize cpuid */ + OPENSSL_add_all_algorithms_noconf(); /* Init OpenSSL */ SSL_load_error_strings(); SSL_library_init(); +#endif + #ifdef WITH_THREAD #ifdef HAVE_OPENSSL_CRYPTO_LOCK /* note that this will start threading if not already started */ @@ -5187,7 +5231,6 @@ PyInit__ssl(void) _ssl_locks_count++; #endif #endif /* WITH_THREAD */ - OpenSSL_add_all_algorithms(); /* Add symbols to module dict */ sslerror_type_slots[0].pfunc = PyExc_OSError; @@ -5362,6 +5405,11 @@ PyInit__ssl(void) #if HAVE_TLSv1_2 PyModule_AddIntConstant(m, "OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); PyModule_AddIntConstant(m, "OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); +#endif +#ifdef SSL_OP_NO_TLSv1_3 + PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); +#else + PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", 0); #endif PyModule_AddIntConstant(m, "OP_CIPHER_SERVER_PREFERENCE", SSL_OP_CIPHER_SERVER_PREFERENCE); @@ -5395,7 +5443,7 @@ PyInit__ssl(void) Py_INCREF(r); PyModule_AddObject(m, "HAS_ECDH", r); -#ifdef OPENSSL_NPN_NEGOTIATED +#if defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG) r = Py_True; #else r = Py_False; @@ -5411,6 +5459,14 @@ PyInit__ssl(void) Py_INCREF(r); PyModule_AddObject(m, "HAS_ALPN", r); +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + r = Py_True; +#else + r = Py_False; +#endif + Py_INCREF(r); + PyModule_AddObject(m, "HAS_TLSv1_3", r); + /* Mappings for error codes */ err_codes_to_names = PyDict_New(); err_names_to_codes = PyDict_New(); diff --git a/Modules/_struct.c b/Modules/_struct.c index 796d1682f094f1..cd3fa2d150673b 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -539,7 +539,7 @@ np_ubyte(char *p, PyObject *v, const formatdef *f) "ubyte format requires 0 <= number <= 255"); return -1; } - *p = (char)x; + *(unsigned char *)p = (unsigned char)x; return 0; } @@ -868,6 +868,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -880,7 +881,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffL); x >>= 8; } while (i > 0); return 0; @@ -891,6 +892,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -901,7 +903,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffUL); x >>= 8; } while (i > 0); return 0; @@ -1087,6 +1089,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -1099,7 +1102,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffL); x >>= 8; } while (--i > 0); return 0; @@ -1110,6 +1113,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -1120,7 +1124,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffUL); x >>= 8; } while (--i > 0); return 0; @@ -1456,7 +1460,8 @@ s_init(PyObject *self, PyObject *args, PyObject *kwds) if (!PyBytes_Check(o_format)) { Py_DECREF(o_format); PyErr_Format(PyExc_TypeError, - "Struct() argument 1 must be a bytes object, not %.200s", + "Struct() argument 1 must be a str or bytes object, " + "not %.200s", Py_TYPE(o_format)->tp_name); return -1; } @@ -1537,7 +1542,7 @@ s_unpack(PyObject *self, PyObject *input) return NULL; if (vbuf.len != soself->s_size) { PyErr_Format(StructError, - "unpack requires a bytes object of length %zd", + "unpack requires a buffer of %zd bytes", soself->s_size); PyBuffer_Release(&vbuf); return NULL; @@ -1601,6 +1606,8 @@ typedef struct { static void unpackiter_dealloc(unpackiterobject *self) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + PyObject_GC_UnTrack(self); Py_XDECREF(self->so); PyBuffer_Release(&self->buf); PyObject_GC_Del(self); @@ -1712,8 +1719,8 @@ s_iter_unpack(PyObject *_so, PyObject *input) } if (self->buf.len % so->s_size != 0) { PyErr_Format(StructError, - "iterative unpacking requires a bytes length " - "multiple of %zd", + "iterative unpacking requires a buffer of " + "a multiple of %zd bytes", so->s_size); Py_DECREF(self); return NULL; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce6851a5eb6..6b8ab34d931cd3 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f63cc5ab..ede675da28505c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,6 +18,10 @@ # include /* struct timeval */ #endif +#ifdef HAVE_SYS_WAIT_H +#include /* For W_STOPCODE */ +#endif + #ifdef WITH_THREAD #include "pythread.h" #endif /* WITH_THREAD */ @@ -866,8 +870,9 @@ test_L_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = -1; - if (PyArg_ParseTuple(tuple, "L:test_L_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { return NULL; + } if (value != 42) return raiseTestError("test_L_code", "L code returned wrong value for long 42"); @@ -880,8 +885,9 @@ test_L_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = -1; - if (PyArg_ParseTuple(tuple, "L:test_L_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { return NULL; + } if (value != 42) return raiseTestError("test_L_code", "L code returned wrong value for int 42"); @@ -1198,8 +1204,9 @@ test_k_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = 0; - if (PyArg_ParseTuple(tuple, "k:test_k_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { return NULL; + } if (value != ULONG_MAX) return raiseTestError("test_k_code", "k code returned wrong value for long 0xFFF...FFF"); @@ -1217,8 +1224,9 @@ test_k_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = 0; - if (PyArg_ParseTuple(tuple, "k:test_k_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { return NULL; + } if (value != (unsigned long)-0x42) return raiseTestError("test_k_code", "k code returned wrong value for long -0xFFF..000042"); @@ -1556,11 +1564,13 @@ test_s_code(PyObject *self) /* These two blocks used to raise a TypeError: * "argument must be string without null bytes, not str" */ - if (PyArg_ParseTuple(tuple, "s:test_s_code1", &value) < 0) - return NULL; + if (!PyArg_ParseTuple(tuple, "s:test_s_code1", &value)) { + return NULL; + } - if (PyArg_ParseTuple(tuple, "z:test_s_code2", &value) < 0) - return NULL; + if (!PyArg_ParseTuple(tuple, "z:test_s_code2", &value)) { + return NULL; + } Py_DECREF(tuple); Py_RETURN_NONE; @@ -1571,7 +1581,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) { PyObject *sub_args; PyObject *sub_kwargs; - char *sub_format; + const char *sub_format; PyObject *sub_keywords; Py_ssize_t i, size; @@ -1584,7 +1594,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) double buffers[8][4]; /* double ensures alignment where necessary */ - if (!PyArg_ParseTuple(args, "OOyO:parse_tuple_and_keywords", + if (!PyArg_ParseTuple(args, "OOsO:parse_tuple_and_keywords", &sub_args, &sub_kwargs, &sub_format, &sub_keywords)) return NULL; @@ -1662,14 +1672,16 @@ test_u_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, obj); value = 0; - if (PyArg_ParseTuple(tuple, "u:test_u_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "u:test_u_code", &value)) { return NULL; + } if (value != PyUnicode_AS_UNICODE(obj)) return raiseTestError("test_u_code", "u code returned wrong value for u'test'"); value = 0; - if (PyArg_ParseTuple(tuple, "u#:test_u_code", &value, &len) < 0) + if (!PyArg_ParseTuple(tuple, "u#:test_u_code", &value, &len)) { return NULL; + } if (value != PyUnicode_AS_UNICODE(obj) || len != PyUnicode_GET_SIZE(obj)) return raiseTestError("test_u_code", @@ -1702,8 +1714,9 @@ test_Z_code(PyObject *self) value2 = PyUnicode_AS_UNICODE(obj); /* Test Z for both values */ - if (PyArg_ParseTuple(tuple, "ZZ:test_Z_code", &value1, &value2) < 0) + if (!PyArg_ParseTuple(tuple, "ZZ:test_Z_code", &value1, &value2)) { return NULL; + } if (value1 != PyUnicode_AS_UNICODE(obj)) return raiseTestError("test_Z_code", "Z code returned wrong value for 'test'"); @@ -1717,9 +1730,11 @@ test_Z_code(PyObject *self) len2 = -1; /* Test Z# for both values */ - if (PyArg_ParseTuple(tuple, "Z#Z#:test_Z_code", &value1, &len1, - &value2, &len2) < 0) + if (!PyArg_ParseTuple(tuple, "Z#Z#:test_Z_code", &value1, &len1, + &value2, &len2)) + { return NULL; + } if (value1 != PyUnicode_AS_UNICODE(obj) || len1 != PyUnicode_GET_SIZE(obj)) return raiseTestError("test_Z_code", @@ -2024,8 +2039,9 @@ test_empty_argparse(PyObject *self) tuple = PyTuple_New(0); if (!tuple) return NULL; - if ((result = PyArg_ParseTuple(tuple, "|:test_empty_argparse")) < 0) + if (!(result = PyArg_ParseTuple(tuple, "|:test_empty_argparse"))) { goto done; + } dict = PyDict_New(); if (!dict) goto done; @@ -2033,8 +2049,9 @@ test_empty_argparse(PyObject *self) done: Py_DECREF(tuple); Py_XDECREF(dict); - if (result < 0) + if (!result) { return NULL; + } else { Py_RETURN_NONE; } @@ -3569,8 +3586,9 @@ test_raise_signal(PyObject* self, PyObject *args) { int signum, err; - if (PyArg_ParseTuple(args, "i:raise_signal", &signum) < 0) + if (!PyArg_ParseTuple(args, "i:raise_signal", &signum)) { return NULL; + } err = raise(signum); if (err) @@ -3812,7 +3830,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); @@ -4027,6 +4045,141 @@ dict_get_version(PyObject *self, PyObject *args) } +static int +fastcall_args(PyObject *args, PyObject ***stack, Py_ssize_t *nargs) +{ + if (args == Py_None) { + *stack = NULL; + *nargs = 0; + } + else if (PyTuple_Check(args)) { + *stack = &PyTuple_GET_ITEM(args, 0); + *nargs = PyTuple_GET_SIZE(args); + } + else { + PyErr_SetString(PyExc_TypeError, "args must be None or a tuple"); + return -1; + } + return 0; +} + + +static PyObject * +test_pyobject_fastcall(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OO", &func, &func_args)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + return _PyObject_FastCall(func, stack, nargs); +} + + +static PyObject * +test_pyobject_fastcalldict(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwargs; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwargs)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwargs == Py_None) { + kwargs = NULL; + } + else if (!PyDict_Check(kwargs)) { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a dict"); + return NULL; + } + + return _PyObject_FastCallDict(func, stack, nargs, kwargs); +} + + +static PyObject * +test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwnames = NULL; + PyObject **stack; + Py_ssize_t nargs, nkw; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwnames)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwnames == Py_None) { + kwnames = NULL; + } + else if (PyTuple_Check(kwnames)) { + nkw = PyTuple_GET_SIZE(kwnames); + if (nargs < nkw) { + PyErr_SetString(PyExc_ValueError, "kwnames longer than args"); + return NULL; + } + nargs -= nkw; + } + else { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a tuple"); + return NULL; + } + return _PyObject_FastCallKeywords(func, stack, nargs, kwnames); +} + + +static PyObject * +raise_SIGINT_then_send_None(PyObject *self, PyObject *args) +{ + PyGenObject *gen; + + if (!PyArg_ParseTuple(args, "O!", &PyGen_Type, &gen)) + return NULL; + + /* This is used in a test to check what happens if a signal arrives just + as we're in the process of entering a yield from chain (see + bpo-30039). + + Needs to be done in C, because: + - we don't have a Python wrapper for raise() + - we need to make sure that the Python-level signal handler doesn't run + *before* we enter the generator frame, which is impossible in Python + because we check for signals before every bytecode operation. + */ + raise(SIGINT); + return _PyGen_Send(gen, Py_None); +} + + +#ifdef W_STOPCODE +static PyObject* +py_w_stopcode(PyObject *self, PyObject *args) +{ + int sig, status; + if (!PyArg_ParseTuple(args, "i", &sig)) { + return NULL; + } + status = W_STOPCODE(sig); + return PyLong_FromLong(status); +} +#endif + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4230,6 +4383,13 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, + {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, + {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, + {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, + {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, +#ifdef W_STOPCODE + {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 3e96aa0cd50c64..68ee26e790ccaf 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -1668,7 +1668,7 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) } /*[clinic input] -_tkinter.tkapp.adderrinfo +_tkinter.tkapp.adderrorinfo msg: str / @@ -1676,8 +1676,8 @@ _tkinter.tkapp.adderrinfo [clinic start generated code]*/ static PyObject * -_tkinter_tkapp_adderrinfo_impl(TkappObject *self, const char *msg) -/*[clinic end generated code: output=0e222ee2050eb357 input=4971399317d4c136]*/ +_tkinter_tkapp_adderrorinfo_impl(TkappObject *self, const char *msg) +/*[clinic end generated code: output=52162eaca2ee53cb input=f4b37aec7c7e8c77]*/ { CHECK_STRING_LENGTH(msg); CHECK_TCL_APPARTMENT; @@ -2291,7 +2291,11 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) if (!PyArg_Parse(arg, "et:splitlist", "utf-8", &list)) return NULL; - CHECK_STRING_LENGTH(list); + if (strlen(list) >= INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "string is too long"); + PyMem_Free(list); + return NULL; + } if (Tcl_SplitList(Tkapp_Interp(self), list, &argc, &argv) == TCL_ERROR) { PyMem_Free(list); @@ -2362,7 +2366,11 @@ _tkinter_tkapp_split(TkappObject *self, PyObject *arg) if (!PyArg_Parse(arg, "et:split", "utf-8", &list)) return NULL; - CHECK_STRING_LENGTH(list); + if (strlen(list) >= INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "string is too long"); + PyMem_Free(list); + return NULL; + } v = Split(list); PyMem_Free(list); return v; @@ -3304,7 +3312,7 @@ static PyMethodDef Tkapp_methods[] = _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF _TKINTER_TKAPP_RECORD_METHODDEF - _TKINTER_TKAPP_ADDERRINFO_METHODDEF + _TKINTER_TKAPP_ADDERRORINFO_METHODDEF {"setvar", Tkapp_SetVar, METH_VARARGS}, {"globalsetvar", Tkapp_GlobalSetVar, METH_VARARGS}, {"getvar", Tkapp_GetVar, METH_VARARGS}, @@ -3615,7 +3623,7 @@ PyInit__tkinter(void) return NULL; } if (str_path != NULL) { - wcs_path = PyUnicode_AsWideCharString(str_path, NULL); + wcs_path = _PyUnicode_AsWideCharString(str_path); if (wcs_path == NULL) { return NULL; } diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index ec8bd960a7f7e0..4f3bb5463f0386 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -77,7 +77,7 @@ typedef struct #ifdef __GNUC__ __attribute__((packed)) #elif defined(_MSC_VER) -_declspec(align(4)) +#pragma pack(push, 4) #endif { /* filename cannot be NULL: "" is used if the Python frame @@ -85,6 +85,9 @@ _declspec(align(4)) PyObject *filename; unsigned int lineno; } frame_t; +#ifdef _MSC_VER +#pragma pack(pop) +#endif typedef struct { @@ -1731,7 +1734,7 @@ _PyTraceMalloc_Init(void) if (key == NULL) return -1; - value = PyDict_GetItemWithError(xoptions, key); + value = PyDict_GetItemWithError(xoptions, key); /* borrowed */ Py_DECREF(key); if (value == NULL) { if (PyErr_Occurred()) @@ -1742,7 +1745,6 @@ _PyTraceMalloc_Init(void) } nframe = parse_sys_xoptions(value); - Py_DECREF(value); if (nframe < 0) { Py_FatalError("-X tracemalloc=NFRAME: invalid number of frames"); } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 91d4f0172c3c50..b98e7789b7876a 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -722,23 +722,46 @@ getenvironment(PyObject* environment) return NULL; } - envsize = PyMapping_Length(environment); - keys = PyMapping_Keys(environment); + if (!keys) { + return NULL; + } values = PyMapping_Values(environment); - if (!keys || !values) + if (!values) { + goto error; + } + + envsize = PySequence_Fast_GET_SIZE(keys); + if (PySequence_Fast_GET_SIZE(values) != envsize) { + PyErr_SetString(PyExc_RuntimeError, + "environment changed size during iteration"); goto error; + } totalsize = 1; /* trailing null character */ for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (! PyUnicode_Check(key) || ! PyUnicode_Check(value)) { PyErr_SetString(PyExc_TypeError, "environment can only contain strings"); goto error; } + if (PyUnicode_FindChar(key, '\0', 0, PyUnicode_GET_LENGTH(key), 1) != -1 || + PyUnicode_FindChar(value, '\0', 0, PyUnicode_GET_LENGTH(value), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto error; + } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key) == 0 || + PyUnicode_FindChar(key, '=', 1, PyUnicode_GET_LENGTH(key), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } if (totalsize > PY_SSIZE_T_MAX - PyUnicode_GET_LENGTH(key) - 1) { PyErr_SetString(PyExc_OverflowError, "environment too long"); goto error; @@ -760,8 +783,8 @@ getenvironment(PyObject* environment) end = buffer + totalsize; for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (!PyUnicode_AsUCS4(key, p, end - p, 0)) goto error; p += PyUnicode_GET_LENGTH(key); @@ -841,12 +864,13 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, if (env_mapping != Py_None) { environment = getenvironment(env_mapping); - if (! environment) + if (environment == NULL) { return NULL; + } + /* contains embedded null characters */ wenvironment = PyUnicode_AsUnicode(environment); - if (wenvironment == NULL) - { - Py_XDECREF(environment); + if (wenvironment == NULL) { + Py_DECREF(environment); return NULL; } } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee5a8e892..64e0f172fd7d82 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -2281,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2352,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 3cdf2d7e56348a..35ebf08ecd3c66 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -97,7 +97,7 @@ atexit_callfuncs(void) Py_XDECREF(exc_tb); } PyErr_Fetch(&exc_type, &exc_value, &exc_tb); - if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { + if (!PyErr_GivenExceptionMatches(exc_type, PyExc_SystemExit)) { PySys_WriteStderr("Error in atexit._run_exitfuncs:\n"); PyErr_NormalizeException(&exc_type, &exc_value, &exc_tb); PyErr_Display(exc_type, exc_value, exc_tb); diff --git a/Modules/audioop.c b/Modules/audioop.c index dcd7788d621862..d58204251913a8 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1293,7 +1293,7 @@ audioop_ratecv_impl(PyObject *module, Py_buffer *fragment, int width, char *cp, *ncp; Py_ssize_t len; int chan, d, *prev_i, *cur_i, cur_o; - PyObject *samps, *str, *rv = NULL; + PyObject *samps, *str, *rv = NULL, *channel; int bytes_per_frame; if (!audioop_check_size(width)) @@ -1354,6 +1354,10 @@ audioop_ratecv_impl(PyObject *module, Py_buffer *fragment, int width, prev_i[chan] = cur_i[chan] = 0; } else { + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "state must be a tuple or None"); + goto exit; + } if (!PyArg_ParseTuple(state, "iO!;audioop.ratecv: illegal state argument", &d, &PyTuple_Type, &samps)) @@ -1364,7 +1368,13 @@ audioop_ratecv_impl(PyObject *module, Py_buffer *fragment, int width, goto exit; } for (chan = 0; chan < nchannels; chan++) { - if (!PyArg_ParseTuple(PyTuple_GetItem(samps, chan), + channel = PyTuple_GetItem(samps, chan); + if (!PyTuple_Check(channel)) { + PyErr_SetString(PyExc_TypeError, + "ratecv(): illegal state argument"); + goto exit; + } + if (!PyArg_ParseTuple(channel, "ii:ratecv", &prev_i[chan], &cur_i[chan])) goto exit; diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index 1a070f2f393219..1fcc220b8db0f4 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -279,7 +279,9 @@ DECODER(gb18030) REQUIRE_INBUF(4); c3 = INBYTE3; c4 = INBYTE4; - if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) + if (c < 0x81 || c > 0xFE || + c3 < 0x81 || c3 > 0xFE || + c4 < 0x30 || c4 > 0x39) return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; @@ -348,15 +350,17 @@ ENCODER(hz) DBCHAR code; if (c < 0x80) { - if (state->i == 0) { - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); - } - else { - WRITEBYTE3('~', '}', (unsigned char)c); - NEXT(1, 3); + if (state->i) { + WRITEBYTE2('~', '}'); + NEXT_OUT(2); state->i = 0; } + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); + if (c == '~') { + WRITEBYTE1('~'); + NEXT_OUT(1); + } continue; } @@ -407,17 +411,14 @@ DECODER(hz) unsigned char c2 = INBYTE2; REQUIRE_INBUF(2); - if (c2 == '~') { + if (c2 == '~' && state->i == 0) OUTCHAR('~'); - NEXT_IN(2); - continue; - } else if (c2 == '{' && state->i == 0) state->i = 1; /* set GB */ + else if (c2 == '\n' && state->i == 0) + ; /* line-continuation */ else if (c2 == '}' && state->i == 1) state->i = 0; /* set ASCII */ - else if (c2 == '\n') - ; /* line-continuation */ else return 1; NEXT_IN(2); diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 1ce4218f3089d0..7394cf67e0e7dd 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -807,15 +807,9 @@ jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length, void *config) case 2: /* second character of unicode pair */ coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], jisx0213_pair_encmap, JISX0213_ENCPAIRS); - if (coded == DBCINV) { - *length = 1; - coded = find_pairencmap((ucs2_t)data[0], 0, - jisx0213_pair_encmap, JISX0213_ENCPAIRS); - if (coded == DBCINV) - return MAP_UNMAPPABLE; - } - else + if (coded != DBCINV) return coded; + /* fall through */ case -1: /* flush unterminated */ *length = 1; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index d1da189ddd3bef..d6efc77d20c883 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1670,6 +1670,9 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se if (r == -1) return NULL; } + /* PySequence_Length() can fail */ + if (PyErr_Occurred()) + return NULL; Py_RETURN_NONE; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 052d252331f6a7..62bf8e7a92df3a 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -28,7 +28,7 @@ _asyncio_Future___init__(PyObject *self, PyObject *args, PyObject *kwargs) int return_value = -1; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|$O:Future", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, &loop)) { @@ -244,7 +244,7 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) static const char * const _keywords[] = {"coro", "loop", NULL}; static _PyArg_Parser _parser = {"O|$O:Task", _keywords, 0}; PyObject *coro; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, &coro, &loop)) { @@ -278,7 +278,7 @@ _asyncio_Task_current_task(PyTypeObject *type, PyObject **args, Py_ssize_t nargs PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:current_task", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -310,7 +310,7 @@ _asyncio_Task_all_tasks(PyTypeObject *type, PyObject **args, Py_ssize_t nargs, P PyObject *return_value = NULL; static const char * const _keywords[] = {"loop", NULL}; static _PyArg_Parser _parser = {"|O:all_tasks", _keywords, 0}; - PyObject *loop = NULL; + PyObject *loop = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &loop)) { @@ -477,7 +477,7 @@ _asyncio_Task__step(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject * PyObject *return_value = NULL; static const char * const _keywords[] = {"exc", NULL}; static _PyArg_Parser _parser = {"|O:_step", _keywords, 0}; - PyObject *exc = NULL; + PyObject *exc = Py_None; if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, &exc)) { @@ -517,4 +517,4 @@ _asyncio_Task__wakeup(TaskObj *self, PyObject **args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=8f036321bb083066 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7441872b13652085 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 29f58384394c98..6f748903f4eb52 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -132,7 +132,7 @@ _ssl__SSLSocket_version(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) return _ssl__SSLSocket_version_impl(self); } -#if defined(OPENSSL_NPN_NEGOTIATED) +#if (defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG)) PyDoc_STRVAR(_ssl__SSLSocket_selected_npn_protocol__doc__, "selected_npn_protocol($self, /)\n" @@ -151,7 +151,7 @@ _ssl__SSLSocket_selected_npn_protocol(PySSLSocket *self, PyObject *Py_UNUSED(ign return _ssl__SSLSocket_selected_npn_protocol_impl(self); } -#endif /* defined(OPENSSL_NPN_NEGOTIATED) */ +#endif /* (defined(OPENSSL_NPN_NEGOTIATED) && !defined(OPENSSL_NO_NEXTPROTONEG)) */ #if defined(HAVE_ALPN) @@ -839,7 +839,7 @@ PyDoc_STRVAR(_ssl_RAND_add__doc__, "Mix string into the OpenSSL PRNG state.\n" "\n" "entropy (a float) is a lower bound on the entropy contained in\n" -"string. See RFC 1750."); +"string. See RFC 4086."); #define _SSL_RAND_ADD_METHODDEF \ {"RAND_add", (PyCFunction)_ssl_RAND_add, METH_VARARGS, _ssl_RAND_add__doc__}, @@ -1168,4 +1168,4 @@ _ssl_enum_crls(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kw #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=a859b21fe68a6115 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a8b184655068c238 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index edd53800058dab..c407f17da4cb84 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -80,27 +80,27 @@ _tkinter_tkapp_record(TkappObject *self, PyObject *arg) return return_value; } -PyDoc_STRVAR(_tkinter_tkapp_adderrinfo__doc__, -"adderrinfo($self, msg, /)\n" +PyDoc_STRVAR(_tkinter_tkapp_adderrorinfo__doc__, +"adderrorinfo($self, msg, /)\n" "--\n" "\n"); -#define _TKINTER_TKAPP_ADDERRINFO_METHODDEF \ - {"adderrinfo", (PyCFunction)_tkinter_tkapp_adderrinfo, METH_O, _tkinter_tkapp_adderrinfo__doc__}, +#define _TKINTER_TKAPP_ADDERRORINFO_METHODDEF \ + {"adderrorinfo", (PyCFunction)_tkinter_tkapp_adderrorinfo, METH_O, _tkinter_tkapp_adderrorinfo__doc__}, static PyObject * -_tkinter_tkapp_adderrinfo_impl(TkappObject *self, const char *msg); +_tkinter_tkapp_adderrorinfo_impl(TkappObject *self, const char *msg); static PyObject * -_tkinter_tkapp_adderrinfo(TkappObject *self, PyObject *arg) +_tkinter_tkapp_adderrorinfo(TkappObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *msg; - if (!PyArg_Parse(arg, "s:adderrinfo", &msg)) { + if (!PyArg_Parse(arg, "s:adderrorinfo", &msg)) { goto exit; } - return_value = _tkinter_tkapp_adderrinfo_impl(self, msg); + return_value = _tkinter_tkapp_adderrorinfo_impl(self, msg); exit: return return_value; @@ -638,4 +638,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=836c578b71d69097 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b0be55aacff2be9b input=a9049054013a1b77]*/ diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index dcb4506429684b..8d288f0f28fddd 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,6 +1,5 @@ -Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers. +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2017 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/amigaconfig.h b/Modules/expat/amigaconfig.h deleted file mode 100644 index 86c611504025fb..00000000000000 --- a/Modules/expat/amigaconfig.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef AMIGACONFIG_H -#define AMIGACONFIG_H - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#define HAVE_BCOPY 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_CHECK_H - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -#endif /* AMIGACONFIG_H */ diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index d10530b09bde69..c3587e57332bff 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #define ASCII_A 0x41 diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index 79a15c28ca14f8..2f59fd92906537 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML, diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index e8eefddc6d8474..d0735bb5c61676 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef Expat_INCLUDED @@ -24,7 +52,6 @@ extern "C" { struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; -/* Should this be defined using stdbool.h when C99 is available? */ typedef unsigned char XML_Bool; #define XML_TRUE ((XML_Bool) 1) #define XML_FALSE ((XML_Bool) 0) @@ -95,7 +122,9 @@ enum XML_Error { /* Added in 2.0. */ XML_ERROR_RESERVED_PREFIX_XML, XML_ERROR_RESERVED_PREFIX_XMLNS, - XML_ERROR_RESERVED_NAMESPACE_URI + XML_ERROR_RESERVED_NAMESPACE_URI, + /* Added in 2.2.1. */ + XML_ERROR_INVALID_ARGUMENT }; enum XML_Content_Type { @@ -342,7 +371,7 @@ XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler); /* OBSOLETE -- OBSOLETE -- OBSOLETE - This handler has been superceded by the EntityDeclHandler above. + This handler has been superseded by the EntityDeclHandler above. It is provided here for backward compatibility. This is called for a declaration of an unparsed (NDATA) entity. @@ -706,6 +735,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser); be called, despite an external subset being parsed. Note: If XML_DTD is not defined when Expat is compiled, returns XML_ERROR_FEATURE_REQUIRES_XML_DTD. + Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT. */ XMLPARSEAPI(enum XML_Error) XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD); @@ -729,15 +759,16 @@ XML_GetBase(XML_Parser parser); to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus this correspondds to an index into the atts array passed to the - XML_StartElementHandler. + XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to - XML_StartElementHandler, or -1 if there is no ID attribute. Each - attribute/value pair counts as 2; thus this correspondds to an - index into the atts array passed to the XML_StartElementHandler. + XML_StartElementHandler, or -1 if there is no ID attribute or + parser == NULL. Each attribute/value pair counts as 2; thus this + correspondds to an index into the atts array passed to the + XML_StartElementHandler. */ XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); @@ -901,6 +932,7 @@ enum XML_ParamEntityParsing { entities is requested; otherwise it will return non-zero. Note: If XML_SetParamEntityParsing is called after XML_Parse or XML_ParseBuffer, then it has no effect and will always return 0. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetParamEntityParsing(XML_Parser parser, @@ -910,13 +942,12 @@ XML_SetParamEntityParsing(XML_Parser parser, Helps in preventing DoS attacks based on predicting hash function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); -#define XML_HAS_SET_HASH_SALT /* Python Only: Defined for pyexpat.c. */ - /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -938,6 +969,10 @@ XML_GetErrorCode(XML_Parser parser); the location is the location of the character at which the error was detected; otherwise the location is the location of the last parse event, as described above. + + Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber + return 0 to indicate an error. + Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); @@ -975,9 +1010,12 @@ XML_FreeContentModel(XML_Parser parser, XML_Content *model); /* Exposing the memory handling functions used in Expat */ XMLPARSEAPI(void *) +XML_ATTR_MALLOC +XML_ATTR_ALLOC_SIZE(2) XML_MemMalloc(XML_Parser parser, size_t size); XMLPARSEAPI(void *) +XML_ATTR_ALLOC_SIZE(3) XML_MemRealloc(XML_Parser parser, void *ptr, size_t size); XMLPARSEAPI(void) @@ -1033,14 +1071,12 @@ XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -/* Expat follows the GNU/Linux convention of odd number minor version for - beta/development releases and even number minor version for stable - releases. Micro is bumped with each release, and set to 0 with each - change to major or minor version. +/* Expat follows the semantic versioning convention. + See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 1 -#define XML_MICRO_VERSION 1 +#define XML_MINOR_VERSION 2 +#define XML_MICRO_VERSION 4 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f337e1c5622a95..81102856496965 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef Expat_External_INCLUDED @@ -69,12 +97,26 @@ #endif #endif /* not defined XML_STATIC */ +#if !defined(XMLIMPORT) && defined(__GNUC__) && (__GNUC__ >= 4) +#define XMLIMPORT __attribute__ ((visibility ("default"))) +#endif /* If we didn't define it above, define it away: */ #ifndef XMLIMPORT #define XMLIMPORT #endif +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +#define XML_ATTR_MALLOC __attribute__((__malloc__)) +#else +#define XML_ATTR_MALLOC +#endif + +#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +#else +#define XML_ATTR_ALLOC_SIZE(x) +#endif #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL @@ -83,7 +125,10 @@ extern "C" { #endif #ifdef XML_UNICODE_WCHAR_T -#define XML_UNICODE +# define XML_UNICODE +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif #endif #ifdef XML_UNICODE /* Information is UTF-16 encoded. */ diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index 24a1d5ccc9a59b..ce4a4bf7edef53 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */ diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index dd5454831da2b4..3c5d6e913d6db3 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -18,6 +18,35 @@ Note: Use of these macros is based on judgement, not hard rules, and therefore subject to change. + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #if defined(__GNUC__) && defined(__i386__) && !defined(__MINGW32__) @@ -71,3 +100,25 @@ #define inline #endif #endif + +#ifndef UNUSED_P +# ifdef __GNUC__ +# define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__)) +# else +# define UNUSED_P(p) UNUSED_ ## p +# endif +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + + +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef); + + +#ifdef __cplusplus +} +#endif diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index 53c25d76b26813..95dfa52b1fbebb 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER, diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c new file mode 100644 index 00000000000000..452ae92db2673c --- /dev/null +++ b/Modules/expat/loadlibrary.c @@ -0,0 +1,143 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2016 - 2017, Steve Holme, . + * Copyright (C) 2017, Expat development team + * + * All rights reserved. + * Licensed under the MIT license: + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH + * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + * + ***************************************************************************/ + +#if defined(_WIN32) + +#include +#include + + +HMODULE _Expat_LoadLibrary(LPCTSTR filename); + + +#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH) +#define LOAD_WITH_ALTERED_SEARCH_PATH 0x00000008 +#endif + +#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +#define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +/* We use our own typedef here since some headers might lack these */ +typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD); + +/* See function definitions in winbase.h */ +#ifdef UNICODE +# ifdef _WIN32_WCE +# define LOADLIBARYEX L"LoadLibraryExW" +# else +# define LOADLIBARYEX "LoadLibraryExW" +# endif +#else +# define LOADLIBARYEX "LoadLibraryExA" +#endif + + +/* + * _Expat_LoadLibrary() + * + * This is used to dynamically load DLLs using the most secure method available + * for the version of Windows that we are running on. + * + * Parameters: + * + * filename [in] - The filename or full path of the DLL to load. If only the + * filename is passed then the DLL will be loaded from the + * Windows system directory. + * + * Returns the handle of the module on success; otherwise NULL. + */ +HMODULE _Expat_LoadLibrary(LPCTSTR filename) +{ + HMODULE hModule = NULL; + LOADLIBRARYEX_FN pLoadLibraryEx = NULL; + + /* Get a handle to kernel32 so we can access it's functions at runtime */ + HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32")); + if(!hKernel32) + return NULL; + + /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 + and above */ + pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + + /* Detect if there's already a path in the filename and load the library if + there is. Note: Both back slashes and forward slashes have been supported + since the earlier days of DOS at an API level although they are not + supported by command prompt */ + if(_tcspbrk(filename, TEXT("\\/"))) { + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(filename); + } + /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only + supported on Windows Vista, Windows Server 2008, Windows 7 and Windows + Server 2008 R2 with this patch or natively on Windows 8 and above */ + else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) { + /* Load the DLL from the Windows system directory */ + hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + } + else { + /* Attempt to get the Windows system path */ + UINT systemdirlen = GetSystemDirectory(NULL, 0); + if(systemdirlen) { + /* Allocate space for the full DLL path (Room for the null terminator + is included in systemdirlen) */ + size_t filenamelen = _tcslen(filename); + TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen)); + if(path && GetSystemDirectory(path, systemdirlen)) { + /* Calculate the full DLL path */ + _tcscpy(path + _tcslen(path), TEXT("\\")); + _tcscpy(path + _tcslen(path), filename); + + /* Load the DLL from the Windows system directory */ + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(path); + + } + free(path); + } + } + + return hModule; +} + +#else /* defined(_WIN32) */ + +/* ISO C requires a translation unit to contain at least one declaration + [-Wempty-translation-unit] */ +typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY; + +#endif /* defined(_WIN32) */ diff --git a/Modules/expat/macconfig.h b/Modules/expat/macconfig.h deleted file mode 100644 index 2725caaf54a2e2..00000000000000 --- a/Modules/expat/macconfig.h +++ /dev/null @@ -1,53 +0,0 @@ -/*================================================================ -** Copyright 2000, Clark Cooper -** All rights reserved. -** -** This is free software. You are permitted to copy, distribute, or modify -** it under the terms of the MIT/X license (contained in the COPYING file -** with this distribution.) -** -*/ - -#ifndef MACCONFIG_H -#define MACCONFIG_H - - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#undef HAVE_BCOPY - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE - -/* Define to 1 if you have a working `mmap' system call. */ -#undef HAVE_MMAP - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#undef XML_CONTEXT_BYTES - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -/* Define to empty if `const' does not conform to ANSI C. */ -#undef const - -/* Define to `long' if does not define. */ -#define off_t long - -/* Define to `unsigned' if does not define. */ -#undef size_t - - -#endif /* ifndef MACCONFIG_H */ diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index b05e62c77a6c62..bfa2bd38cd95cd 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -1,3 +1,35 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + static const unsigned namingBitmap[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h new file mode 100644 index 00000000000000..581872df7b4992 --- /dev/null +++ b/Modules/expat/siphash.h @@ -0,0 +1,374 @@ +/* ========================================================================== + * siphash.h - SipHash-2-4 in a single header file + * -------------------------------------------------------------------------- + * Derived by William Ahern from the reference implementation[1] published[2] + * by Jean-Philippe Aumasson and Daniel J. Berstein. + * Minimal changes by Sebastian Pipping and Victor Stinner on top, see below. + * Licensed under the CC0 Public Domain Dedication license. + * + * 1. https://www.131002.net/siphash/siphash24.c + * 2. https://www.131002.net/siphash/ + * -------------------------------------------------------------------------- + * HISTORY: + * + * 2017-07-25 (Vadim Zeitlin) + * - Fix use of SIPHASH_MAIN macro + * + * 2017-07-05 (Sebastian Pipping) + * - Use _SIP_ULL macro to not require a C++11 compiler if compiled as C++ + * - Add const qualifiers at two places + * - Ensure <=80 characters line length (assuming tab width 4) + * + * 2017-06-23 (Victor Stinner) + * - Address Win64 compile warnings + * + * 2017-06-18 (Sebastian Pipping) + * - Clarify license note in the header + * - Address C89 issues: + * - Stop using inline keyword (and let compiler decide) + * - Replace _Bool by int + * - Turn macro siphash24 into a function + * - Address invalid conversion (void pointer) by explicit cast + * - Address lack of stdint.h for Visual Studio 2003 to 2008 + * - Always expose sip24_valid (for self-tests) + * + * 2012-11-04 - Born. (William Ahern) + * -------------------------------------------------------------------------- + * USAGE: + * + * SipHash-2-4 takes as input two 64-bit words as the key, some number of + * message bytes, and outputs a 64-bit word as the message digest. This + * implementation employs two data structures: a struct sipkey for + * representing the key, and a struct siphash for representing the hash + * state. + * + * For converting a 16-byte unsigned char array to a key, use either the + * macro sip_keyof or the routine sip_tokey. The former instantiates a + * compound literal key, while the latter requires a key object as a + * parameter. + * + * unsigned char secret[16]; + * arc4random_buf(secret, sizeof secret); + * struct sipkey *key = sip_keyof(secret); + * + * For hashing a message, use either the convenience macro siphash24 or the + * routines sip24_init, sip24_update, and sip24_final. + * + * struct siphash state; + * void *msg; + * size_t len; + * uint64_t hash; + * + * sip24_init(&state, key); + * sip24_update(&state, msg, len); + * hash = sip24_final(&state); + * + * or + * + * hash = siphash24(msg, len, key); + * + * To convert the 64-bit hash value to a canonical 8-byte little-endian + * binary representation, use either the macro sip_binof or the routine + * sip_tobin. The former instantiates and returns a compound literal array, + * while the latter requires an array object as a parameter. + * -------------------------------------------------------------------------- + * NOTES: + * + * o Neither sip_keyof, sip_binof, nor siphash24 will work with compilers + * lacking compound literal support. Instead, you must use the lower-level + * interfaces which take as parameters the temporary state objects. + * + * o Uppercase macros may evaluate parameters more than once. Lowercase + * macros should not exhibit any such side effects. + * ========================================================================== + */ +#ifndef SIPHASH_H +#define SIPHASH_H + +#include /* size_t */ + +#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) + /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; +#else + #include /* uint64_t uint32_t uint8_t */ +#endif + + +/* + * Workaround to not require a C++11 compiler for using ULL suffix + * if this code is included and compiled as C++; related GCC warning is: + * warning: use of C++11 long long integer constant [-Wlong-long] + */ +#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low) + + +#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) + +#define SIP_U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v) >> 0); (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24); + +#define SIP_U64TO8_LE(p, v) \ + SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0)); \ + SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define SIP_U8TO64_LE(p) \ + (((uint64_t)((p)[0]) << 0) | \ + ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | \ + ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | \ + ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | \ + ((uint64_t)((p)[7]) << 56)) + + +#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 } + +struct siphash { + uint64_t v0, v1, v2, v3; + + unsigned char buf[8], *p; + uint64_t c; +}; /* struct siphash */ + + +#define SIP_KEYLEN 16 + +struct sipkey { + uint64_t k[2]; +}; /* struct sipkey */ + +#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k)) + +static struct sipkey *sip_tokey(struct sipkey *key, const void *src) { + key->k[0] = SIP_U8TO64_LE((const unsigned char *)src); + key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8); + return key; +} /* sip_tokey() */ + + +#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v)) + +static void *sip_tobin(void *dst, uint64_t u64) { + SIP_U64TO8_LE((unsigned char *)dst, u64); + return dst; +} /* sip_tobin() */ + + +static void sip_round(struct siphash *H, const int rounds) { + int i; + + for (i = 0; i < rounds; i++) { + H->v0 += H->v1; + H->v1 = SIP_ROTL(H->v1, 13); + H->v1 ^= H->v0; + H->v0 = SIP_ROTL(H->v0, 32); + + H->v2 += H->v3; + H->v3 = SIP_ROTL(H->v3, 16); + H->v3 ^= H->v2; + + H->v0 += H->v3; + H->v3 = SIP_ROTL(H->v3, 21); + H->v3 ^= H->v0; + + H->v2 += H->v1; + H->v1 = SIP_ROTL(H->v1, 17); + H->v1 ^= H->v2; + H->v2 = SIP_ROTL(H->v2, 32); + } +} /* sip_round() */ + + +static struct siphash *sip24_init(struct siphash *H, + const struct sipkey *key) { + H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; + + H->p = H->buf; + H->c = 0; + + return H; +} /* sip24_init() */ + + +#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) + +static struct siphash *sip24_update(struct siphash *H, const void *src, + size_t len) { + const unsigned char *p = (const unsigned char *)src, *pe = p + len; + uint64_t m; + + do { + while (p < pe && H->p < sip_endof(H->buf)) + *H->p++ = *p++; + + if (H->p < sip_endof(H->buf)) + break; + + m = SIP_U8TO64_LE(H->buf); + H->v3 ^= m; + sip_round(H, 2); + H->v0 ^= m; + + H->p = H->buf; + H->c += 8; + } while (p < pe); + + return H; +} /* sip24_update() */ + + +static uint64_t sip24_final(struct siphash *H) { + const char left = (char)(H->p - H->buf); + uint64_t b = (H->c + left) << 56; + + switch (left) { + case 7: b |= (uint64_t)H->buf[6] << 48; + case 6: b |= (uint64_t)H->buf[5] << 40; + case 5: b |= (uint64_t)H->buf[4] << 32; + case 4: b |= (uint64_t)H->buf[3] << 24; + case 3: b |= (uint64_t)H->buf[2] << 16; + case 2: b |= (uint64_t)H->buf[1] << 8; + case 1: b |= (uint64_t)H->buf[0] << 0; + case 0: break; + } + + H->v3 ^= b; + sip_round(H, 2); + H->v0 ^= b; + H->v2 ^= 0xff; + sip_round(H, 4); + + return H->v0 ^ H->v1 ^ H->v2 ^ H->v3; +} /* sip24_final() */ + + +static uint64_t siphash24(const void *src, size_t len, + const struct sipkey *key) { + struct siphash state = SIPHASH_INITIALIZER; + return sip24_final(sip24_update(sip24_init(&state, key), src, len)); +} /* siphash24() */ + + +/* + * SipHash-2-4 output with + * k = 00 01 02 ... + * and + * in = (empty string) + * in = 00 (1 byte) + * in = 00 01 (2 bytes) + * in = 00 01 02 (3 bytes) + * ... + * in = 00 01 02 ... 3e (63 bytes) + */ +static int sip24_valid(void) { + static const unsigned char vectors[64][8] = { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + unsigned char in[64]; + struct sipkey k; + size_t i; + + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011" + "\012\013\014\015\016\017"); + + for (i = 0; i < sizeof in; ++i) { + in[i] = (unsigned char)i; + + if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) + return 0; + } + + return 1; +} /* sip24_valid() */ + + +#ifdef SIPHASH_MAIN + +#include + +int main(void) { + const int ok = sip24_valid(); + + if (ok) + puts("OK"); + else + puts("FAIL"); + + return !ok; +} /* main() */ + +#endif /* SIPHASH_MAIN */ + + +#endif /* SIPHASH_H */ diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index 7bb3e77603fa75..fa0bed6f5d751d 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -1,7 +1,34 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. -*/ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, /* 0x84 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, diff --git a/Modules/expat/watcomconfig.h b/Modules/expat/watcomconfig.h deleted file mode 100644 index 2f05e3f227c162..00000000000000 --- a/Modules/expat/watcomconfig.h +++ /dev/null @@ -1,47 +0,0 @@ -/* expat_config.h for use with Open Watcom 1.5 and above. */ - -#ifndef WATCOMCONFIG_H -#define WATCOMCONFIG_H - -#ifdef __NT__ -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN -#endif - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 1234 - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "expat-bugs@mail.libexpat.org" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "expat" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "expat 2.0.0" - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.0.0" - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD 1 - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS 1 - -#endif - diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index c1b791d62d0d0c..17fea4689001f4 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -1,10 +1,33 @@ -/*================================================================ -** Copyright 2000, Clark Cooper -** All rights reserved. -** -** This is free software. You are permitted to copy, distribute, or modify -** it under the terms of the MIT/X license (contained in the COPYING file -** with this distribution.) +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef WINCONFIG_H @@ -17,6 +40,12 @@ #include #include + +#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ +# include +#else /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #define XML_NS 1 #define XML_DTD 1 #define XML_CONTEXT_BYTES 1024 @@ -27,4 +56,8 @@ /* Windows has memmove() available. */ #define HAVE_MEMMOVE + +#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 412838794d703b..0df68830f05e3a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,29 +1,115 @@ -/* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* 8c6b2be7c6281da65ce05218fc15c339f02a811706340824ab596aa86e1fd51a (2.2.4+) + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#define XML_BUILDING_EXPAT 1 - -#ifdef COMPILED_FROM_DSP -#include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" -#elif defined(HAVE_EXPAT_CONFIG_H) -#include -#endif /* ndef COMPILED_FROM_DSP */ +#if !defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +#endif #include #include /* memset(), memcpy() */ #include #include /* UINT_MAX */ -#include /* time() */ +#include /* fprintf */ +#include /* getenv */ + +#ifdef _WIN32 +#define getpid GetCurrentProcessId +#else +#include /* gettimeofday() */ +#include /* getpid() */ +#include /* getpid() */ +#include /* O_RDONLY */ +#include +#endif + +#define XML_BUILDING_EXPAT 1 + +#ifdef _WIN32 +#include "winconfig.h" +#elif defined(HAVE_EXPAT_CONFIG_H) +#include +#endif /* ndef _WIN32 */ #include "ascii.h" #include "expat.h" +#include "siphash.h" + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# if defined(HAVE_GETRANDOM) +# include /* getrandom */ +# else +# include /* syscall */ +# include /* SYS_getrandom */ +# endif +# if ! defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 0x0001 +# endif /* defined(GRND_NONBLOCK) */ +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + +#if defined(HAVE_LIBBSD) \ + && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) +# include +#endif + +#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ + && !defined(_WIN32) \ + && !defined(XML_POOR_ENTROPY) +# error \ + You do not have support for any sources of high quality entropy \ + enabled. For end user security, that is probably not what you want. \ + \ + Your options include: \ + * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ + * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ + * Windows (RtlGenRandom): _WIN32. \ + \ + If insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY; you have been warned. \ + \ + If you have reasons to patch this detection code away or need changes \ + to the build system, please open a bug. Thank you! +#endif + #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX @@ -102,17 +188,11 @@ typedef struct { const XML_Memory_Handling_Suite *mem; } HASH_TABLE; -/* Basic character hash algorithm, taken from Python's string hash: - h = h * 1000003 ^ character, the constant being a prime number. +static size_t +keylen(KEY s); -*/ -#ifdef XML_UNICODE -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned short)(c)) -#else -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned char)(c)) -#endif +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key); /* For probing (after a collision) we need a step size relative prime to the hash table size, which is a power of 2. We use double-hashing, @@ -348,6 +428,8 @@ doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ +static void +freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, BINDING **bindingsPtr); @@ -432,7 +514,10 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static unsigned long generate_hash_secret_salt(void); +static XML_Char *copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite); + +static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); static XML_Parser @@ -690,12 +775,215 @@ static const XML_Char implicitContext[] = { ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0' }; + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + +/* Obtain entropy on Linux 3.17+ */ +static int +writeRandomBytes_getrandom_nonblock(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + const unsigned int getrandomFlags = GRND_NONBLOCK; + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const int bytesWrittenMore = +#if defined(HAVE_GETRANDOM) + getrandom(currentTarget, bytesToWrite, getrandomFlags); +#else + syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); +#endif + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR)); + + return success; +} + +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + + +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + +/* Extract entropy from /dev/urandom */ +static int +writeRandomBytes_dev_urandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + + const int fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + return 0; + } + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR)); + + close(fd); + return success; +} + +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ + + +#if defined(HAVE_ARC4RANDOM) + +static void +writeRandomBytes_arc4random(void * target, size_t count) { + size_t bytesWrittenTotal = 0; + + while (bytesWrittenTotal < count) { + const uint32_t random32 = arc4random(); + size_t i = 0; + + for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); + i++, bytesWrittenTotal++) { + const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); + ((uint8_t *)target)[bytesWrittenTotal] = random8; + } + } +} + +#endif /* defined(HAVE_ARC4RANDOM) */ + + +#ifdef _WIN32 + +typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); +HMODULE _Expat_LoadLibrary(LPCTSTR filename); /* see loadlibrary.c */ + +/* Obtain entropy on Windows XP / Windows Server 2003 and later. + * Hint on RtlGenRandom and the following article from libsodium. + * + * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI + * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/ + */ +static int +writeRandomBytes_RtlGenRandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL")); + + if (advapi32) { + const RTLGENRANDOM_FUNC RtlGenRandom + = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036"); + if (RtlGenRandom) { + if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) { + success = 1; + } + } + FreeLibrary(advapi32); + } + + return success; +} + +#endif /* _WIN32 */ + + +#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + static unsigned long -generate_hash_secret_salt(void) +gather_time_entropy(void) { - unsigned int seed = time(NULL) % UINT_MAX; - srand(seed); - return rand(); +#ifdef _WIN32 + FILETIME ft; + GetSystemTimeAsFileTime(&ft); /* never fails */ + return ft.dwHighDateTime ^ ft.dwLowDateTime; +#else + struct timeval tv; + int gettimeofday_res; + + gettimeofday_res = gettimeofday(&tv, NULL); + +#if defined(NDEBUG) + (void)gettimeofday_res; +#else + assert (gettimeofday_res == 0); +#endif /* defined(NDEBUG) */ + + /* Microseconds time is <20 bits entropy */ + return tv.tv_usec; +#endif +} + +#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + + +static unsigned long +ENTROPY_DEBUG(const char * label, unsigned long entropy) { + const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); + if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { + fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", + label, + (int)sizeof(entropy) * 2, entropy, + (unsigned long)sizeof(entropy)); + } + return entropy; +} + +static unsigned long +generate_hash_secret_salt(XML_Parser parser) +{ + unsigned long entropy; + (void)parser; +#if defined(HAVE_ARC4RANDOM_BUF) + arc4random_buf(&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random_buf", entropy); +#elif defined(HAVE_ARC4RANDOM) + writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random", entropy); +#else + /* Try high quality providers first .. */ +#ifdef _WIN32 + if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("RtlGenRandom", entropy); + } +#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getrandom", entropy); + } +#endif +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("/dev/urandom", entropy); + } +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ + /* .. and self-made low quality for backup: */ + + /* Process ID is 0 bits entropy if attacker has local access */ + entropy = gather_time_entropy() ^ getpid(); + + /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ + if (sizeof(unsigned long) == 4) { + return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + } else { + return ENTROPY_DEBUG("fallback(8)", + entropy * (unsigned long)2305843009213693951ULL); + } +#endif +} + +static unsigned long +get_hash_secret_salt(XML_Parser parser) { + if (parser->m_parentParser != NULL) + return get_hash_secret_salt(parser->m_parentParser); + return parser->m_hash_secret_salt; } static XML_Bool /* only valid for root parser */ @@ -703,7 +991,7 @@ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ if (hash_secret_salt == 0) - hash_secret_salt = generate_hash_secret_salt(); + hash_secret_salt = generate_hash_secret_salt(parser); if (ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -815,6 +1103,8 @@ parserCreate(const XML_Char *encodingName, nsAttsVersion = 0; nsAttsPower = 0; + protocolEncodingName = NULL; + poolInit(&tempPool, &(parser->m_mem)); poolInit(&temp2Pool, &(parser->m_mem)); parserInit(parser, encodingName); @@ -841,9 +1131,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { processor = prologInitProcessor; XmlPrologStateInit(&prologState); - protocolEncodingName = (encodingName != NULL - ? poolCopyString(&tempPool, encodingName) - : NULL); + if (encodingName != NULL) { + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + } curBase = NULL; XmlInitEncoding(&initEncoding, &encoding, 0); userData = NULL; @@ -926,6 +1216,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; OPEN_INTERNAL_ENTITY *openEntityList; + + if (parser == NULL) + return XML_FALSE; + if (parentParser) return XML_FALSE; /* move tagStack to freeTagList */ @@ -952,6 +1246,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) unknownEncodingRelease(unknownEncodingData); poolClear(&tempPool); poolClear(&temp2Pool); + FREE((void *)protocolEncodingName); + protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(_dtd, &parser->m_mem); return XML_TRUE; @@ -960,16 +1256,24 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { + if (parser == NULL) + return XML_STATUS_ERROR; /* Block after XML_Parse()/XML_ParseBuffer() has been called. XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return XML_STATUS_ERROR; + + /* Get rid of any previous encoding name */ + FREE((void *)protocolEncodingName); + if (encodingName == NULL) + /* No new encoding name */ protocolEncodingName = NULL; else { - protocolEncodingName = poolCopyString(&tempPool, encodingName); + /* Copy the new encoding name into allocated memory */ + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); if (!protocolEncodingName) return XML_STATUS_ERROR; } @@ -983,52 +1287,88 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, { XML_Parser parser = oldParser; DTD *newDtd = NULL; - DTD *oldDtd = _dtd; - XML_StartElementHandler oldStartElementHandler = startElementHandler; - XML_EndElementHandler oldEndElementHandler = endElementHandler; - XML_CharacterDataHandler oldCharacterDataHandler = characterDataHandler; - XML_ProcessingInstructionHandler oldProcessingInstructionHandler - = processingInstructionHandler; - XML_CommentHandler oldCommentHandler = commentHandler; - XML_StartCdataSectionHandler oldStartCdataSectionHandler - = startCdataSectionHandler; - XML_EndCdataSectionHandler oldEndCdataSectionHandler - = endCdataSectionHandler; - XML_DefaultHandler oldDefaultHandler = defaultHandler; - XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler - = unparsedEntityDeclHandler; - XML_NotationDeclHandler oldNotationDeclHandler = notationDeclHandler; - XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler - = startNamespaceDeclHandler; - XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler - = endNamespaceDeclHandler; - XML_NotStandaloneHandler oldNotStandaloneHandler = notStandaloneHandler; - XML_ExternalEntityRefHandler oldExternalEntityRefHandler - = externalEntityRefHandler; - XML_SkippedEntityHandler oldSkippedEntityHandler = skippedEntityHandler; - XML_UnknownEncodingHandler oldUnknownEncodingHandler - = unknownEncodingHandler; - XML_ElementDeclHandler oldElementDeclHandler = elementDeclHandler; - XML_AttlistDeclHandler oldAttlistDeclHandler = attlistDeclHandler; - XML_EntityDeclHandler oldEntityDeclHandler = entityDeclHandler; - XML_XmlDeclHandler oldXmlDeclHandler = xmlDeclHandler; - ELEMENT_TYPE * oldDeclElementType = declElementType; - - void *oldUserData = userData; - void *oldHandlerArg = handlerArg; - XML_Bool oldDefaultExpandInternalEntities = defaultExpandInternalEntities; - XML_Parser oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; + DTD *oldDtd; + XML_StartElementHandler oldStartElementHandler; + XML_EndElementHandler oldEndElementHandler; + XML_CharacterDataHandler oldCharacterDataHandler; + XML_ProcessingInstructionHandler oldProcessingInstructionHandler; + XML_CommentHandler oldCommentHandler; + XML_StartCdataSectionHandler oldStartCdataSectionHandler; + XML_EndCdataSectionHandler oldEndCdataSectionHandler; + XML_DefaultHandler oldDefaultHandler; + XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler; + XML_NotationDeclHandler oldNotationDeclHandler; + XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler; + XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler; + XML_NotStandaloneHandler oldNotStandaloneHandler; + XML_ExternalEntityRefHandler oldExternalEntityRefHandler; + XML_SkippedEntityHandler oldSkippedEntityHandler; + XML_UnknownEncodingHandler oldUnknownEncodingHandler; + XML_ElementDeclHandler oldElementDeclHandler; + XML_AttlistDeclHandler oldAttlistDeclHandler; + XML_EntityDeclHandler oldEntityDeclHandler; + XML_XmlDeclHandler oldXmlDeclHandler; + ELEMENT_TYPE * oldDeclElementType; + + void *oldUserData; + void *oldHandlerArg; + XML_Bool oldDefaultExpandInternalEntities; + XML_Parser oldExternalEntityRefHandlerArg; #ifdef XML_DTD - enum XML_ParamEntityParsing oldParamEntityParsing = paramEntityParsing; - int oldInEntityValue = prologState.inEntityValue; + enum XML_ParamEntityParsing oldParamEntityParsing; + int oldInEntityValue; +#endif + XML_Bool oldns_triplets; + /* Note that the new parser shares the same hash secret as the old + parser, so that dtdCopy and copyEntityTable can lookup values + from hash tables associated with either parser without us having + to worry which hash secrets each table has. + */ + unsigned long oldhash_secret_salt; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) + return NULL; + + /* Stash the original parser contents on the stack */ + oldDtd = _dtd; + oldStartElementHandler = startElementHandler; + oldEndElementHandler = endElementHandler; + oldCharacterDataHandler = characterDataHandler; + oldProcessingInstructionHandler = processingInstructionHandler; + oldCommentHandler = commentHandler; + oldStartCdataSectionHandler = startCdataSectionHandler; + oldEndCdataSectionHandler = endCdataSectionHandler; + oldDefaultHandler = defaultHandler; + oldUnparsedEntityDeclHandler = unparsedEntityDeclHandler; + oldNotationDeclHandler = notationDeclHandler; + oldStartNamespaceDeclHandler = startNamespaceDeclHandler; + oldEndNamespaceDeclHandler = endNamespaceDeclHandler; + oldNotStandaloneHandler = notStandaloneHandler; + oldExternalEntityRefHandler = externalEntityRefHandler; + oldSkippedEntityHandler = skippedEntityHandler; + oldUnknownEncodingHandler = unknownEncodingHandler; + oldElementDeclHandler = elementDeclHandler; + oldAttlistDeclHandler = attlistDeclHandler; + oldEntityDeclHandler = entityDeclHandler; + oldXmlDeclHandler = xmlDeclHandler; + oldDeclElementType = declElementType; + + oldUserData = userData; + oldHandlerArg = handlerArg; + oldDefaultExpandInternalEntities = defaultExpandInternalEntities; + oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; +#ifdef XML_DTD + oldParamEntityParsing = paramEntityParsing; + oldInEntityValue = prologState.inEntityValue; #endif - XML_Bool oldns_triplets = ns_triplets; + oldns_triplets = ns_triplets; /* Note that the new parser shares the same hash secret as the old parser, so that dtdCopy and copyEntityTable can lookup values from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt = hash_secret_salt; + oldhash_secret_salt = hash_secret_salt; #ifdef XML_DTD if (!context) @@ -1168,6 +1508,7 @@ XML_ParserFree(XML_Parser parser) destroyBindings(inheritedBindings, parser); poolDestroy(&tempPool); poolDestroy(&temp2Pool); + FREE((void *)protocolEncodingName); #ifdef XML_DTD /* external parameter entity parsers share the DTD structure parser->m_dtd with the root parser, so we must not destroy it @@ -1194,12 +1535,15 @@ XML_ParserFree(XML_Parser parser) void XMLCALL XML_UseParserAsHandlerArg(XML_Parser parser) { - handlerArg = parser; + if (parser != NULL) + handlerArg = parser; } enum XML_Error XMLCALL XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) @@ -1214,6 +1558,8 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) void XMLCALL XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { + if (parser == NULL) + return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return; @@ -1223,6 +1569,8 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) void XMLCALL XML_SetUserData(XML_Parser parser, void *p) { + if (parser == NULL) + return; if (handlerArg == userData) handlerArg = userData = p; else @@ -1232,6 +1580,8 @@ XML_SetUserData(XML_Parser parser, void *p) enum XML_Status XMLCALL XML_SetBase(XML_Parser parser, const XML_Char *p) { + if (parser == NULL) + return XML_STATUS_ERROR; if (p) { p = poolCopyString(&_dtd->pool, p); if (!p) @@ -1246,18 +1596,24 @@ XML_SetBase(XML_Parser parser, const XML_Char *p) const XML_Char * XMLCALL XML_GetBase(XML_Parser parser) { + if (parser == NULL) + return NULL; return curBase; } int XMLCALL XML_GetSpecifiedAttributeCount(XML_Parser parser) { + if (parser == NULL) + return -1; return nSpecifiedAtts; } int XMLCALL XML_GetIdAttributeIndex(XML_Parser parser) { + if (parser == NULL) + return -1; return idAttIndex; } @@ -1265,6 +1621,8 @@ XML_GetIdAttributeIndex(XML_Parser parser) const XML_AttrInfo * XMLCALL XML_GetAttributeInfo(XML_Parser parser) { + if (parser == NULL) + return NULL; return attInfo; } #endif @@ -1274,6 +1632,8 @@ XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { + if (parser == NULL) + return; startElementHandler = start; endElementHandler = end; } @@ -1281,34 +1641,39 @@ XML_SetElementHandler(XML_Parser parser, void XMLCALL XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) { - startElementHandler = start; + if (parser != NULL) + startElementHandler = start; } void XMLCALL XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) { - endElementHandler = end; + if (parser != NULL) + endElementHandler = end; } void XMLCALL XML_SetCharacterDataHandler(XML_Parser parser, XML_CharacterDataHandler handler) { - characterDataHandler = handler; + if (parser != NULL) + characterDataHandler = handler; } void XMLCALL XML_SetProcessingInstructionHandler(XML_Parser parser, XML_ProcessingInstructionHandler handler) { - processingInstructionHandler = handler; + if (parser != NULL) + processingInstructionHandler = handler; } void XMLCALL XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) { - commentHandler = handler; + if (parser != NULL) + commentHandler = handler; } void XMLCALL @@ -1316,6 +1681,8 @@ XML_SetCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start, XML_EndCdataSectionHandler end) { + if (parser == NULL) + return; startCdataSectionHandler = start; endCdataSectionHandler = end; } @@ -1323,19 +1690,23 @@ XML_SetCdataSectionHandler(XML_Parser parser, void XMLCALL XML_SetStartCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start) { - startCdataSectionHandler = start; + if (parser != NULL) + startCdataSectionHandler = start; } void XMLCALL XML_SetEndCdataSectionHandler(XML_Parser parser, XML_EndCdataSectionHandler end) { - endCdataSectionHandler = end; + if (parser != NULL) + endCdataSectionHandler = end; } void XMLCALL XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_FALSE; } @@ -1344,6 +1715,8 @@ void XMLCALL XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_TRUE; } @@ -1353,6 +1726,8 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start, XML_EndDoctypeDeclHandler end) { + if (parser == NULL) + return; startDoctypeDeclHandler = start; endDoctypeDeclHandler = end; } @@ -1360,27 +1735,31 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start) { - startDoctypeDeclHandler = start; + if (parser != NULL) + startDoctypeDeclHandler = start; } void XMLCALL XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) { - endDoctypeDeclHandler = end; + if (parser != NULL) + endDoctypeDeclHandler = end; } void XMLCALL XML_SetUnparsedEntityDeclHandler(XML_Parser parser, XML_UnparsedEntityDeclHandler handler) { - unparsedEntityDeclHandler = handler; + if (parser != NULL) + unparsedEntityDeclHandler = handler; } void XMLCALL XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) { - notationDeclHandler = handler; + if (parser != NULL) + notationDeclHandler = handler; } void XMLCALL @@ -1388,6 +1767,8 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start, XML_EndNamespaceDeclHandler end) { + if (parser == NULL) + return; startNamespaceDeclHandler = start; endNamespaceDeclHandler = end; } @@ -1395,32 +1776,38 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start) { - startNamespaceDeclHandler = start; + if (parser != NULL) + startNamespaceDeclHandler = start; } void XMLCALL XML_SetEndNamespaceDeclHandler(XML_Parser parser, XML_EndNamespaceDeclHandler end) { - endNamespaceDeclHandler = end; + if (parser != NULL) + endNamespaceDeclHandler = end; } void XMLCALL XML_SetNotStandaloneHandler(XML_Parser parser, XML_NotStandaloneHandler handler) { - notStandaloneHandler = handler; + if (parser != NULL) + notStandaloneHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandler(XML_Parser parser, XML_ExternalEntityRefHandler handler) { - externalEntityRefHandler = handler; + if (parser != NULL) + externalEntityRefHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) { + if (parser == NULL) + return; if (arg) externalEntityRefHandlerArg = (XML_Parser)arg; else @@ -1431,7 +1818,8 @@ void XMLCALL XML_SetSkippedEntityHandler(XML_Parser parser, XML_SkippedEntityHandler handler) { - skippedEntityHandler = handler; + if (parser != NULL) + skippedEntityHandler = handler; } void XMLCALL @@ -1439,6 +1827,8 @@ XML_SetUnknownEncodingHandler(XML_Parser parser, XML_UnknownEncodingHandler handler, void *data) { + if (parser == NULL) + return; unknownEncodingHandler = handler; unknownEncodingHandlerData = data; } @@ -1447,33 +1837,39 @@ void XMLCALL XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) { - elementDeclHandler = eldecl; + if (parser != NULL) + elementDeclHandler = eldecl; } void XMLCALL XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) { - attlistDeclHandler = attdecl; + if (parser != NULL) + attlistDeclHandler = attdecl; } void XMLCALL XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) { - entityDeclHandler = handler; + if (parser != NULL) + entityDeclHandler = handler; } void XMLCALL XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) { - xmlDeclHandler = handler; + if (parser != NULL) + xmlDeclHandler = handler; } int XMLCALL XML_SetParamEntityParsing(XML_Parser parser, enum XML_ParamEntityParsing peParsing) { + if (parser == NULL) + return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1489,6 +1885,10 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + if (parser == NULL) + return 0; + if (parser->m_parentParser) + return XML_SetHashSalt(parser->m_parentParser, hash_salt); /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1499,6 +1899,11 @@ XML_SetHashSalt(XML_Parser parser, enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { + if (parser != NULL) + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1531,9 +1936,22 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) if (errorCode == XML_ERROR_NONE) { switch (ps_parsing) { case XML_SUSPENDED: + /* It is hard to be certain, but it seems that this case + * cannot occur. This code is cleaning up a previous parse + * with no new data (since len == 0). Changing the parsing + * state requires getting to execute a handler function, and + * there doesn't seem to be an opportunity for that while in + * this circumstance. + * + * Given the uncertainty, we retain the code but exclude it + * from coverage tests. + * + * LCOV_EXCL_START + */ XmlUpdatePosition(encoding, positionPtr, bufferPtr, &position); positionPtr = bufferPtr; return XML_STATUS_SUSPENDED; + /* LCOV_EXCL_STOP */ case XML_INITIALIZED: case XML_PARSING: ps_parsing = XML_FINISHED; @@ -1551,6 +1969,13 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) const char *end; int nLeftOver; enum XML_Status result; + /* Detect overflow (a+b > MAX <==> b > MAX-a) */ + if (len > ((XML_Size)-1) / 2 - parseEndByteIndex) { + errorCode = XML_ERROR_NO_MEMORY; + eventPtr = eventEndPtr = NULL; + processor = errorProcessor; + return XML_STATUS_ERROR; + } parseEndByteIndex += len; positionPtr = s; ps_finalBuffer = (XML_Bool)isFinal; @@ -1583,11 +2008,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) nLeftOver = s + len - end; if (nLeftOver) { if (buffer == NULL || nLeftOver > bufferLim - buffer) { - /* FIXME avoid integer overflow */ - char *temp; - temp = (buffer == NULL - ? (char *)MALLOC(len * 2) - : (char *)REALLOC(buffer, len * 2)); + /* avoid _signed_ integer overflow */ + char *temp = NULL; + const int bytesToAllocate = (int)((unsigned)len * 2U); + if (bytesToAllocate > 0) { + temp = (buffer == NULL + ? (char *)MALLOC(bytesToAllocate) + : (char *)REALLOC(buffer, bytesToAllocate)); + } if (temp == NULL) { errorCode = XML_ERROR_NO_MEMORY; eventPtr = eventEndPtr = NULL; @@ -1595,7 +2023,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) return XML_STATUS_ERROR; } buffer = temp; - bufferLim = buffer + len * 2; + bufferLim = buffer + bytesToAllocate; } memcpy(buffer, end, nLeftOver); } @@ -1625,6 +2053,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) const char *start; enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1678,6 +2108,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) void * XMLCALL XML_GetBuffer(XML_Parser parser, int len) { + if (parser == NULL) + return NULL; if (len < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -1695,15 +2127,15 @@ XML_GetBuffer(XML_Parser parser, int len) if (len > bufferLim - bufferEnd) { #ifdef XML_CONTEXT_BYTES int keep; -#endif - int neededSize = len + (int)(bufferEnd - bufferPtr); +#endif /* defined XML_CONTEXT_BYTES */ + /* Do not invoke signed arithmetic overflow: */ + int neededSize = (int) ((unsigned)len + (unsigned)(bufferEnd - bufferPtr)); if (neededSize < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; } #ifdef XML_CONTEXT_BYTES keep = (int)(bufferPtr - buffer); - if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; neededSize += keep; @@ -1728,7 +2160,8 @@ XML_GetBuffer(XML_Parser parser, int len) if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { - bufferSize *= 2; + /* Do not invoke signed arithmetic overflow: */ + bufferSize = (int) (2U * (unsigned) bufferSize); } while (bufferSize < neededSize && bufferSize > 0); if (bufferSize <= 0) { errorCode = XML_ERROR_NO_MEMORY; @@ -1773,6 +2206,8 @@ XML_GetBuffer(XML_Parser parser, int len) enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: if (resumable) { @@ -1805,6 +2240,8 @@ XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; if (ps_parsing != XML_SUSPENDED) { errorCode = XML_ERROR_NOT_SUSPENDED; return XML_STATUS_ERROR; @@ -1841,6 +2278,8 @@ XML_ResumeParser(XML_Parser parser) void XMLCALL XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) { + if (parser == NULL) + return; assert(status != NULL); *status = parser->m_parsingStatus; } @@ -1848,20 +2287,26 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) enum XML_Error XMLCALL XML_GetErrorCode(XML_Parser parser) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; return errorCode; } XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { + if (parser == NULL) + return -1; if (eventPtr) - return parseEndByteIndex - (parseEndPtr - eventPtr); + return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; } int XMLCALL XML_GetCurrentByteCount(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventEndPtr && eventPtr) return (int)(eventEndPtr - eventPtr); return 0; @@ -1871,11 +2316,19 @@ const char * XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { #ifdef XML_CONTEXT_BYTES + if (parser == NULL) + return NULL; if (eventPtr && buffer) { - *offset = (int)(eventPtr - buffer); - *size = (int)(bufferEnd - buffer); + if (offset != NULL) + *offset = (int)(eventPtr - buffer); + if (size != NULL) + *size = (int)(bufferEnd - buffer); return buffer; } +#else + (void)parser; + (void)offset; + (void)size; #endif /* defined XML_CONTEXT_BYTES */ return (char *) 0; } @@ -1883,6 +2336,8 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) XML_Size XMLCALL XML_GetCurrentLineNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1893,6 +2348,8 @@ XML_GetCurrentLineNumber(XML_Parser parser) XML_Size XMLCALL XML_GetCurrentColumnNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1903,30 +2360,38 @@ XML_GetCurrentColumnNumber(XML_Parser parser) void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - FREE(model); + if (parser != NULL) + FREE(model); } void * XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { + if (parser == NULL) + return NULL; return MALLOC(size); } void * XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { + if (parser == NULL) + return NULL; return REALLOC(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - FREE(ptr); + if (parser != NULL) + FREE(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { + if (parser == NULL) + return; if (defaultHandler) { if (openInternalEntities) reportDefault(parser, @@ -2429,11 +2894,11 @@ doContent(XML_Parser parser, for (;;) { int bufSize; int convLen; - XmlConvert(enc, + const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if (fromPtr == rawNameEnd) { + if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2476,8 +2941,10 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; poolFinish(&tempPool); result = storeAtts(parser, enc, s, &name, &bindings); - if (result) + if (result != XML_ERROR_NONE) { + freeBindings(parser, bindings); return result; + } poolFinish(&tempPool); if (startElementHandler) { startElementHandler(handlerArg, name.str, (const XML_Char **)atts); @@ -2492,15 +2959,7 @@ doContent(XML_Parser parser, if (noElmHandlers && defaultHandler) reportDefault(parser, enc, s, next); poolClear(&tempPool); - while (bindings) { - BINDING *b = bindings; - if (endNamespaceDeclHandler) - endNamespaceDeclHandler(handlerArg, b->prefix->name); - bindings = bindings->nextTagBinding; - b->nextTagBinding = freeBindingList; - freeBindingList = b; - b->prefix->binding = b->prevPrefixBinding; - } + freeBindings(parser, bindings); } if (tagLevel == 0) return epilogProcessor(parser, next, end, nextPtr); @@ -2654,11 +3113,11 @@ doContent(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -2681,9 +3140,17 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; break; default: + /* All of the tokens produced by XmlContentTok() have their own + * explicit cases, so this default is not strictly necessary. + * However it is a useful safety net, so we retain the code and + * simply exclude it from the coverage tests. + * + * LCOV_EXCL_START + */ if (defaultHandler) reportDefault(parser, enc, s, next); break; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; switch (ps_parsing) { @@ -2698,6 +3165,29 @@ doContent(XML_Parser parser, /* not reached */ } +/* This function does not call free() on the allocated memory, merely + * moving it to the parser's freeBindingList where it can be freed or + * reused as appropriate. + */ +static void +freeBindings(XML_Parser parser, BINDING *bindings) +{ + while (bindings) { + BINDING *b = bindings; + + /* startNamespaceDeclHandler will have been called for this + * binding in addBindings(), so call the end handler now. + */ + if (endNamespaceDeclHandler) + endNamespaceDeclHandler(handlerArg, b->prefix->name); + + bindings = bindings->nextTagBinding; + b->nextTagBinding = freeBindingList; + freeBindingList = b; + b->prefix->binding = b->prevPrefixBinding; + } +} + /* Precondition: all arguments must be non-NULL; Purpose: - normalize attributes @@ -2751,13 +3241,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc, #endif attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; temp = (ATTRIBUTE *)REALLOC((void *)atts, attsSize * sizeof(ATTRIBUTE)); - if (temp == NULL) + if (temp == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } atts = temp; #ifdef XML_ATTR_INFO temp2 = (XML_AttrInfo *)REALLOC((void *)attInfo, attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) + if (temp2 == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } attInfo = temp2; #endif if (n > oldAttsSize) @@ -2894,6 +3388,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, int j; /* hash table index */ unsigned long version = nsAttsVersion; int nsAttsSize = (int)1 << nsAttsPower; + unsigned char oldNsAttsPower = nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> nsAttsPower) { /* true for nsAttsPower = 0 */ NS_ATT *temp; @@ -2903,8 +3398,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, nsAttsPower = 3; nsAttsSize = (int)1 << nsAttsPower; temp = (NS_ATT *)REALLOC(nsAtts, nsAttsSize * sizeof(NS_ATT)); - if (!temp) + if (!temp) { + /* Restore actual size of memory in nsAtts */ + nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; + } nsAtts = temp; version = 0; /* force re-initialization of nsAtts hash table */ } @@ -2922,31 +3420,56 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (s[-1] == 2) { /* prefixed */ ATTRIBUTE_ID *id; const BINDING *b; - unsigned long uriHash = hash_secret_salt; + unsigned long uriHash; + struct siphash sip_state; + struct sipkey sip_key; + + copy_salt_to_sipkey(parser, &sip_key); + sip24_init(&sip_state, &sip_key); + ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); - if (!id || !id->prefix) - return XML_ERROR_NO_MEMORY; + if (!id || !id->prefix) { + /* This code is walking through the appAtts array, dealing + * with (in this case) a prefixed attribute name. To be in + * the array, the attribute must have already been bound, so + * has to have passed through the hash table lookup once + * already. That implies that an entry for it already + * exists, so the lookup above will return a pointer to + * already allocated memory. There is no opportunaity for + * the allocator to fail, so the condition above cannot be + * fulfilled. + * + * Since it is difficult to be certain that the above + * analysis is complete, we retain the test and merely + * remove the code from coverage tests. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } b = id->prefix->binding; if (!b) return XML_ERROR_UNBOUND_PREFIX; - /* as we expand the name we also calculate its hash value */ for (j = 0; j < b->uriLen; j++) { const XML_Char c = b->uri[j]; if (!poolAppendChar(&tempPool, c)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } + + sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); + while (*s++ != XML_T(ASCII_COLON)) ; + + sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); + do { /* copies null terminator */ - const XML_Char c = *s; if (!poolAppendChar(&tempPool, *s)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } while (*s++); + uriHash = (unsigned long)sip24_final(&sip_state); + { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). */ @@ -3264,11 +3787,11 @@ doCdataSection(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = next; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -3299,8 +3822,16 @@ doCdataSection(XML_Parser parser, } return XML_ERROR_UNCLOSED_CDATA_SECTION; default: + /* Every token returned by XmlCdataSectionTok() has its own + * explicit case, so this default case will never be executed. + * We retain it as a safety net and exclude it from the coverage + * statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; @@ -3360,8 +3891,20 @@ doIgnoreSection(XML_Parser parser, eventEndPP = &eventEndPtr; } else { + /* It's not entirely clear, but it seems the following two lines + * of code cannot be executed. The only occasions on which 'enc' + * is not 'parser->m_encoding' are when this function is called + * from the internal entity processing, and IGNORE sections are an + * error in internal entities. + * + * Since it really isn't clear that this is true, we keep the code + * and just remove it from our coverage tests. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } *eventPP = s; *startPtr = NULL; @@ -3394,8 +3937,16 @@ doIgnoreSection(XML_Parser parser, } return XML_ERROR_SYNTAX; /* XML_ERROR_UNCLOSED_IGNORE_SECTION */ default: + /* All of the tokens that XmlIgnoreSectionTok() returns have + * explicit cases to handle them, so this default case is never + * executed. We keep it as a safety net anyway, and remove it + * from our test coverage statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } /* not reached */ } @@ -3408,6 +3959,7 @@ initializeEncoding(XML_Parser parser) const char *s; #ifdef XML_UNICODE char encodingBuf[128]; + /* See comments abount `protoclEncodingName` in parserInit() */ if (!protocolEncodingName) s = NULL; else { @@ -3491,7 +4043,14 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, reportDefault(parser, encoding, s, next); if (protocolEncodingName == NULL) { if (newEncoding) { - if (newEncoding->minBytesPerChar != encoding->minBytesPerChar) { + /* Check that the specified encoding does not conflict with what + * the parser has already deduced. Do we have the same number + * of bytes in the smallest representation of a character? If + * this is UTF-16, is it the same endianness? + */ + if (newEncoding->minBytesPerChar != encoding->minBytesPerChar + || (newEncoding->minBytesPerChar == 2 && + newEncoding != encoding)) { eventPtr = encodingName; return XML_ERROR_INCORRECT_ENCODING; } @@ -3636,15 +4195,14 @@ entityValueInitProcessor(XML_Parser parser, result = processXmlDecl(parser, 0, start, next); if (result != XML_ERROR_NONE) return result; - switch (ps_parsing) { - case XML_SUSPENDED: - *nextPtr = next; - return XML_ERROR_NONE; - case XML_FINISHED: + /* At this point, ps_parsing cannot be XML_SUSPENDED. For that + * to happen, a parameter entity parsing handler must have + * attempted to suspend the parser, which fails and raises an + * error. The parser can be aborted, but can't be suspended. + */ + if (ps_parsing == XML_FINISHED) return XML_ERROR_ABORTED; - default: - *nextPtr = next; - } + *nextPtr = next; /* stop scanning for text declaration - we found one */ processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); @@ -3660,6 +4218,14 @@ entityValueInitProcessor(XML_Parser parser, *nextPtr = next; return XML_ERROR_NONE; } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "paramEntities, externalSubsetName, sizeof(ENTITY)); - if (!entity) - return XML_ERROR_NO_MEMORY; + if (!entity) { + /* The external subset name "#" will have already been + * inserted into the hash table at the start of the + * external entity parsing, so no allocation will happen + * and lookup() cannot fail. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } if (useForeignDTD) entity->base = curBase; dtd->paramEntityRead = XML_FALSE; @@ -4439,8 +5011,10 @@ doProlog(XML_Parser parser, if (prologState.level >= groupSize) { if (groupSize) { char *temp = (char *)REALLOC(groupConnector, groupSize *= 2); - if (temp == NULL) + if (temp == NULL) { + groupSize /= 2; return XML_ERROR_NO_MEMORY; + } groupConnector = temp; if (dtd->scaffIndex) { int *temp = (int *)REALLOC(dtd->scaffIndex, @@ -4452,8 +5026,10 @@ doProlog(XML_Parser parser, } else { groupConnector = (char *)MALLOC(groupSize = 32); - if (!groupConnector) + if (!groupConnector) { + groupSize = 0; return XML_ERROR_NO_MEMORY; + } } } groupConnector[prologState.level] = 0; @@ -4516,8 +5092,29 @@ doProlog(XML_Parser parser, : !dtd->hasParamEntityRefs)) { if (!entity) return XML_ERROR_UNDEFINED_ENTITY; - else if (!entity->is_internal) - return XML_ERROR_ENTITY_DECLARED_IN_PE; + else if (!entity->is_internal) { + /* It's hard to exhaustively search the code to be sure, + * but there doesn't seem to be a way of executing the + * following line. There are two cases: + * + * If 'standalone' is false, the DTD must have no + * parameter entities or we wouldn't have passed the outer + * 'if' statement. That measn the only entity in the hash + * table is the external subset name "#" which cannot be + * given as a parameter entity name in XML syntax, so the + * lookup must have returned NULL and we don't even reach + * the test for an internal entity. + * + * If 'standalone' is true, it does not seem to be + * possible to create entities taking this code path that + * are not internal entities, so fail the test above. + * + * Because this analysis is very uncertain, the code is + * being left in place and merely removed from the + * coverage test statistics. + */ + return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */ + } } else if (!entity) { dtd->keepProcessing = dtd->standalone; @@ -4836,6 +5433,8 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, openEntity->internalEventEndPtr = NULL; textStart = (char *)entity->textPtr; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4881,6 +5480,8 @@ internalEntityProcessor(XML_Parser parser, entity = openEntity->entity; textStart = ((char *)entity->textPtr) + entity->processed; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4927,9 +5528,9 @@ internalEntityProcessor(XML_Parser parser, static enum XML_Error PTRCALL errorProcessor(XML_Parser parser, - const char *s, - const char *end, - const char **nextPtr) + const char *UNUSED_P(s), + const char *UNUSED_P(end), + const char **UNUSED_P(nextPtr)) { return errorCode; } @@ -4985,11 +5586,15 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) break; n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = ptr; - return XML_ERROR_BAD_CHAR_REF; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (!poolAppendChar(pool, buf[i])) return XML_ERROR_NO_MEMORY; @@ -5063,8 +5668,26 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; } if (entity->open) { - if (enc == encoding) - eventPtr = ptr; + if (enc == encoding) { + /* It does not appear that this line can be executed. + * + * The "if (entity->open)" check catches recursive entity + * definitions. In order to be called with an open + * entity, it must have gone through this code before and + * been through the recursive call to + * appendAttributeValue() some lines below. That call + * sets the local encoding ("enc") to the parser's + * internal encoding (internal_utf8 or internal_utf16), + * which can never be the same as the principle encoding. + * It doesn't appear there is another code path that gets + * here with entity->open being TRUE. + * + * Since it is not certain that this logic is watertight, + * we keep the line and merely exclude it from coverage + * tests. + */ + eventPtr = ptr; /* LCOV_EXCL_LINE */ + } return XML_ERROR_RECURSIVE_ENTITY_REF; } if (entity->notation) { @@ -5091,9 +5714,21 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } break; default: + /* The only token returned by XmlAttributeValueTok() that does + * not have an explicit case here is XML_TOK_PARTIAL_CHAR. + * Getting that would require an entity name to contain an + * incomplete XML character (e.g. \xE2\x82); however previous + * tokenisers will have already recognised and rejected such + * names before XmlAttributeValueTok() gets a look-in. This + * default case should be retained as a safety net, but the code + * excluded from coverage tests. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = ptr; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } ptr = next; } @@ -5226,12 +5861,15 @@ storeEntityValue(XML_Parser parser, goto endEntityValue; } n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = entityTextPtr; - result = XML_ERROR_BAD_CHAR_REF; - goto endEntityValue; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (pool->end == pool->ptr && !poolGrow(pool)) { result = XML_ERROR_NO_MEMORY; @@ -5252,10 +5890,18 @@ storeEntityValue(XML_Parser parser, result = XML_ERROR_INVALID_TOKEN; goto endEntityValue; default: + /* This default case should be unnecessary -- all the tokens + * that XmlEntityValueTok() can return have their own explicit + * cases -- but should be retained for safety. We do however + * exclude it from the coverage statistics. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = entityTextPtr; result = XML_ERROR_UNEXPECTED_STATE; goto endEntityValue; + /* LCOV_EXCL_STOP */ } entityTextPtr = next; } @@ -5345,6 +5991,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, const char *end) { if (MUST_CONVERT(enc, s)) { + enum XML_Convert_Result convert_res; const char **eventPP; const char **eventEndPP; if (enc == encoding) { @@ -5352,16 +5999,33 @@ reportDefault(XML_Parser parser, const ENCODING *enc, eventEndPP = &eventEndPtr; } else { + /* To get here, two things must be true; the parser must be + * using a character encoding that is not the same as the + * encoding passed in, and the encoding passed in must need + * conversion to the internal format (UTF-8 unless XML_UNICODE + * is defined). The only occasions on which the encoding passed + * in is not the same as the parser's encoding are when it is + * the internal encoding (e.g. a previously defined parameter + * entity, already converted to internal format). This by + * definition doesn't need conversion, so the whole branch never + * gets executed. + * + * For safety's sake we don't delete these lines and merely + * exclude them from coverage statistics. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } do { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); + convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; defaultHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); *eventPP = s; - } while (s != end); + } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else defaultHandler(handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s)); @@ -5522,9 +6186,30 @@ getContext(XML_Parser parser) len = dtd->defaultPrefix.binding->uriLen; if (namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) - return NULL; + for (i = 0; i < len; i++) { + if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } + } needSep = XML_TRUE; } @@ -5536,8 +6221,15 @@ getContext(XML_Parser parser) PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (!prefix) break; - if (!prefix->binding) - continue; + if (!prefix->binding) { + /* This test appears to be (justifiable) paranoia. There does + * not seem to be a way of injecting a prefix without a binding + * that doesn't get errored long before this function is called. + * The test should remain for safety's sake, so we instead + * exclude the following line from the coverage statistics. + */ + continue; /* LCOV_EXCL_LINE */ + } if (needSep && !poolAppendChar(&tempPool, CONTEXT_SEP)) return NULL; for (s = prefix->name; *s; s++) @@ -5840,7 +6532,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H newE->defaultAtts = (DEFAULT_ATTRIBUTE *) ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (!newE->defaultAtts) { - ms->free_fcn(newE); return 0; } } @@ -5975,13 +6666,32 @@ keyeq(KEY s1, KEY s2) return XML_FALSE; } +static size_t +keylen(KEY s) +{ + size_t len = 0; + for (; *s; s++, len++); + return len; +} + +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key) +{ + key->k[0] = 0; + key->k[1] = get_hash_secret_salt(parser); +} + static unsigned long FASTCALL hash(XML_Parser parser, KEY s) { - unsigned long h = hash_secret_salt; - while (*s) - h = CHAR_HASH(h, *s++); - return h; + struct siphash state; + struct sipkey key; + (void)sip_tobin; + (void)sip24_valid; + copy_salt_to_sipkey(parser, &key); + sip24_init(&state, &key); + sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + return (unsigned long)sip24_final(&state); } static NAMED * @@ -6166,8 +6876,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, if (!pool->ptr && !poolGrow(pool)) return NULL; for (;;) { - XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); - if (ptr == end) + const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; if (!poolGrow(pool)) return NULL; @@ -6190,8 +6900,20 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { - if (!pool->ptr && !poolGrow(pool)) - return NULL; + if (!pool->ptr && !poolGrow(pool)) { + /* The following line is unreachable given the current usage of + * poolCopyStringN(). Currently it is called from exactly one + * place to copy the text of a simple general entity. By that + * point, the name of the entity is already stored in the pool, so + * pool->ptr cannot be NULL. + * + * If poolCopyStringN() is used elsewhere as it well might be, + * this line may well become executable again. Regardless, this + * sort of check shouldn't be removed lightly, so we just exclude + * it from the coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } for (; n > 0; --n, s++) { if (!poolAppendChar(pool, *s)) return NULL; @@ -6224,6 +6946,35 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, return pool->start; } +static size_t +poolBytesToAllocateFor(int blockSize) +{ + /* Unprotected math would be: + ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char); + ** + ** Detect overflow, avoiding _signed_ overflow undefined behavior + ** For a + b * c we check b * c in isolation first, so that addition of a + ** on top has no chance of making us accept a small non-negative number + */ + const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */ + + if (blockSize <= 0) + return 0; + + if (blockSize > (int)(INT_MAX / stretch)) + return 0; + + { + const int stretchedBlockSize = blockSize * (int)stretch; + const int bytesToAllocate = (int)( + offsetof(BLOCK, s) + (unsigned)stretchedBlockSize); + if (bytesToAllocate < 0) + return 0; + + return (size_t)bytesToAllocate; + } +} + static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { @@ -6251,28 +7002,71 @@ poolGrow(STRING_POOL *pool) } } if (pool->blocks && pool->start == pool->blocks->s) { - int blockSize = (int)(pool->end - pool->start)*2; - BLOCK *temp = (BLOCK *) - pool->mem->realloc_fcn(pool->blocks, - (offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char))); + BLOCK *temp; + int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + size_t bytesToAllocate; + + // NOTE: Needs to be calculated prior to calling `realloc` + // to avoid dangling pointers: + const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX/2 bytes have already been allocated. This isn't + * readily testable, since it is unlikely that an average + * machine will have that much memory, so we exclude it from the + * coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + temp = (BLOCK *) + pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; pool->blocks->size = blockSize; - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + offsetInsideBlock; pool->start = pool->blocks->s; pool->end = pool->start + blockSize; } else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + size_t bytesToAllocate; + + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX bytes have already been allocated (which is prevented + * by various pieces of program logic, not least this one, never + * mind the unlikelihood of actually having that much memory) or + * the pool control fields have been corrupted (which could + * conceivably happen in an extremely buggy user handler + * function). Either way it isn't readily testable, so we + * exclude it from the coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } + if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; - else + else { + /* Detect overflow, avoiding _signed_ overflow undefined behavior */ + if ((int)((unsigned)blockSize * 2U) < 0) { + return XML_FALSE; + } blockSize *= 2; - tem = (BLOCK *)pool->mem->malloc_fcn(offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char)); + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate); if (!tem) return XML_FALSE; tem->size = blockSize; @@ -6419,3 +7213,26 @@ getElementType(XML_Parser parser, } return ret; } + +static XML_Char * +copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite) +{ + int charsRequired = 0; + XML_Char *result; + + /* First determine how long the string is */ + while (s[charsRequired] != 0) { + charsRequired++; + } + /* Include the terminator */ + charsRequired++; + + /* Now allocate space for the copy */ + result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + if (result == NULL) + return NULL; + /* Copy the original into place */ + memcpy(result, s, charsRequired * sizeof(XML_Char)); + return result; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 44772e21dd337b..708507d575be84 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -1,22 +1,44 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include -#ifdef COMPILED_FROM_DSP +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -176,7 +198,14 @@ prolog1(PROLOG_STATE *state, case XML_TOK_COMMENT: return XML_ROLE_COMMENT; case XML_TOK_BOM: - return XML_ROLE_NONE; + /* This case can never arise. To reach this role function, the + * parse must have passed through prolog0 and therefore have had + * some form of input, even if only a space. At that point, a + * byte order mark is no longer a valid character (though + * technically it should be interpreted as a non-breaking space), + * so will be rejected by the tokenizing stages. + */ + return XML_ROLE_NONE; /* LCOV_EXCL_LINE */ case XML_TOK_DECL_OPEN: if (!XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), @@ -195,9 +224,9 @@ prolog1(PROLOG_STATE *state, static int PTRCALL prolog2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -216,9 +245,9 @@ prolog2(PROLOG_STATE *state, static int PTRCALL doctype0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -264,9 +293,9 @@ doctype1(PROLOG_STATE *state, static int PTRCALL doctype2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -281,9 +310,9 @@ doctype2(PROLOG_STATE *state, static int PTRCALL doctype3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -298,9 +327,9 @@ doctype3(PROLOG_STATE *state, static int PTRCALL doctype4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -318,9 +347,9 @@ doctype4(PROLOG_STATE *state, static int PTRCALL doctype5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -437,9 +466,9 @@ externalSubset1(PROLOG_STATE *state, static int PTRCALL entity0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -457,9 +486,9 @@ entity0(PROLOG_STATE *state, static int PTRCALL entity1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -502,9 +531,9 @@ entity2(PROLOG_STATE *state, static int PTRCALL entity3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -519,9 +548,9 @@ entity3(PROLOG_STATE *state, static int PTRCALL entity4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -559,9 +588,9 @@ entity5(PROLOG_STATE *state, static int PTRCALL entity6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -605,9 +634,9 @@ entity7(PROLOG_STATE *state, static int PTRCALL entity8(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -622,9 +651,9 @@ entity8(PROLOG_STATE *state, static int PTRCALL entity9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -639,9 +668,9 @@ entity9(PROLOG_STATE *state, static int PTRCALL entity10(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -656,9 +685,9 @@ entity10(PROLOG_STATE *state, static int PTRCALL notation0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -697,9 +726,9 @@ notation1(PROLOG_STATE *state, static int PTRCALL notation2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -714,9 +743,9 @@ notation2(PROLOG_STATE *state, static int PTRCALL notation3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -732,9 +761,9 @@ notation3(PROLOG_STATE *state, static int PTRCALL notation4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -753,9 +782,9 @@ notation4(PROLOG_STATE *state, static int PTRCALL attlist0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -771,9 +800,9 @@ attlist0(PROLOG_STATE *state, static int PTRCALL attlist1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -833,9 +862,9 @@ attlist2(PROLOG_STATE *state, static int PTRCALL attlist3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -852,9 +881,9 @@ attlist3(PROLOG_STATE *state, static int PTRCALL attlist4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -872,9 +901,9 @@ attlist4(PROLOG_STATE *state, static int PTRCALL attlist5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -889,9 +918,9 @@ attlist5(PROLOG_STATE *state, static int PTRCALL attlist6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -906,9 +935,9 @@ attlist6(PROLOG_STATE *state, static int PTRCALL attlist7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -967,9 +996,9 @@ attlist8(PROLOG_STATE *state, static int PTRCALL attlist9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -984,9 +1013,9 @@ attlist9(PROLOG_STATE *state, static int PTRCALL element0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1072,9 +1101,9 @@ element2(PROLOG_STATE *state, static int PTRCALL element3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1097,9 +1126,9 @@ element3(PROLOG_STATE *state, static int PTRCALL element4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1115,9 +1144,9 @@ element4(PROLOG_STATE *state, static int PTRCALL element5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1136,9 +1165,9 @@ element5(PROLOG_STATE *state, static int PTRCALL element6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1166,9 +1195,9 @@ element6(PROLOG_STATE *state, static int PTRCALL element7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1240,9 +1269,9 @@ condSect0(PROLOG_STATE *state, static int PTRCALL condSect1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1258,9 +1287,9 @@ condSect1(PROLOG_STATE *state, static int PTRCALL condSect2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1277,9 +1306,9 @@ condSect2(PROLOG_STATE *state, static int PTRCALL declClose(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1291,15 +1320,36 @@ declClose(PROLOG_STATE *state, return common(state, tok); } +/* This function will only be invoked if the internal logic of the + * parser has broken down. It is used in two cases: + * + * 1: When the XML prolog has been finished. At this point the + * processor (the parser level above these role handlers) should + * switch from prologProcessor to contentProcessor and reinitialise + * the handler function. + * + * 2: When an error has been detected (via common() below). At this + * point again the processor should be switched to errorProcessor, + * which will never call a handler. + * + * The result of this is that error() can only be called if the + * processor switch failed to happen, which is an internal error and + * therefore we shouldn't be able to provoke it simply by using the + * library. It is a necessary backstop, however, so we merely exclude + * it from the coverage statistics. + * + * LCOV_EXCL_START + */ static int PTRCALL -error(PROLOG_STATE *state, - int tok, - const char *ptr, - const char *end, - const ENCODING *enc) +error(PROLOG_STATE *UNUSED_P(state), + int UNUSED_P(tok), + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { return XML_ROLE_NONE; } +/* LCOV_EXCL_STOP */ static int FASTCALL common(PROLOG_STATE *state, int tok) diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index 4dd9f06f97679e..e5f048eab55c6e 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef XmlRole_INCLUDED diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index bf09dfc72b9fc8..007aed0640a2ef 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -1,22 +1,46 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include +#include +#include // memcpy -#ifdef COMPILED_FROM_DSP +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -46,7 +70,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1 << ((lo) & 0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -56,7 +80,7 @@ (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \ + ((((byte)[0]) & 3) << 1) \ + ((((byte)[1]) >> 5) & 1)] \ - & (1 << (((byte)[1]) & 0x1F))) + & (1u << (((byte)[1]) & 0x1F))) /* A 3 byte UTF-8 representation splits the characters 16 bits between the bottom 4, 6 and 6 bits of the bytes. We need 8 bits to index @@ -69,7 +93,7 @@ << 3) \ + ((((byte)[1]) & 3) << 1) \ + ((((byte)[2]) >> 5) & 1)] \ - & (1 << (((byte)[2]) & 0x1F))) + & (1u << (((byte)[2]) & 0x1F))) #define UTF8_GET_NAMING(pages, p, n) \ ((n) == 2 \ @@ -122,19 +146,19 @@ ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) static int PTRFASTCALL -isNever(const ENCODING *enc, const char *p) +isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p)) { return 0; } static int PTRFASTCALL -utf8_isName2(const ENCODING *enc, const char *p) +utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isName3(const ENCODING *enc, const char *p) +utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); } @@ -142,13 +166,13 @@ utf8_isName3(const ENCODING *enc, const char *p) #define utf8_isName4 isNever static int PTRFASTCALL -utf8_isNmstrt2(const ENCODING *enc, const char *p) +utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isNmstrt3(const ENCODING *enc, const char *p) +utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); } @@ -156,19 +180,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) #define utf8_isNmstrt4 isNever static int PTRFASTCALL -utf8_isInvalid2(const ENCODING *enc, const char *p) +utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID2((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid3(const ENCODING *enc, const char *p) +utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID3((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid4(const ENCODING *enc, const char *p) +utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID4((const unsigned char *)p); } @@ -222,6 +246,17 @@ struct normal_encoding { E ## isInvalid3, \ E ## isInvalid4 +#define NULL_VTABLE \ + /* isName2 */ NULL, \ + /* isName3 */ NULL, \ + /* isName4 */ NULL, \ + /* isNmstrt2 */ NULL, \ + /* isNmstrt3 */ NULL, \ + /* isNmstrt4 */ NULL, \ + /* isInvalid2 */ NULL, \ + /* isInvalid3 */ NULL, \ + /* isInvalid4 */ NULL + static int FASTCALL checkCharRefNumber(int); #include "xmltok_impl.h" @@ -318,39 +353,100 @@ enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */ UTF8_cval4 = 0xf0 }; -static void PTRCALL -utf8_toUtf8(const ENCODING *enc, +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef) +{ + const char * fromLim = *fromLimRef; + size_t walked = 0; + for (; fromLim > from; fromLim--, walked++) { + const unsigned char prev = (unsigned char)fromLim[-1]; + if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */ + if (walked + 1 >= 4) { + fromLim += 4 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */ + if (walked + 1 >= 3) { + fromLim += 3 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */ + if (walked + 1 >= 2) { + fromLim += 2 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */ + break; + } + } + *fromLimRef = fromLim; +} + +static enum XML_Convert_Result PTRCALL +utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - char *to; - const char *from; - if (fromLim - *fromP > toLim - *toP) { - /* Avoid copying partial characters. */ - for (fromLim = *fromP + (toLim - *toP); fromLim > *fromP; fromLim--) - if (((unsigned char)fromLim[-1] & 0xc0) != 0x80) - break; + bool input_incomplete = false; + bool output_exhausted = false; + + /* Avoid copying partial characters (due to limited space). */ + const ptrdiff_t bytesAvailable = fromLim - *fromP; + const ptrdiff_t bytesStorable = toLim - *toP; + if (bytesAvailable > bytesStorable) { + fromLim = *fromP + bytesStorable; + output_exhausted = true; } - for (to = *toP, from = *fromP; from != fromLim; from++, to++) - *to = *from; - *fromP = from; - *toP = to; + + /* Avoid copying partial characters (from incomplete input). */ + const char * const fromLimBefore = fromLim; + align_limit_to_full_utf8_characters(*fromP, &fromLim); + if (fromLim < fromLimBefore) { + input_incomplete = true; + } + + const ptrdiff_t bytesToCopy = fromLim - *fromP; + memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); + *fromP += bytesToCopy; + *toP += bytesToCopy; + + if (output_exhausted) // needs to go first + return XML_CONVERT_OUTPUT_EXHAUSTED; + else if (input_incomplete) + return XML_CONVERT_INPUT_INCOMPLETE; + else + return XML_CONVERT_COMPLETED; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; unsigned short *to = *toP; const char *from = *fromP; - while (from != fromLim && to != toLim) { + while (from < fromLim && to < toLim) { switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { case BT_LEAD2: + if (fromLim - from < 2) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; break; case BT_LEAD3: + if (fromLim - from < 3) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); from += 3; @@ -358,8 +454,14 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD4: { unsigned long n; - if (to + 1 == toLim) + if (toLim - to < 2) { + res = XML_CONVERT_OUTPUT_EXHAUSTED; goto after; + } + if (fromLim - from < 4) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12) | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f); n -= 0x10000; @@ -374,9 +476,12 @@ utf8_toUtf16(const ENCODING *enc, break; } } + if (from < fromLim) + res = XML_CONVERT_OUTPUT_EXHAUSTED; after: *fromP = from; *toP = to; + return res; } #ifdef XML_NS @@ -425,38 +530,43 @@ static const struct normal_encoding internal_utf8_encoding = { STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_) }; -static void PTRCALL -latin1_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { for (;;) { unsigned char c; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; c = (unsigned char)**fromP; if (c & 0x80) { if (toLim - *toP < 2) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = (char)((c >> 6) | UTF8_cval2); *(*toP)++ = (char)((c & 0x3f) | 0x80); (*fromP)++; } else { if (*toP == toLim) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = *(*fromP)++; } } } -static void PTRCALL -latin1_toUtf16(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf16(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = (unsigned char)*(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -467,7 +577,7 @@ static const struct normal_encoding latin1_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -480,16 +590,21 @@ static const struct normal_encoding latin1_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; -static void PTRCALL -ascii_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +ascii_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = *(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -500,7 +615,7 @@ static const struct normal_encoding ascii_encoding_ns = { #include "asciitab.h" /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -513,7 +628,7 @@ static const struct normal_encoding ascii_encoding = { #undef BT_COLON /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; static int PTRFASTCALL @@ -536,13 +651,14 @@ unicode_byte_type(char hi, char lo) } #define DEFINE_UTF16_TO_UTF8(E) \ -static void PTRCALL \ -E ## toUtf8(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf8(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ char **toP, const char *toLim) \ { \ - const char *from; \ - for (from = *fromP; from != fromLim; from += 2) { \ + const char *from = *fromP; \ + fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */ \ + for (; from < fromLim; from += 2) { \ int plane; \ unsigned char lo2; \ unsigned char lo = GET_LO(from); \ @@ -552,7 +668,7 @@ E ## toUtf8(const ENCODING *enc, \ if (lo < 0x80) { \ if (*toP == toLim) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = lo; \ break; \ @@ -562,7 +678,7 @@ E ## toUtf8(const ENCODING *enc, \ case 0x4: case 0x5: case 0x6: case 0x7: \ if (toLim - *toP < 2) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2); \ *(*toP)++ = ((lo & 0x3f) | 0x80); \ @@ -570,7 +686,7 @@ E ## toUtf8(const ENCODING *enc, \ default: \ if (toLim - *toP < 3) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \ *(*toP)++ = ((hi >> 4) | UTF8_cval3); \ @@ -580,7 +696,11 @@ E ## toUtf8(const ENCODING *enc, \ case 0xD8: case 0xD9: case 0xDA: case 0xDB: \ if (toLim - *toP < 4) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + } \ + if (fromLim - from < 4) { \ + *fromP = from; \ + return XML_CONVERT_INPUT_INCOMPLETE; \ } \ plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \ *(*toP)++ = ((plane >> 2) | UTF8_cval4); \ @@ -596,20 +716,32 @@ E ## toUtf8(const ENCODING *enc, \ } \ } \ *fromP = from; \ + if (from < fromLim) \ + return XML_CONVERT_INPUT_INCOMPLETE; \ + else \ + return XML_CONVERT_COMPLETED; \ } #define DEFINE_UTF16_TO_UTF16(E) \ -static void PTRCALL \ -E ## toUtf16(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf16(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) \ { \ + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ + fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ /* Avoid copying first half only of surrogate */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) \ + && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ fromLim -= 2; \ - for (; *fromP != fromLim && *toP != toLim; *fromP += 2) \ + res = XML_CONVERT_INPUT_INCOMPLETE; \ + } \ + for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \ *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \ + if ((*toP == toLim) && (*fromP < fromLim)) \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + else \ + return res; \ } #define SET2(ptr, ch) \ @@ -726,7 +858,7 @@ static const struct normal_encoding little2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -745,7 +877,7 @@ static const struct normal_encoding little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #if BYTEORDER != 4321 @@ -758,7 +890,7 @@ static const struct normal_encoding internal_little2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -771,7 +903,7 @@ static const struct normal_encoding internal_little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -867,7 +999,7 @@ static const struct normal_encoding big2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -886,7 +1018,7 @@ static const struct normal_encoding big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #if BYTEORDER != 1234 @@ -899,7 +1031,7 @@ static const struct normal_encoding internal_big2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -912,7 +1044,7 @@ static const struct normal_encoding internal_big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -928,7 +1060,11 @@ streqci(const char *s1, const char *s2) if (ASCII_a <= c1 && c1 <= ASCII_z) c1 += ASCII_A - ASCII_a; if (ASCII_a <= c2 && c2 <= ASCII_z) - c2 += ASCII_A - ASCII_a; + /* The following line will never get executed. streqci() is + * only called from two places, both of which guarantee to put + * upper-case strings into s2. + */ + c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */ if (c1 != c2) return 0; if (!c1) @@ -938,7 +1074,7 @@ streqci(const char *s1, const char *s2) } static void PTRCALL -initUpdatePosition(const ENCODING *enc, const char *ptr, +initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, POSITION *pos) { normal_updatePosition(&utf8_encoding.enc, ptr, end, pos); @@ -1200,7 +1336,7 @@ XmlUtf8Encode(int c, char *buf) }; if (c < 0) - return 0; + return 0; /* LCOV_EXCL_LINE: this case is always eliminated beforehand */ if (c < min2) { buf[0] = (char)(c | UTF8_cval1); return 1; @@ -1223,7 +1359,7 @@ XmlUtf8Encode(int c, char *buf) buf[3] = (char)((c & 0x3f) | 0x80); return 4; } - return 0; + return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } int FASTCALL @@ -1288,7 +1424,7 @@ unknown_isInvalid(const ENCODING *enc, const char *p) return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) @@ -1299,21 +1435,21 @@ unknown_toUtf8(const ENCODING *enc, const char *utf8; int n; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; utf8 = uenc->utf8[(unsigned char)**fromP]; n = *utf8++; if (n == 0) { int c = uenc->convert(uenc->userData, *fromP); n = XmlUtf8Encode(c, buf); if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; utf8 = buf; *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP] - (BT_LEAD2 - 2)); } else { if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; (*fromP)++; } do { @@ -1322,13 +1458,13 @@ unknown_toUtf8(const ENCODING *enc, } } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); - while (*fromP != fromLim && *toP != toLim) { + while (*fromP < fromLim && *toP < toLim) { unsigned short c = uenc->utf16[(unsigned char)**fromP]; if (c == 0) { c = (unsigned short) @@ -1340,6 +1476,11 @@ unknown_toUtf16(const ENCODING *enc, (*fromP)++; *(*toP)++ = c; } + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } ENCODING * @@ -1369,6 +1510,9 @@ XmlInitUnknownEncoding(void *mem, else if (c < 0) { if (c < -4) return 0; + /* Multi-byte sequences need a converter function */ + if (!convert) + return 0; e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2)); e->utf8[i][0] = 0; e->utf16[i] = 0; @@ -1503,7 +1647,7 @@ initScan(const ENCODING * const *encodingTable, { const ENCODING **encPtr; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; encPtr = enc->encPtr; if (ptr + 1 == end) { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index ca867aa6b429cf..6d31879b331d83 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -1,5 +1,33 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef XmlTok_INCLUDED @@ -130,6 +158,12 @@ typedef int (PTRCALL *SCANNER)(const ENCODING *, const char *, const char **); +enum XML_Convert_Result { + XML_CONVERT_COMPLETED = 0, + XML_CONVERT_INPUT_INCOMPLETE = 1, + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining as well */ +}; + struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; @@ -158,12 +192,12 @@ struct encoding { const char *ptr, const char *end, const char **badPtr); - void (PTRCALL *utf8Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim); - void (PTRCALL *utf16Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 9c2895b87739ef..93328b841a168b 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,8 +1,35 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* This file is included! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/* This file is included! */ #ifdef XML_TOK_IMPL_C #ifndef IS_INVALID_CHAR @@ -87,27 +114,45 @@ #define PREFIX(ident) ident #endif + +#define HAS_CHARS(enc, ptr, end, count) \ + (end - ptr >= count * MINBPC(enc)) + +#define HAS_CHAR(enc, ptr, end) \ + HAS_CHARS(enc, ptr, end, 1) + +#define REQUIRE_CHARS(enc, ptr, end, count) \ + { \ + if (! HAS_CHARS(enc, ptr, end, count)) { \ + return XML_TOK_PARTIAL; \ + } \ + } + +#define REQUIRE_CHAR(enc, ptr, end) \ + REQUIRE_CHARS(enc, ptr, end, 1) + + /* ptr points to character following " */ switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) { case BT_S: case BT_CR: case BT_LF: case BT_PERCNT: @@ -175,7 +218,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, +PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, int *tokPtr) { int upper = 0; @@ -225,15 +268,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, { int tok; const char *target = ptr; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -242,13 +284,12 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_QUEST: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -266,8 +307,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -282,15 +322,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, +PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, const char **nextTokPtr) { static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB }; int i; /* CDATA[ */ - if (end - ptr < 6 * MINBPC(enc)) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 6); for (i = 0; i < 6; i++, ptr += MINBPC(enc)) { if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) { *nextTokPtr = ptr; @@ -305,7 +344,7 @@ static int PTRCALL PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -319,13 +358,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, switch (BYTE_TYPE(enc, ptr)) { case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); break; @@ -334,8 +371,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, return XML_TOK_CDATA_SECT_CLOSE; case BT_CR: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); *nextTokPtr = ptr; @@ -348,7 +384,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -383,19 +419,18 @@ static int PTRCALL PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: case BT_LF: break; @@ -432,7 +467,7 @@ static int PTRCALL PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -441,7 +476,7 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -464,7 +499,7 @@ static int PTRCALL PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { if (CHAR_MATCHES(enc, ptr, ASCII_x)) return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); switch (BYTE_TYPE(enc, ptr)) { @@ -474,7 +509,7 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: break; @@ -496,8 +531,7 @@ static int PTRCALL PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_NUM: @@ -506,7 +540,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -529,7 +563,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon = 0; #endif - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -540,8 +574,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -555,8 +588,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, int t; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == BT_EQUALS) break; @@ -579,8 +611,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #endif for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); open = BYTE_TYPE(enc, ptr); if (open == BT_QUOT || open == BT_APOS) break; @@ -598,8 +629,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* in attribute value */ for (;;) { int t; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == open) break; @@ -624,8 +654,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: @@ -642,8 +671,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to closing quote */ for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -655,8 +683,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -688,13 +715,12 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon; #endif - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_EXCL: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_MINUS: return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -716,7 +742,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, hadColon = 0; #endif /* we have a start-tag */ - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -727,8 +753,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -740,7 +765,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_CR: case BT_LF: { ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_GT: @@ -765,8 +790,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -785,7 +809,7 @@ static int PTRCALL PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -803,7 +827,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); case BT_CR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -814,12 +838,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_NEWLINE; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); @@ -832,7 +856,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -845,12 +869,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) #undef LEAD_CASE case BT_RSQB: - if (ptr + MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 2)) { if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) { ptr += MINBPC(enc); break; } - if (ptr + 2*MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 3)) { if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) { ptr += MINBPC(enc); break; @@ -884,8 +908,7 @@ static int PTRCALL PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_LF: case BT_CR: case BT_PERCNT: @@ -895,7 +918,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -913,15 +936,14 @@ static int PTRCALL PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_CR: case BT_LF: case BT_S: @@ -941,7 +963,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { int t = BYTE_TYPE(enc, ptr); switch (t) { INVALID_CASES(ptr, nextTokPtr) @@ -950,7 +972,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, ptr += MINBPC(enc); if (t != open) break; - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_LITERAL; *nextTokPtr = ptr; switch (BYTE_TYPE(enc, ptr)) { @@ -973,7 +995,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -992,8 +1014,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_LT: { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_EXCL: return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -1021,7 +1042,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_LF: for (;;) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) break; switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_LF: @@ -1048,11 +1069,10 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_BRACKET; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_BRACKET; if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if (ptr + MINBPC(enc) == end) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 2); if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) { *nextTokPtr = ptr + 2*MINBPC(enc); return XML_TOK_COND_SECT_CLOSE; @@ -1065,7 +1085,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_PAREN; case BT_RPAR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_PAREN; switch (BYTE_TYPE(enc, ptr)) { case BT_AST: @@ -1141,7 +1161,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_GT: case BT_RPAR: case BT_COMMA: @@ -1154,8 +1174,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); switch (tok) { case XML_TOK_NAME: - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); tok = XML_TOK_PREFIXED_NAME; switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) @@ -1204,10 +1223,18 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1232,7 +1259,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1262,10 +1289,18 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1294,7 +1329,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1326,15 +1361,15 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, end = ptr + n; } } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_LT: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_EXCL)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_LSQB)) { ++level; ptr += MINBPC(enc); @@ -1342,11 +1377,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, } break; case BT_RSQB: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr += MINBPC(enc); if (level == 0) { @@ -1373,7 +1408,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, { ptr += MINBPC(enc); end -= MINBPC(enc); - for (; ptr != end; ptr += MINBPC(enc)) { + for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -1521,7 +1556,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, } static int PTRFASTCALL -PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) +PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr) { int result = 0; /* skip &# */ @@ -1565,7 +1600,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) } static int PTRCALL -PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, +PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end) { switch ((end - ptr)/MINBPC(enc)) { @@ -1618,6 +1653,14 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, return 0; } +/* This function does not appear to be called from anywhere within the + * library code. It is used via the macro XmlSameName(), which is + * defined but never used. Since it appears in the encoding function + * table, removing it is not a thing to be undertaken lightly. For + * the moment, we simply exclude it from coverage tests. + * + * LCOV_EXCL_START + */ static int PTRCALL PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) { @@ -1681,14 +1724,21 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } /* not reached */ } +/* LCOV_EXCL_STOP */ static int PTRCALL -PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, +PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (ptr1 == end1) - return 0; + if (end1 - ptr1 < MINBPC(enc)) { + /* This line cannot be executed. THe incoming data has already + * been tokenized once, so imcomplete characters like this have + * already been eliminated from the input. Retaining the + * paranoia check is still valuable, however. + */ + return 0; /* LCOV_EXCL_LINE */ + } if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; } @@ -1744,7 +1794,7 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *end, POSITION *pos) { - while (ptr < end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -1760,7 +1810,7 @@ PREFIX(updatePosition)(const ENCODING *enc, case BT_CR: pos->lineNumber++; ptr += MINBPC(enc); - if (ptr != end && BYTE_TYPE(enc, ptr) == BT_LF) + if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); pos->columnNumber = (XML_Size)-1; break; diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index da0ea60a657d66..a6420f48eedc04 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -1,6 +1,33 @@ /* -Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd -See the file COPYING for copying permission. + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ enum { diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index c3b88fdf4e3eec..23d31e8e424916 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -1,8 +1,35 @@ -/* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd - See the file COPYING for copying permission. +/* This file is included! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 1997-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000-2017 Expat development team + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/* This file is included! */ #ifdef XML_TOK_NS_C const ENCODING * diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb7d17760..2efa998ff89aac 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -55,6 +55,9 @@ static struct { int fd; int all_threads; PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif } fatal_error = {0, NULL, -1, 0}; #ifdef FAULTHANDLER_LATER @@ -124,6 +127,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -369,8 +373,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) DWORD code = exc_info->ExceptionRecord->ExceptionCode; DWORD flags = exc_info->ExceptionRecord->ExceptionFlags; - /* only log fatal exceptions */ - if (flags & EXCEPTION_NONCONTINUABLE) { + /* bpo-30557: only log fatal exceptions */ + if (!(code & 0x80000000)) { /* call the next exception handler */ return EXCEPTION_CONTINUE_SEARCH; } @@ -387,8 +391,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: - PUTS(fd, "code "); - _Py_DumpDecimal(fd, code); + PUTS(fd, "code 0x"); + _Py_DumpHexadecimal(fd, code, 8); } PUTS(fd, "\n\n"); @@ -461,7 +465,8 @@ faulthandler_enable(void) } #ifdef MS_WINDOWS - AddVectoredExceptionHandler(1, faulthandler_exc_handler); + assert(fatal_error.exc_handler == NULL); + fatal_error.exc_handler = AddVectoredExceptionHandler(1, faulthandler_exc_handler); #endif return 0; } @@ -513,7 +518,12 @@ faulthandler_disable(void) faulthandler_disable_fatal_handler(handler); } } - +#ifdef MS_WINDOWS + if (fatal_error.exc_handler != NULL) { + RemoveVectoredExceptionHandler(fatal_error.exc_handler); + fatal_error.exc_handler = NULL; + } +#endif Py_CLEAR(fatal_error.file); } @@ -1310,7 +1320,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1376,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 0971a64fccbadc..5f941a26e1d54c 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -21,47 +21,47 @@ #endif /* XXX Only unix build process has been tested */ -#ifndef HGVERSION -#define HGVERSION "" +#ifndef GITVERSION +#define GITVERSION "" #endif -#ifndef HGTAG -#define HGTAG "" +#ifndef GITTAG +#define GITTAG "" #endif -#ifndef HGBRANCH -#define HGBRANCH "" +#ifndef GITBRANCH +#define GITBRANCH "" #endif const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(HGVERSION) + - ((sizeof(HGTAG) > sizeof(HGBRANCH)) ? - sizeof(HGTAG) : sizeof(HGBRANCH))]; - const char *revision = _Py_hgversion(); + static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; - const char *hgid = _Py_hgidentifier(); - if (!(*hgid)) - hgid = "default"; + const char *gitid = _Py_gitidentifier(); + if (!(*gitid)) + gitid = "default"; PyOS_snprintf(buildinfo, sizeof(buildinfo), - "%s%s%s, %.20s, %.9s", hgid, sep, revision, + "%s%s%s, %.20s, %.9s", gitid, sep, revision, DATE, TIME); return buildinfo; } const char * -_Py_hgversion(void) +_Py_gitversion(void) { - return HGVERSION; + return GITVERSION; } const char * -_Py_hgidentifier(void) +_Py_gitidentifier(void) { - const char *hgtag, *hgid; - hgtag = HGTAG; - if ((*hgtag) && strcmp(hgtag, "tip") != 0) - hgid = hgtag; + const char *gittag, *gitid; + gittag = GITTAG; + if ((*gittag) && strcmp(gittag, "undefined") != 0) + gitid = gittag; else - hgid = HGBRANCH; - return hgid; + gitid = GITBRANCH; + return gitid; } diff --git a/Modules/getpath.c b/Modules/getpath.c index 0f916436c515d6..dd3387a9d77f98 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -735,7 +735,7 @@ calculate_path(void) bufsz += wcslen(zip_path) + 1; bufsz += wcslen(exec_prefix) + 1; - buf = PyMem_New(wchar_t, bufsz); + buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { Py_FatalError( "Not enough memory for dynamic PYTHONPATH"); diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 9437ae7a5b1a14..f577fd3ab4ec3b 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -151,6 +151,7 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) if ((bytes = PyUnicode_EncodeFSDefault(name)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name_chars, NULL) == -1) goto out; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cbee31c35..3ff597b0a35774 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -72,10 +72,37 @@ groupby_traverse(groupbyobject *gbo, visitproc visit, void *arg) return 0; } +Py_LOCAL_INLINE(int) +groupby_step(groupbyobject *gbo) +{ + PyObject *newvalue, *newkey, *oldvalue; + + newvalue = PyIter_Next(gbo->it); + if (newvalue == NULL) + return -1; + + if (gbo->keyfunc == Py_None) { + newkey = newvalue; + Py_INCREF(newvalue); + } else { + newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); + if (newkey == NULL) { + Py_DECREF(newvalue); + return -1; + } + } + + oldvalue = gbo->currvalue; + gbo->currvalue = newvalue; + Py_XSETREF(gbo->currkey, newkey); + Py_XDECREF(oldvalue); + return 0; +} + static PyObject * groupby_next(groupbyobject *gbo) { - PyObject *newvalue, *newkey, *r, *grouper; + PyObject *r, *grouper; /* skip to next iteration group */ for (;;) { @@ -93,25 +120,9 @@ groupby_next(groupbyobject *gbo) break; } - newvalue = PyIter_Next(gbo->it); - if (newvalue == NULL) + if (groupby_step(gbo) < 0) return NULL; - - if (gbo->keyfunc == Py_None) { - newkey = newvalue; - Py_INCREF(newvalue); - } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); - if (newkey == NULL) { - Py_DECREF(newvalue); - return NULL; - } - } - - Py_XSETREF(gbo->currkey, newkey); - Py_XSETREF(gbo->currvalue, newvalue); } - Py_INCREF(gbo->currkey); Py_XSETREF(gbo->tgtkey, gbo->currkey); @@ -174,8 +185,9 @@ static PyMethodDef groupby_methods[] = { }; PyDoc_STRVAR(groupby_doc, -"groupby(iterable[, keyfunc]) -> create an iterator which returns\n\ -(key, sub-iterator) grouped by each value of key(value).\n"); +"groupby(iterable, key=None) -> make an iterator that returns consecutive\n\ +keys and groups from the iterable. If the key function is not specified or\n\ +is None, the element itself is used for grouping.\n"); static PyTypeObject groupby_type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -281,28 +293,12 @@ static PyObject * _grouper_next(_grouperobject *igo) { groupbyobject *gbo = (groupbyobject *)igo->parent; - PyObject *newvalue, *newkey, *r; + PyObject *r; int rcmp; if (gbo->currvalue == NULL) { - newvalue = PyIter_Next(gbo->it); - if (newvalue == NULL) + if (groupby_step(gbo) < 0) return NULL; - - if (gbo->keyfunc == Py_None) { - newkey = newvalue; - Py_INCREF(newvalue); - } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); - if (newkey == NULL) { - Py_DECREF(newvalue); - return NULL; - } - } - - assert(gbo->currkey == NULL); - gbo->currkey = newkey; - gbo->currvalue = newvalue; } assert(gbo->currkey != NULL); @@ -1864,33 +1860,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * diff --git a/Modules/main.c b/Modules/main.c index 2e6a60b1673f88..475a2fdc36db9b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ @@ -225,55 +225,60 @@ static int RunModule(wchar_t *modname, int set_argv0) return 0; } -static int -RunMainFromImporter(wchar_t *filename) +static PyObject * +AsImportPathEntry(wchar_t *filename) { - PyObject *argv0 = NULL, *importer, *sys_path, *sys_path0; - int sts; + PyObject *sys_path0 = NULL, *importer; - argv0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (argv0 == NULL) + sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + if (sys_path0 == NULL) goto error; - importer = PyImport_GetImporter(argv0); + importer = PyImport_GetImporter(sys_path0); if (importer == NULL) goto error; if (importer == Py_None) { - Py_DECREF(argv0); + Py_DECREF(sys_path0); Py_DECREF(importer); - return -1; + return NULL; } Py_DECREF(importer); + return sys_path0; + +error: + Py_XDECREF(sys_path0); + PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + PyErr_Print(); + PyErr_Clear(); + return NULL; +} + + +static int +RunMainFromImporter(PyObject *sys_path0) +{ + PyObject *sys_path; + int sts; - /* argv0 is usable as an import source, so put it in sys.path[0] - and import __main__ */ + /* Assume sys_path0 has already been checked by AsImportPathEntry, + * so put it in sys.path[0] and import __main__ */ sys_path = PySys_GetObject("path"); if (sys_path == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); goto error; } - sys_path0 = PyList_GetItem(sys_path, 0); - sts = 0; - if (!sys_path0) { - PyErr_Clear(); - sts = PyList_Append(sys_path, argv0); - } else if (PyObject_IsTrue(sys_path0)) { - sts = PyList_Insert(sys_path, 0, argv0); - } else { - sts = PyList_SetItem(sys_path, 0, argv0); - } + sts = PyList_Insert(sys_path, 0, sys_path0); if (sts) { - argv0 = NULL; + sys_path0 = NULL; goto error; } - Py_INCREF(argv0); sts = RunModule(L"__main__", 0); return sts != 0; error: - Py_XDECREF(argv0); + Py_XDECREF(sys_path0); PyErr_Print(); return 1; } @@ -358,6 +363,7 @@ Py_Main(int argc, wchar_t **argv) int saw_unbuffered_flag = 0; char *opt; PyCompilerFlags cf; + PyObject *main_importer_path = NULL; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -714,7 +720,17 @@ Py_Main(int argc, wchar_t **argv) argv[_PyOS_optind] = L"-m"; } - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + if (filename != NULL) { + main_importer_path = AsImportPathEntry(filename); + } + + if (main_importer_path != NULL) { + /* Let RunMainFromImporter adjust sys.path[0] later */ + PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + } else { + /* Use config settings to decide whether or not to update sys.path[0] */ + PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + } if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && isatty(fileno(stdin)) && @@ -744,11 +760,11 @@ Py_Main(int argc, wchar_t **argv) sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (main_importer_path != NULL) { + sts = RunMainFromImporter(main_importer_path); } - if (sts==-1 && filename!=NULL) { + if (sts==-1 && filename != NULL) { fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615ff82992c..426b7cacebb573 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Modules/nismodule.c b/Modules/nismodule.c index b6a855c5775c74..a9028bbfe54c6c 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -169,6 +169,7 @@ nis_match (PyObject *self, PyObject *args, PyObject *kwdict) return NULL; if ((bkey = PyUnicode_EncodeFSDefault(ukey)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bkey, &key, &keylen) == -1) { Py_DECREF(bkey); return NULL; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 0aa8657898da13..17581e0a80d0d7 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1151,7 +1151,7 @@ ConnectPipe(OverlappedObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "U", &AddressObj)) return NULL; - Address = PyUnicode_AsWideCharString(AddressObj, NULL); + Address = _PyUnicode_AsWideCharString(AddressObj); if (Address == NULL) return NULL; diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index b2566951d06de1..a4443350ef1cdc 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -698,7 +698,7 @@ validate_node(node *tree) short a_label = dfa_state->s_arc[arc].a_lbl; assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); if (_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) { - /* The child is acceptable; if non-terminal, validate it recursively. */ + /* The child is acceptable; if non-terminal, validate it recursively. */ if (ISNONTERMINAL(ch_type) && !validate_node(ch)) return 0; @@ -776,32 +776,35 @@ parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) */ tree = build_node_tree(tuple); if (tree != 0) { - node *validation_root = tree; + node *validation_root = NULL; int tree_type = 0; switch (TYPE(tree)) { case eval_input: /* Might be an eval form. */ tree_type = PyST_EXPR; + validation_root = tree; break; case encoding_decl: /* This looks like an encoding_decl so far. */ - if (NCH(tree) != 1) + if (NCH(tree) == 1) { + tree_type = PyST_SUITE; + validation_root = CHILD(tree, 0); + } + else { err_string("Error Parsing encoding_decl"); - validation_root = CHILD(tree, 0); - /* Fall through */ + } + break; case file_input: /* This looks like an exec form so far. */ - tree_type = PyST_SUITE; + validation_root = tree; break; default: /* This is a fragment, at best. */ - PyNode_Free(tree); err_string("parse tree does not use a valid start symbol"); - return (0); } - if (validate_node(validation_root)) + if (validation_root != NULL && validate_node(validation_root)) st = parser_newstobject(tree, tree_type); else PyNode_Free(tree); @@ -831,6 +834,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_ssize_t i; int err; + if (len < 0) { + return NULL; + } for (i = 1; i < len; ++i) { /* elem must always be a sequence, however simple */ PyObject* elem = PySequence_GetItem(tuple, i); @@ -851,7 +857,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if (type == -1 && PyErr_Occurred()) { Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } } Py_DECREF(temp); @@ -863,7 +869,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) PyErr_SetObject(parser_error, err); Py_XDECREF(err); Py_XDECREF(elem); - return (0); + return NULL; } if (ISTERMINAL(type)) { Py_ssize_t len = PyObject_Size(elem); @@ -872,11 +878,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if ((len != 2) && (len != 3)) { err_string("terminal nodes must have 2 or 3 entries"); - return 0; + Py_DECREF(elem); + return NULL; } temp = PySequence_GetItem(elem, 1); - if (temp == NULL) - return 0; + if (temp == NULL) { + Py_DECREF(elem); + return NULL; + } if (!PyUnicode_Check(temp)) { PyErr_Format(parser_error, "second item in terminal node must be a string," @@ -884,46 +893,49 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_TYPE(temp)->tp_name); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } if (len == 3) { PyObject *o = PySequence_GetItem(elem, 2); - if (o != NULL) { - if (PyLong_Check(o)) { - int num = _PyLong_AsInt(o); - if (num == -1 && PyErr_Occurred()) { - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return 0; - } - *line_num = num; - } - else { - PyErr_Format(parser_error, - "third item in terminal node must be an" - " integer, found %s", - Py_TYPE(temp)->tp_name); + if (o == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (PyLong_Check(o)) { + int num = _PyLong_AsInt(o); + if (num == -1 && PyErr_Occurred()) { Py_DECREF(o); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } + *line_num = num; + } + else { + PyErr_Format(parser_error, + "third item in terminal node must be an" + " integer, found %s", + Py_TYPE(temp)->tp_name); Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; } + Py_DECREF(o); } temp_str = PyUnicode_AsUTF8AndSize(temp, &len); if (temp_str == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); - return 0; + Py_DECREF(elem); + return NULL; } strn = (char *)PyObject_MALLOC(len + 1); if (strn == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); + Py_DECREF(elem); PyErr_NoMemory(); - return 0; + return NULL; } (void) memcpy(strn, temp_str, len + 1); Py_DECREF(temp); @@ -933,20 +945,21 @@ build_node_children(PyObject *tuple, node *root, int *line_num) * It has to be one or the other; this is an error. * Raise an exception. */ - PyObject *err = Py_BuildValue("os", elem, "unknown node type."); + PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } err = PyNode_AddChild(root, type, strn, *line_num, 0); if (err == E_NOMEM) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); - return (node *) PyErr_NoMemory(); + PyErr_NoMemory(); + return NULL; } if (err == E_OVERFLOW) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); PyErr_SetString(PyExc_ValueError, "unsupported number of child nodes"); @@ -957,14 +970,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) node* new_child = CHILD(root, i - 1); if (new_child != build_node_children(elem, new_child, line_num)) { - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } } else if (type == NEWLINE) { /* It's true: we increment the */ ++(*line_num); /* line number *after* the newline! */ } - Py_XDECREF(elem); + Py_DECREF(elem); } return root; } @@ -999,10 +1012,23 @@ build_node_tree(PyObject *tuple) if (num == encoding_decl) { encoding = PySequence_GetItem(tuple, 2); + if (encoding == NULL) { + PyErr_SetString(parser_error, "missed encoding"); + return NULL; + } + if (!PyUnicode_Check(encoding)) { + PyErr_Format(parser_error, + "encoding must be a string, found %.200s", + Py_TYPE(encoding)->tp_name); + Py_DECREF(encoding); + return NULL; + } /* tuple isn't borrowed anymore here, need to DECREF */ tuple = PySequence_GetSlice(tuple, 0, 2); - if (tuple == NULL) + if (tuple == NULL) { + Py_DECREF(encoding); return NULL; + } } res = PyNode_New(num); if (res != NULL) { @@ -1015,31 +1041,33 @@ build_node_tree(PyObject *tuple) const char *temp; temp = PyUnicode_AsUTF8AndSize(encoding, &len); if (temp == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); return NULL; } res->n_str = (char *)PyObject_MALLOC(len + 1); if (res->n_str == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); PyErr_NoMemory(); return NULL; } (void) memcpy(res->n_str, temp, len + 1); - Py_DECREF(encoding); - Py_DECREF(tuple); } } + if (encoding != NULL) { + Py_DECREF(encoding); + Py_DECREF(tuple); + } } else { /* The tuple is illegal -- if the number is neither TERMINAL nor * NONTERMINAL, we can't use it. Not sure the implementation * allows this condition, but the API doesn't preclude it. */ - PyObject *err = Py_BuildValue("os", tuple, + PyObject *err = Py_BuildValue("Os", tuple, "Illegal component tuple."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); @@ -1074,7 +1102,6 @@ parser__pickler(PyObject *self, PyObject *args) result = Py_BuildValue("O(O)", pickle_constructor, tuple); Py_DECREF(tuple); } - Py_DECREF(empty_dict); Py_DECREF(newargs); } finally: diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba255ec48f6..ee27fa4dec6f4e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -55,6 +55,11 @@ corresponding Unix manual entries for more information on calls."); #include #endif +#ifdef HAVE_SYS_SYSMACROS_H +/* GNU C Library: major(), minor(), makedev() */ +#include +#endif + #ifdef HAVE_SYS_TYPES_H #include #endif /* HAVE_SYS_TYPES_H */ @@ -949,6 +954,8 @@ path_converter(PyObject *o, void *p) Py_INCREF(bytes); } else if (is_buffer) { + /* XXX Replace PyObject_CheckBuffer with PyBytes_Check in other code + after removing suport of non-bytes buffer objects. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", @@ -1932,12 +1939,8 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT - PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); -#else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); -#endif + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLongLong(st->st_ino)); #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); #else @@ -1951,12 +1954,8 @@ _pystat_fromstructstat(STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, 4, _PyLong_FromUid(st->st_uid)); PyStructSequence_SET_ITEM(v, 5, _PyLong_FromGid(st->st_gid)); #endif -#ifdef HAVE_LARGEFILE_SUPPORT - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long)st->st_size)); -#else - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong(st->st_size)); -#endif + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(st->st_size)); + PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong(st->st_size)); #if defined(HAVE_STAT_TV_NSEC) ansec = st->st_atim.tv_nsec; @@ -3509,8 +3508,8 @@ _posix_listdir(path_t *path, PyObject *list) const char *name; if (path->narrow) { name = path->narrow; - /* only return bytes if they specified a bytes object */ - return_str = !(PyBytes_Check(path->object)); + /* only return bytes if they specified a bytes-like object */ + return_str = !PyObject_CheckBuffer(path->object); } else { name = "."; @@ -3678,7 +3677,7 @@ os__getfinalpathname_impl(PyObject *module, PyObject *path) PyObject *result; const wchar_t *path_wchar; - path_wchar = PyUnicode_AsUnicode(path); + path_wchar = _PyUnicode_AsUnicode(path); if (path_wchar == NULL) return NULL; @@ -4815,6 +4814,16 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) Py_DECREF(key2); goto error; } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key2) == 0 || + PyUnicode_FindChar(key2, '=', 1, PyUnicode_GET_LENGTH(key2), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + Py_DECREF(key2); + Py_DECREF(val2); + goto error; + } keyval = PyUnicode_FromFormat("%U=%U", key2, val2); #else if (!PyUnicode_FSConverter(key, &key2)) @@ -4823,6 +4832,14 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) Py_DECREF(key2); goto error; } + if (PyBytes_GET_SIZE(key2) == 0 || + strchr(PyBytes_AS_STRING(key2) + 1, '=') != NULL) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + Py_DECREF(key2); + Py_DECREF(val2); + goto error; + } keyval = PyBytes_FromFormat("%s=%s", PyBytes_AS_STRING(key2), PyBytes_AS_STRING(val2)); #endif @@ -5086,7 +5103,7 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) return NULL; } if (i == 0 && !argvlist[0][0]) { - free_string_array(argvlist, i); + free_string_array(argvlist, i + 1); PyErr_SetString( PyExc_ValueError, "spawnv() arg 2 first element cannot be empty"); @@ -5187,7 +5204,7 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, goto fail_1; } if (i == 0 && !argvlist[0][0]) { - lastarg = i; + lastarg = i + 1; PyErr_SetString( PyExc_ValueError, "spawnv() arg 2 first element cannot be empty"); @@ -6648,7 +6665,7 @@ static PyObject * os_setgroups(PyObject *module, PyObject *groups) /*[clinic end generated code: output=3fcb32aad58c5ecd input=fa742ca3daf85a7e]*/ { - int i, len; + Py_ssize_t i, len; gid_t grouplist[MAX_GROUPS]; if (!PySequence_Check(groups)) { @@ -6656,6 +6673,9 @@ os_setgroups(PyObject *module, PyObject *groups) return NULL; } len = PySequence_Size(groups); + if (len < 0) { + return NULL; + } if (len > MAX_GROUPS) { PyErr_SetString(PyExc_ValueError, "too many groups"); return NULL; @@ -7065,7 +7085,7 @@ win_readlink(PyObject *self, PyObject *args, PyObject *kwargs) )) return NULL; - path = PyUnicode_AsUnicode(po); + path = _PyUnicode_AsUnicode(po); if (path == NULL) return NULL; @@ -7884,9 +7904,9 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) static Py_ssize_t -iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { - int i, j; + Py_ssize_t i, j; Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); @@ -7963,8 +7983,7 @@ static Py_ssize_t os_readv_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=792da062d3fcebdb input=e679eb5dbfa0357d]*/ { - int cnt; - Py_ssize_t n; + Py_ssize_t cnt, n; int async_err = 0; struct iovec *iov; Py_buffer *buf; @@ -7976,6 +7995,8 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; @@ -8114,15 +8135,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() headers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.hdr_cnt = PySequence_Size(headers); - if (sf.hdr_cnt > 0 && - (i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(headers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); return NULL; + } + if (i > 0) { + sf.hdr_cnt = (int)i; + i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } if (trailers != NULL) { @@ -8131,15 +8161,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() trailers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.trl_cnt = PySequence_Size(trailers); - if (sf.trl_cnt > 0 && - (i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(trailers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() trailer is too large"); return NULL; + } + if (i > 0) { + sf.trl_cnt = (int)i; + i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } @@ -8409,7 +8448,7 @@ static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ { - int cnt; + Py_ssize_t cnt; Py_ssize_t result; int async_err = 0; struct iovec *iov; @@ -8421,6 +8460,8 @@ os_writev_impl(PyObject *module, int fd, PyObject *buffers) return -1; } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) { return -1; @@ -8837,22 +8878,35 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic end generated code: output=d29a567d6b2327d2 input=ba586581c2e6105f]*/ { const wchar_t *env; + Py_ssize_t size; + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(name) == 0 || + PyUnicode_FindChar(name, '=', 1, PyUnicode_GET_LENGTH(name), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + return NULL; + } PyObject *unicode = PyUnicode_FromFormat("%U=%U", name, value); if (unicode == NULL) { - PyErr_NoMemory(); return NULL; } - if (_MAX_ENV < PyUnicode_GET_LENGTH(unicode)) { + + env = PyUnicode_AsUnicodeAndSize(unicode, &size); + if (env == NULL) + goto error; + if (size > _MAX_ENV) { PyErr_Format(PyExc_ValueError, "the environment variable is longer than %u characters", _MAX_ENV); goto error; } - - env = PyUnicode_AsUnicode(unicode); - if (env == NULL) + if (wcslen(env) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); goto error; + } + if (_wputenv(env)) { posix_error(); goto error; @@ -8882,12 +8936,15 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) { PyObject *bytes = NULL; char *env; - const char *name_string = PyBytes_AsString(name); - const char *value_string = PyBytes_AsString(value); + const char *name_string = PyBytes_AS_STRING(name); + const char *value_string = PyBytes_AS_STRING(value); + if (strchr(name_string, '=') != NULL) { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + return NULL; + } bytes = PyBytes_FromFormat("%s=%s", name_string, value_string); if (bytes == NULL) { - PyErr_NoMemory(); return NULL; } @@ -10976,9 +11033,22 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS - SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - ncpu = sysinfo.dwNumberOfProcessors; + /* Vista is supported and the GetMaximumProcessorCount API is Win7+ + Need to fallback to Vista behavior if this call isn't present */ + HINSTANCE hKernel32; + hKernel32 = GetModuleHandleW(L"KERNEL32"); + + static DWORD(CALLBACK *_GetMaximumProcessorCount)(WORD) = NULL; + *(FARPROC*)&_GetMaximumProcessorCount = GetProcAddress(hKernel32, + "GetMaximumProcessorCount"); + if (_GetMaximumProcessorCount != NULL) { + ncpu = _GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS); + } + else { + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + ncpu = sysinfo.dwNumberOfProcessors; + } #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) @@ -11156,7 +11226,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,13 +11489,11 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ -#ifdef HAVE_LARGEFILE_SUPPORT - return PyLong_FromLongLong((long long)self->d_ino); -#else - return PyLong_FromLong((long)self->d_ino); -#endif + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->d_ino)); + return PyLong_FromUnsignedLongLong(self->d_ino); #endif } @@ -11647,7 +11715,7 @@ DirEntry_from_posix_info(path_t *path, const char *name, Py_ssize_t name_len, if (!joined_path) goto error; - if (!path->narrow || !PyBytes_Check(path->object)) { + if (!path->narrow || !PyObject_CheckBuffer(path->object)) { entry->name = PyUnicode_DecodeFSDefaultAndSize(name, name_len); entry->path = PyUnicode_DecodeFSDefault(joined_path); } @@ -12729,7 +12797,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif #ifdef SCHED_SPORADIC - if (PyModule_AddIntMacro(m, SCHED_SPORADIC) return -1; + if (PyModule_AddIntMacro(m, SCHED_SPORADIC)) return -1; #endif #ifdef SCHED_BATCH if (PyModule_AddIntMacro(m, SCHED_BATCH)) return -1; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index 784e9d0349fa79..bbef2de9c52253 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -158,6 +158,7 @@ pwd_getpwnam_impl(PyObject *module, PyObject *arg) if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getpwnam(name)) == NULL) { diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d57aa114c..47c3e86c209d9c 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,10 +1191,8 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) - /* This feature was added upstream in libexpat 2.1.0. Our expat copy - * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT - * to indicate that we can still use it. */ +#if XML_COMBINED_VERSION >= 20100 + /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); #endif diff --git a/Modules/readline.c b/Modules/readline.c index 3b94d4a0d1d759..575479cff249be 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1347,15 +1347,17 @@ call_readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) if (should_auto_add_history && n > 0) { const char *line; int length = _py_get_history_length(); - if (length > 0) + if (length > 0) { + HIST_ENTRY *hist_ent; #ifdef __APPLE__ if (using_libedit_emulation) { /* handle older 0-based or newer 1-based indexing */ - line = (const char *)history_get(length + libedit_history_start - 1)->line; + hist_ent = history_get(length + libedit_history_start - 1); } else #endif /* __APPLE__ */ - line = (const char *)history_get(length)->line; - else + hist_ent = history_get(length); + line = hist_ent ? hist_ent->line : ""; + } else line = ""; if (strcmp(p, line)) add_history(p); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 47da49399f737d..07a20c85d76dab 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -68,8 +68,8 @@ typedef struct { static void reap_obj(pylist fd2obj[FD_SETSIZE + 1]) { - int i; - for (i = 0; i < FD_SETSIZE + 1 && fd2obj[i].sentinel >= 0; i++) { + unsigned int i; + for (i = 0; i < (unsigned int)FD_SETSIZE + 1 && fd2obj[i].sentinel >= 0; i++) { Py_CLEAR(fd2obj[i].obj); } fd2obj[0].sentinel = -1; @@ -83,7 +83,7 @@ static int seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) { int max = -1; - int index = 0; + unsigned int index = 0; Py_ssize_t i; PyObject* fast_seq = NULL; PyObject* o = NULL; @@ -120,7 +120,7 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) FD_SET(v, set); /* add object and its file descriptor to the list */ - if (index >= FD_SETSIZE) { + if (index >= (unsigned int)FD_SETSIZE) { PyErr_SetString(PyExc_ValueError, "too many file descriptors in select()"); goto finally; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index e27075b2001946..0e6099fd361cb5 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -93,7 +93,7 @@ static pid_t main_pid; #endif static volatile struct { - sig_atomic_t tripped; + _Py_atomic_int tripped; PyObject *func; } Handlers[NSIG]; @@ -113,7 +113,7 @@ static volatile sig_atomic_t wakeup_fd = -1; #endif /* Speed up sigcheck() when none tripped */ -static volatile sig_atomic_t is_tripped = 0; +static _Py_atomic_int is_tripped; static PyObject *DefaultHandler; static PyObject *IgnoreHandler; @@ -139,6 +139,10 @@ timeval_from_double(double d, struct timeval *tv) { tv->tv_sec = floor(d); tv->tv_usec = fmod(d, 1.0) * 1000000.0; + /* Don't disable the timer if the computation above rounds down to zero. */ + if (d > 0.0 && tv->tv_sec == 0 && tv->tv_usec == 0) { + tv->tv_usec = 1; + } } Py_LOCAL_INLINE(double) @@ -188,12 +192,6 @@ The default handler for SIGINT installed by Python.\n\ It raises KeyboardInterrupt."); -static int -checksignals_witharg(void * unused) -{ - return PyErr_CheckSignals(); -} - static int report_wakeup_write_error(void *data) { @@ -242,7 +240,33 @@ trip_signal(int sig_num) int fd; Py_ssize_t rc; - Handlers[sig_num].tripped = 1; + _Py_atomic_store_relaxed(&Handlers[sig_num].tripped, 1); + + /* Set is_tripped after setting .tripped, as it gets + cleared in PyErr_CheckSignals() before .tripped. */ + _Py_atomic_store(&is_tripped, 1); + + /* Notify ceval.c */ + _PyEval_SignalReceived(); + + /* And then write to the wakeup fd *after* setting all the globals and + doing the _PyEval_SignalReceived. We used to write to the wakeup fd + and then set the flag, but this allowed the following sequence of events + (especially on windows, where trip_signal may run in a new thread): + + - main thread blocks on select([wakeup_fd], ...) + - signal arrives + - trip_signal writes to the wakeup fd + - the main thread wakes up + - the main thread checks the signal flags, sees that they're unset + - the main thread empties the wakeup fd + - the main thread goes back to sleep + - trip_signal sets the flags to request the Python-level signal handler + be run + - the main thread doesn't notice, because it's asleep + + See bpo-30038 for more details. + */ #ifdef MS_WINDOWS fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); @@ -263,6 +287,8 @@ trip_signal(int sig_num) wakeup.send_err_set = 1; wakeup.send_errno = errno; wakeup.send_win_error = GetLastError(); + /* Py_AddPendingCall() isn't signal-safe, but we + still use it for this exceptional case. */ Py_AddPendingCall(report_wakeup_send_error, NULL); } } @@ -276,18 +302,13 @@ trip_signal(int sig_num) rc = _Py_write_noraise(fd, &byte, 1); if (rc < 0) { + /* Py_AddPendingCall() isn't signal-safe, but we + still use it for this exceptional case. */ Py_AddPendingCall(report_wakeup_write_error, (void *)(intptr_t)errno); } } } - - if (!is_tripped) { - /* Set is_tripped after setting .tripped, as it gets - cleared in PyErr_CheckSignals() before .tripped. */ - is_tripped = 1; - Py_AddPendingCall(checksignals_witharg, NULL); - } } static void @@ -446,7 +467,7 @@ signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) return NULL; } old_handler = Handlers[signalnum].func; - Handlers[signalnum].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[signalnum].tripped, 0); Py_INCREF(handler); Handlers[signalnum].func = handler; if (old_handler != NULL) @@ -1245,11 +1266,11 @@ PyInit__signal(void) goto finally; Py_INCREF(IntHandler); - Handlers[0].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[0].tripped, 0); for (i = 1; i < NSIG; i++) { void (*t)(int); t = PyOS_getsig(i); - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); if (t == SIG_DFL) Handlers[i].func = DefaultHandler; else if (t == SIG_IGN) @@ -1473,7 +1494,7 @@ finisignal(void) for (i = 1; i < NSIG; i++) { func = Handlers[i].func; - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); Handlers[i].func = NULL; if (i != SIGINT && func != NULL && func != Py_None && func != DefaultHandler && func != IgnoreHandler) @@ -1494,7 +1515,7 @@ PyErr_CheckSignals(void) int i; PyObject *f; - if (!is_tripped) + if (!_Py_atomic_load(&is_tripped)) return 0; #ifdef WITH_THREAD @@ -1516,24 +1537,26 @@ PyErr_CheckSignals(void) * we receive a signal i after we zero is_tripped and before we * check Handlers[i].tripped. */ - is_tripped = 0; + _Py_atomic_store(&is_tripped, 0); if (!(f = (PyObject *)PyEval_GetFrame())) f = Py_None; for (i = 1; i < NSIG; i++) { - if (Handlers[i].tripped) { + if (_Py_atomic_load_relaxed(&Handlers[i].tripped)) { PyObject *result = NULL; PyObject *arglist = Py_BuildValue("(iO)", i, f); - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); if (arglist) { result = PyEval_CallObject(Handlers[i].func, arglist); Py_DECREF(arglist); } - if (!result) + if (!result) { + _Py_atomic_store(&is_tripped, 1); return -1; + } Py_DECREF(result); } @@ -1570,12 +1593,12 @@ PyOS_FiniInterrupts(void) int PyOS_InterruptOccurred(void) { - if (Handlers[SIGINT].tripped) { + if (_Py_atomic_load_relaxed(&Handlers[SIGINT].tripped)) { #ifdef WITH_THREAD if (PyThread_get_thread_ident() != main_thread) return 0; #endif - Handlers[SIGINT].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[SIGINT].tripped, 0); return 1; } return 0; @@ -1585,11 +1608,11 @@ static void _clear_pending_signals(void) { int i; - if (!is_tripped) + if (!_Py_atomic_load(&is_tripped)) return; - is_tripped = 0; + _Py_atomic_store(&is_tripped, 0); for (i = 1; i < NSIG; ++i) { - Handlers[i].tripped = 0; + _Py_atomic_store_relaxed(&Handlers[i].tripped, 0); } } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc062fd6539..a351faa7f9fabf 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ if_indextoname(index) -- return the corresponding interface name\n\ #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" @@ -1242,9 +1212,9 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) { struct sockaddr_un *a = (struct sockaddr_un *) addr; #ifdef __linux__ - if (a->sun_path[0] == 0) { /* Linux abstract namespace */ - addrlen -= offsetof(struct sockaddr_un, sun_path); - return PyBytes_FromStringAndSize(a->sun_path, addrlen); + size_t linuxaddrlen = addrlen - offsetof(struct sockaddr_un, sun_path); + if (linuxaddrlen > 0 && a->sun_path[0] == 0) { /* Linux abstract namespace */ + return PyBytes_FromStringAndSize(a->sun_path, linuxaddrlen); } else #endif /* linux */ @@ -1494,7 +1464,10 @@ idna_converter(PyObject *obj, struct maybe_idna *data) len = PyByteArray_Size(obj); } else if (PyUnicode_Check(obj)) { - if (PyUnicode_READY(obj) == 0 && PyUnicode_IS_COMPACT_ASCII(obj)) { + if (PyUnicode_READY(obj) == -1) { + return 0; + } + if (PyUnicode_IS_COMPACT_ASCII(obj)) { data->buf = PyUnicode_DATA(obj); len = PyUnicode_GET_LENGTH(obj); } @@ -2734,7 +2707,9 @@ sock_close(PySocketSockObject *s) Py_BEGIN_ALLOW_THREADS res = SOCKETCLOSE(fd); Py_END_ALLOW_THREADS - if (res < 0) { + /* bpo-30319: The peer can already have closed the connection. + Python ignores ECONNRESET on close(). */ + if (res < 0 && errno != ECONNRESET) { return s->errorhandler(); } } diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927e0b35f0..03f982b91083f3 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 556a715a242482..1601ec0f2fc4e7 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -134,6 +134,7 @@ spwd_getspnam_impl(PyObject *module, PyObject *arg) if ((bytes = PyUnicode_EncodeFSDefault(arg)) == NULL) return NULL; + /* check for embedded null bytes */ if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getspnam(name)) == NULL) { diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ebd44ad525bd81..d6c34b9225c1b9 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -279,7 +279,7 @@ static PyTypeObject StructTimeType; static PyObject * tmtotuple(struct tm *p #ifndef HAVE_STRUCT_TM_TM_ZONE - , const char *zone, int gmtoff + , const char *zone, time_t gmtoff #endif ) { @@ -305,7 +305,7 @@ tmtotuple(struct tm *p #else PyStructSequence_SET_ITEM(v, 9, PyUnicode_DecodeLocale(zone, "surrogateescape")); - SET(10, gmtoff); + PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { @@ -397,7 +397,7 @@ time_localtime(PyObject *self, PyObject *args) { struct tm local = buf; char zone[100]; - int gmtoff; + time_t gmtoff; strftime(zone, sizeof(zone), "%Z", &buf); gmtoff = timegm(&buf) - when; return tmtotuple(&local, zone, gmtoff); @@ -607,7 +607,7 @@ time_strftime(PyObject *self, PyObject *args) buf.tm_isdst = 1; #ifdef HAVE_WCSFTIME - format = PyUnicode_AsWideCharString(format_arg, NULL); + format = _PyUnicode_AsWideCharString(format_arg); if (format == NULL) return NULL; fmt = format; diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aaae41f51..5bf31cc0605451 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -580,7 +580,8 @@ zipimporter_get_data(PyObject *obj, PyObject *args) return NULL; #ifdef ALTSEP - path = _PyObject_CallMethodId(path, &PyId_replace, "CC", ALTSEP, SEP); + path = _PyObject_CallMethodId((PyObject *)&PyUnicode_Type, &PyId_replace, + "OCC", path, ALTSEP, SEP); if (!path) return NULL; #else @@ -1102,7 +1103,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1262,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1404,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/Objects/abstract.c b/Objects/abstract.c index d838856d452872..6c6c86cd44a9bc 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -809,6 +809,21 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) PyObject *result = binary_op1(v, w, op_slot); if (result == Py_NotImplemented) { Py_DECREF(result); + + if (op_slot == NB_SLOT(nb_rshift) && + PyCFunction_Check(v) && + strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) + { + PyErr_Format(PyExc_TypeError, + "unsupported operand type(s) for %.100s: " + "'%.100s' and '%.100s'. Did you mean \"print(, " + "file=)\"?", + op_name, + v->ob_type->tp_name, + w->ob_type->tp_name); + return NULL; + } + return binop_type_error(v, w, op_name); } return result; @@ -2389,9 +2404,9 @@ _PyStack_AsDict(PyObject **values, PyObject *kwnames) return kwdict; } -PyObject ** +int _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **p_kwnames, PyObject *func) + PyObject ***p_stack, PyObject **p_kwnames) { PyObject **stack, **kwstack; Py_ssize_t nkwargs; @@ -2402,27 +2417,27 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, assert(nargs >= 0); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - nkwargs = (kwargs != NULL) ? PyDict_Size(kwargs) : 0; - if (!nkwargs) { + if (kwargs == NULL || (nkwargs = PyDict_Size(kwargs)) == 0) { + *p_stack = args; *p_kwnames = NULL; - return args; + return 0; } if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { PyErr_NoMemory(); - return NULL; + return -1; } stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); if (stack == NULL) { PyErr_NoMemory(); - return NULL; + return -1; } kwnames = PyTuple_New(nkwargs); if (kwnames == NULL) { PyMem_Free(stack); - return NULL; + return -1; } /* Copy position arguments (borrowed references) */ @@ -2441,8 +2456,9 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, i++; } + *p_stack = stack; *p_kwnames = kwnames; - return stack; + return 0; } PyObject * @@ -3192,8 +3208,8 @@ _PySequence_BytesToCharpArray(PyObject* self) array[i] = NULL; goto fail; } - data = PyBytes_AsString(item); - if (data == NULL) { + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(item, &data, NULL) < 0) { /* NULL terminate before freeing. */ array[i] = NULL; goto fail; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d69802508736..a9c8ca6f1f35a8 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6346fc53..625e242d563d89 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d484409600852..4c55294d9d8c8a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } @@ -619,11 +632,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); @@ -1425,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -1670,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d8a675f9fbfc6..d45be4c96796e6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -22,8 +22,7 @@ all_name_chars(PyObject *o) static const unsigned char *name_chars = (unsigned char *)NAME_CHARS; const unsigned char *s, *e; - if (!PyUnicode_Check(o) || PyUnicode_READY(o) == -1 || - !PyUnicode_IS_ASCII(o)) + if (!PyUnicode_IS_ASCII(o)) return 0; if (ok_name_char[*name_chars] == 0) { @@ -64,6 +63,10 @@ intern_string_constants(PyObject *tuple) for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { + if (PyUnicode_READY(v) == -1) { + PyErr_Clear(); + continue; + } if (all_name_chars(v)) { PyObject *w = v; PyUnicode_InternInPlace(&v); @@ -411,18 +414,19 @@ static void code_dealloc(PyCodeObject *co) { if (co->co_extra != NULL) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); _PyCodeObjectExtra *co_extra = co->co_extra; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { - freefunc free_extra = tstate->co_extra_freefuncs[i]; + freefunc free_extra = state->co_extra_freefuncs[i]; if (free_extra != NULL) { free_extra(co_extra->ce_extras[i]); } } - PyMem_FREE(co->co_extra); + PyMem_Free(co_extra->ce_extras); + PyMem_Free(co_extra); } Py_XDECREF(co->co_code); @@ -446,11 +450,15 @@ code_dealloc(PyCodeObject *co) static PyObject * code_sizeof(PyCodeObject *co, void *unused) { - Py_ssize_t res; + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; - res = _PyObject_SIZE(Py_TYPE(co)); if (co->co_cell2arg != NULL && co->co_cellvars != NULL) - res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(unsigned char); + res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(Py_ssize_t); + + if (co_extra != NULL) + res += co_extra->ce_size * sizeof(co_extra->ce_extras[0]); + return PyLong_FromSsize_t(res); } @@ -821,8 +829,6 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds) int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { - assert(*extra == NULL); - if (!PyCode_Check(code)) { PyErr_BadInternalCall(); return -1; @@ -833,6 +839,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) if (co_extra == NULL || co_extra->ce_size <= index) { + *extra = NULL; return 0; } @@ -844,10 +851,10 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); if (!PyCode_Check(code) || index < 0 || - index >= tstate->co_extra_user_count) { + index >= state->co_extra_user_count) { PyErr_BadInternalCall(); return -1; } @@ -856,37 +863,48 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL) { - o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc( - sizeof(_PyCodeObjectExtra)); - if (o->co_extra == NULL) { + co_extra = PyMem_Malloc(sizeof(_PyCodeObjectExtra)); + if (co_extra == NULL) { return -1; } - co_extra = (_PyCodeObjectExtra *) o->co_extra; co_extra->ce_extras = PyMem_Malloc( - tstate->co_extra_user_count * sizeof(void*)); + state->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { + PyMem_Free(co_extra); return -1; } - co_extra->ce_size = tstate->co_extra_user_count; + co_extra->ce_size = state->co_extra_user_count; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; } + + o->co_extra = co_extra; } else if (co_extra->ce_size <= index) { - co_extra->ce_extras = PyMem_Realloc( - co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); + void** ce_extras = PyMem_Realloc( + co_extra->ce_extras, state->co_extra_user_count * sizeof(void*)); - if (co_extra->ce_extras == NULL) { + if (ce_extras == NULL) { return -1; } - co_extra->ce_size = tstate->co_extra_user_count; + for (Py_ssize_t i = co_extra->ce_size; + i < state->co_extra_user_count; + i++) { + ce_extras[i] = NULL; + } + + co_extra->ce_extras = ce_extras; + co_extra->ce_size = state->co_extra_user_count; + } - for (Py_ssize_t i = co_extra->ce_size; i < co_extra->ce_size; i++) { - co_extra->ce_extras[i] = NULL; + if (co_extra->ce_extras[index] != NULL) { + freefunc free = state->co_extra_freefuncs[index]; + if (free != NULL) { + free(co_extra->ce_extras[index]); } } diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31e12784cc34f9..cfaba688c684f6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1025,11 +1025,11 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } cr.real = PyFloat_AsDouble(tmp); - cr.imag = 0.0; /* Shut up compiler warning */ + cr.imag = 0.0; Py_DECREF(tmp); } if (i == NULL) { - ci.real = 0.0; + ci.real = cr.imag; } else if (PyComplex_Check(i)) { ci = ((PyComplexObject*)i)->cval; @@ -1051,7 +1051,7 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ci_is_complex) { cr.real -= ci.imag; } - if (cr_is_complex) { + if (cr_is_complex && i != NULL) { ci.real += cr.imag; } return complex_subtype_from_doubles(type, cr.real, ci.real); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7b403bcecc5da..690ef3bd2b3ae0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { @@ -676,7 +676,8 @@ Christian Tismer. lookdict() is general-purpose, and may return DKIX_ERROR if (and only if) a comparison raises an exception. lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return DKIX_ERROR. +never raise an exception; that function can never return DKIX_ERROR when key +is string. Otherwise, it falls back to lookdict(). lookdict_unicode_nodummy is further specialized for string keys that cannot be the value. For both, when the key isn't found a DKIX_EMPTY is returned. hashpos returns @@ -1114,18 +1115,18 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) PyDictKeyEntry *ep, *ep0; Py_ssize_t hashpos, ix; + Py_INCREF(key); + Py_INCREF(value); if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp) < 0) - return -1; + goto Fail; } ix = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr, &hashpos); - if (ix == DKIX_ERROR) { - return -1; - } + if (ix == DKIX_ERROR) + goto Fail; assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); - Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); /* When insertion order is different from shared key, we can't share @@ -1134,10 +1135,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (_PyDict_HasSplitTable(mp) && ((ix >= 0 && *value_addr == NULL && mp->ma_used != ix) || (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); ix = DKIX_EMPTY; } @@ -1146,16 +1145,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) /* Insert into new slot. */ if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); } ep0 = DK_ENTRIES(mp->ma_keys); ep = &ep0[mp->ma_keys->dk_nentries]; dk_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - Py_INCREF(key); ep->me_key = key; ep->me_hash = hash; if (mp->ma_values) { @@ -1183,6 +1179,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) assert(_PyDict_CheckConsistency(mp)); Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + Py_DECREF(key); return 0; } @@ -1193,7 +1190,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) mp->ma_used++; mp->ma_version_tag = DICT_NEXT_VERSION(); assert(_PyDict_CheckConsistency(mp)); + Py_DECREF(key); return 0; + +Fail: + Py_DECREF(value); + Py_DECREF(key); + return -1; } /* @@ -1928,7 +1931,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } @@ -2003,6 +2006,8 @@ dict_dealloc(PyDictObject *mp) PyObject **values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; + + /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(mp); Py_TRASHCAN_SAFE_BEGIN(mp) if (values != NULL) { @@ -2431,11 +2436,18 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) /* Update/merge with this (key, value) pair. */ key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); + Py_INCREF(key); + Py_INCREF(value); if (override || PyDict_GetItem(d, key) == NULL) { int status = PyDict_SetItem(d, key, value); - if (status < 0) + if (status < 0) { + Py_DECREF(key); + Py_DECREF(value); goto Fail; + } } + Py_DECREF(key); + Py_DECREF(value); Py_DECREF(fast); Py_DECREF(item); } @@ -2736,14 +2748,15 @@ dict_equal(PyDictObject *a, PyDictObject *b) bval = NULL; else bval = *vaddr; - Py_DECREF(key); if (bval == NULL) { + Py_DECREF(key); Py_DECREF(aval); if (PyErr_Occurred()) return -1; return 0; } cmp = PyObject_RichCompareBool(aval, bval, Py_EQ); + Py_DECREF(key); Py_DECREF(aval); if (cmp <= 0) /* error or not equal */ return cmp; @@ -3421,6 +3434,8 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) static void dictiter_dealloc(dictiterobject *di) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + _PyObject_GC_UNTRACK(di); Py_XDECREF(di->di_dict); Py_XDECREF(di->di_result); PyObject_GC_Del(di); @@ -3632,7 +3647,7 @@ PyTypeObject PyDictIterValue_Type = { static PyObject * dictiter_iternextitem(dictiterobject *di) { - PyObject *key, *value, *result = di->di_result; + PyObject *key, *value, *result; Py_ssize_t i, n; PyDictObject *d = di->di_dict; @@ -3673,20 +3688,25 @@ dictiter_iternextitem(dictiterobject *di) } di->di_pos = i+1; di->len--; - if (result->ob_refcnt == 1) { + Py_INCREF(key); + Py_INCREF(value); + result = di->di_result; + if (Py_REFCNT(result) == 1) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ Py_INCREF(result); - Py_DECREF(PyTuple_GET_ITEM(result, 0)); - Py_DECREF(PyTuple_GET_ITEM(result, 1)); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); } else { result = PyTuple_New(2); if (result == NULL) return NULL; + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -3784,6 +3804,8 @@ dictiter_reduce(dictiterobject *di) static void dictview_dealloc(_PyDictViewObject *dv) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + _PyObject_GC_UNTRACK(dv); Py_XDECREF(dv->dv_dict); PyObject_GC_Del(dv); } @@ -4179,6 +4201,7 @@ dictitems_iter(_PyDictViewObject *dv) static int dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { + int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) return 0; @@ -4192,7 +4215,10 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) return -1; return 0; } - return PyObject_RichCompareBool(value, found, Py_EQ); + Py_INCREF(found); + result = PyObject_RichCompareBool(value, found, Py_EQ); + Py_DECREF(found); + return result; } static PySequenceMethods dictitems_as_sequence = { @@ -4376,15 +4402,19 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, } if (value == NULL) { res = PyDict_DelItem(dict, key); - if (cached != ((PyDictObject *)dict)->ma_keys) { + // Since key sharing dict doesn't allow deletion, PyDict_DelItem() + // always converts dict to combined form. + if ((cached = CACHED_KEYS(tp)) != NULL) { CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } } else { - int was_shared = cached == ((PyDictObject *)dict)->ma_keys; + int was_shared = (cached == ((PyDictObject *)dict)->ma_keys); res = PyDict_SetItem(dict, key, value); - if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + if (was_shared && + (cached = CACHED_KEYS(tp)) != NULL && + cached != ((PyDictObject *)dict)->ma_keys) { /* PyDict_SetItem() may call dictresize and convert split table * into combined table. In such case, convert it to split * table again and update type's shared key only when this is diff --git a/Objects/enumobject.c b/Objects/enumobject.c index dae166d5adf468..b92b8987fbb0cd 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -87,19 +87,25 @@ enum_next_long(enumobject *en, PyObject* next_item) if (en->en_longindex == NULL) { en->en_longindex = PyLong_FromSsize_t(PY_SSIZE_T_MAX); - if (en->en_longindex == NULL) + if (en->en_longindex == NULL) { + Py_DECREF(next_item); return NULL; + } } if (one == NULL) { one = PyLong_FromLong(1); - if (one == NULL) + if (one == NULL) { + Py_DECREF(next_item); return NULL; + } } next_index = en->en_longindex; assert(next_index != NULL); stepped_up = PyNumber_Add(next_index, one); - if (stepped_up == NULL) + if (stepped_up == NULL) { + Py_DECREF(next_item); return NULL; + } en->en_longindex = stepped_up; if (result->ob_refcnt == 1) { diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a145bcb1..271e293e325e88 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; @@ -1308,11 +1356,16 @@ SyntaxError_init(PySyntaxErrorObject *self, PyObject *args, PyObject *kwds) Py_DECREF(info); - /* Issue #21669: Custom error for 'print' & 'exec' as statements */ - if (self->text && PyUnicode_Check(self->text)) { - if (_report_missing_parentheses(self) < 0) { - return -1; - } + /* + * Issue #21669: Custom error for 'print' & 'exec' as statements + * + * Only applies to SyntaxError instances, not to subclasses such + * as TabError or IndentationError (see issue #31161) + */ + if ((PyObject*)Py_TYPE(self) == PyExc_SyntaxError && + self->text && PyUnicode_Check(self->text) && + _report_missing_parentheses(self) < 0) { + return -1; } } return 0; @@ -2833,6 +2886,49 @@ _PyErr_TrySetFromCause(const char *format, ...) * or minus, using the stream redirection syntax). */ + +// Static helper for setting legacy print error message +static int +_set_legacy_print_statement_msg(PySyntaxErrorObject *self, Py_ssize_t start) +{ + PyObject *strip_sep_obj = PyUnicode_FromString(" \t\r\n"); + if (strip_sep_obj == NULL) + return -1; + + // PRINT_OFFSET is to remove `print ` word from the data. + const int PRINT_OFFSET = 6; + Py_ssize_t text_len = PyUnicode_GET_LENGTH(self->text); + PyObject *data = PyUnicode_Substring(self->text, PRINT_OFFSET, text_len); + + if (data == NULL) { + Py_DECREF(strip_sep_obj); + return -1; + } + PyObject *new_data = _PyUnicode_XStrip(data, 2, strip_sep_obj); + Py_DECREF(data); + Py_DECREF(strip_sep_obj); + + if (new_data == NULL) { + return -1; + } + // gets the modified text_len after stripping `print ` + text_len = PyUnicode_GET_LENGTH(new_data); + const char *maybe_end_arg = ""; + if (text_len > 0 && PyUnicode_READ_CHAR(new_data, text_len-1) == ',') { + maybe_end_arg = " end=\" \""; + } + PyObject *error_msg = PyUnicode_FromFormat( + "Missing parentheses in call to 'print'. Did you mean print(%U%s)?", + new_data, maybe_end_arg + ); + Py_DECREF(new_data); + if (error_msg == NULL) + return -1; + + Py_XSETREF(self->msg, error_msg); + return 1; +} + static int _check_for_legacy_statements(PySyntaxErrorObject *self, Py_ssize_t start) { @@ -2868,9 +2964,8 @@ _check_for_legacy_statements(PySyntaxErrorObject *self, Py_ssize_t start) } if (PyUnicode_Tailmatch(self->text, print_prefix, start, text_len, -1)) { - Py_XSETREF(self->msg, - PyUnicode_FromString("Missing parentheses in call to 'print'")); - return 1; + + return _set_legacy_print_statement_msg(self, start); } /* Check for legacy exec statements */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 80bf71efd21a4e..1f134aa113154b 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2183,12 +2183,14 @@ _PyFloat_Pack4(double x, unsigned char *p, int le) } else { float y = (float)x; - const unsigned char *s = (unsigned char*)&y; int i, incr = 1; if (Py_IS_INFINITY(y) && !Py_IS_INFINITY(x)) goto Overflow; + unsigned char s[sizeof(float)]; + memcpy(s, &y, sizeof(float)); + if ((float_format == ieee_little_endian_format && !le) || (float_format == ieee_big_endian_format && le)) { p += 3; @@ -2196,7 +2198,7 @@ _PyFloat_Pack4(double x, unsigned char *p, int le) } for (i = 0; i < 4; i++) { - *p = *s++; + *p = s[i]; p += incr; } return 0; diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0e129da4..1c29e296afe2a8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5efe5b0fe..547bdf0b95f1ee 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 515375772e6317..3dab2b98661695 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. diff --git a/Objects/longobject.c b/Objects/longobject.c index ad239ce84e237a..450087b5b15ffb 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2049,15 +2049,15 @@ long_from_binary_base(const char **str, int base, PyLongObject **res) } *str = p; - /* n <- # of Python digits needed, = ceiling(n/PyLong_SHIFT). */ - n = digits * bits_per_char + PyLong_SHIFT - 1; - if (n / bits_per_char < p - start) { + /* n <- the number of Python digits needed, + = ceiling((digits * bits_per_char) / PyLong_SHIFT). */ + if (digits > (PY_SSIZE_T_MAX - (PyLong_SHIFT - 1)) / bits_per_char) { PyErr_SetString(PyExc_ValueError, "int string too large to convert"); *res = NULL; return 0; } - n = n / PyLong_SHIFT; + n = (digits * bits_per_char + PyLong_SHIFT - 1) / PyLong_SHIFT; z = _PyLong_New(n); if (z == NULL) { *res = NULL; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2073d108..e1ac7281783f38 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c2001f0169b030..fe52545667f2ed 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -243,8 +243,7 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs, PyObject *kwnames; _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - stack = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames, func_obj); - if (stack == NULL) { + if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { return NULL; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 79be51a806b707..f357af2b2118f6 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -667,14 +667,11 @@ module_getattro(PyModuleObject *m, PyObject *name) if (m->md_dict) { _Py_IDENTIFIER(__name__); mod_name = _PyDict_GetItemId(m->md_dict, &PyId___name__); - if (mod_name) { + if (mod_name && PyUnicode_Check(mod_name)) { PyErr_Format(PyExc_AttributeError, "module '%U' has no attribute '%U'", mod_name, name); return NULL; } - else if (PyErr_Occurred()) { - PyErr_Clear(); - } } PyErr_Format(PyExc_AttributeError, "module has no attribute '%U'", name); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3b09ad9e..32e7ecbe1e0436 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND diff --git a/Objects/odictobject.c b/Objects/odictobject.c index f9f1bf362e7e4d..65f90e83434edd 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -875,8 +875,7 @@ odict_eq(PyObject *a, PyObject *b) PyDoc_STRVAR(odict_init__doc__, "Initialize an ordered dictionary. The signature is the same as\n\ - regular dictionaries, but keyword arguments are not recommended because\n\ - their insertion order is arbitrary.\n\ + regular dictionaries. Keyword argument order is preserved.\n\ \n\ "); @@ -1155,10 +1154,12 @@ _odict_popkey(PyObject *od, PyObject *key, PyObject *failobj) /* popitem() */ PyDoc_STRVAR(odict_popitem__doc__, -"od.popitem() -> (k, v), return and remove a (key, value) pair.\n\ - Pairs are returned in LIFO order if last is true or FIFO order if false.\n\ -\n\ - "); +"popitem($self, /, last=True)\n" +"--\n" +"\n" +"Remove and return a (key, value) pair from the dictionary.\n" +"\n" +"Pairs are returned in LIFO order if last is true or FIFO order if false."); static PyObject * odict_popitem(PyObject *od, PyObject *args, PyObject *kwargs) diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7a24d394..8f5fc434bd4955 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d3600d9fad..24272b4d14a505 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -557,6 +556,7 @@ set_dealloc(PySetObject *so) setentry *entry; Py_ssize_t used = so->used; + /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(so); Py_TRASHCAN_SAFE_BEGIN(so) if (so->weakreflist != NULL) @@ -646,8 +646,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -813,6 +813,8 @@ typedef struct { static void setiter_dealloc(setiterobject *si) { + /* bpo-31095: UnTrack is needed before calling any callbacks */ + _PyObject_GC_UNTRACK(si); Py_XDECREF(si->si_set); PyObject_GC_Del(si); } @@ -990,7 +992,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1513,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * @@ -1551,20 +1553,26 @@ set_difference(PySetObject *so, PyObject *other) PyObject *key; Py_hash_t hash; setentry *entry; - Py_ssize_t pos = 0; + Py_ssize_t pos = 0, other_size; int rv; if (PySet_GET_SIZE(so) == 0) { return set_copy(so); } - if (!PyAnySet_Check(other) && !PyDict_CheckExact(other)) { + if (PyAnySet_Check(other)) { + other_size = PySet_GET_SIZE(other); + } + else if (PyDict_CheckExact(other)) { + other_size = PyDict_Size(other); + } + else { return set_copy_and_difference(so, other); } /* If len(so) much more than len(other), it's more efficient to simply copy * so and then iterate other looking for common elements. */ - if ((PySet_GET_SIZE(so) >> 2) > PyObject_Size(other)) { + if ((PySet_GET_SIZE(so) >> 2) > other_size) { return set_copy_and_difference(so, other); } diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 6a690213d14518..4e4b32d32441a8 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -197,6 +197,8 @@ PySlice_Unpack(PyObject *_r, PySliceObject *r = (PySliceObject*)_r; /* this is harder to get right than you might think */ + Py_BUILD_ASSERT(PY_SSIZE_T_MIN + 1 <= -PY_SSIZE_T_MAX); + if (r->step == Py_None) { *step = 1; } @@ -217,14 +219,14 @@ PySlice_Unpack(PyObject *_r, } if (r->start == Py_None) { - *start = *step < 0 ? PY_SSIZE_T_MAX-1 : 0;; + *start = *step < 0 ? PY_SSIZE_T_MAX : 0; } else { if (!_PyEval_SliceIndex(r->start, start)) return -1; } if (r->stop == Py_None) { - *stop = *step < 0 ? -PY_SSIZE_T_MAX : PY_SSIZE_T_MAX; + *stop = *step < 0 ? PY_SSIZE_T_MIN : PY_SSIZE_T_MAX; } else { if (!_PyEval_SliceIndex(r->stop, stop)) return -1; @@ -258,7 +260,7 @@ PySlice_AdjustIndices(Py_ssize_t length, *stop = (step < 0) ? -1 : 0; } } - else if (*stop >= length) { + else if (*stop >= length) { *stop = (step < 0) ? length - 1 : length; } diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 43f2f3266fd778..f019d9a96bfba3 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -330,7 +330,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, case _Py_ERROR_REPLACE: memset(p, '?', endpos - startpos); p += (endpos - startpos); - /* fall through the ignore handler */ + /* fall through */ case _Py_ERROR_IGNORE: i += (endpos - startpos - 1); break; @@ -378,7 +378,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } startpos = k; assert(startpos < endpos); - /* fall through the default handler */ + /* fall through */ default: rep = unicode_encode_call_errorhandler( errors, &error_handler_obj, "utf-8", "surrogates not allowed", diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 14fa28ea54a5c8..7314af02a4a669 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -412,18 +412,22 @@ get_field_object(SubString *input, PyObject *args, PyObject *kwargs, if (index == -1) { /* look up in kwargs */ PyObject *key = SubString_new_object(&first); - if (key == NULL) + if (key == NULL) { goto error; - - /* Use PyObject_GetItem instead of PyDict_GetItem because this - code is no longer just used with kwargs. It might be passed - a non-dict when called through format_map. */ - if ((kwargs == NULL) || (obj = PyObject_GetItem(kwargs, key)) == NULL) { + } + if (kwargs == NULL) { PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); goto error; } + /* Use PyObject_GetItem instead of PyDict_GetItem because this + code is no longer just used with kwargs. It might be passed + a non-dict when called through format_map. */ + obj = PyObject_GetItem(kwargs, key); Py_DECREF(key); + if (obj == NULL) { + goto error; + } } else { /* If args is NULL, we have a format string with a positional field diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499e72c7dd..0dada74dc7c986 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 329261b037320a..cb5e235628dd74 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -22,9 +22,9 @@ #define MCACHE_HASH_METHOD(type, name) \ MCACHE_HASH((type)->tp_version_tag, \ ((PyASCIIObject *)(name))->hash) -#define MCACHE_CACHEABLE_NAME(name) \ - PyUnicode_CheckExact(name) && \ - PyUnicode_READY(name) != -1 && \ +#define MCACHE_CACHEABLE_NAME(name) \ + PyUnicode_CheckExact(name) && \ + PyUnicode_IS_READY(name) && \ PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE struct method_cache_entry { @@ -3075,6 +3075,7 @@ type_getattro(PyTypeObject *type, PyObject *name) static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { + int res; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format( PyExc_TypeError, @@ -3082,9 +3083,35 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) type->tp_name); return -1; } - if (PyObject_GenericSetAttr((PyObject *)type, name, value) < 0) - return -1; - return update_slot(type, name); + if (PyUnicode_Check(name)) { + if (PyUnicode_CheckExact(name)) { + if (PyUnicode_READY(name) == -1) + return -1; + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; + } + } + else { + /* Will fail in _PyObject_GenericSetAttrWithDict. */ + Py_INCREF(name); + } + res = PyObject_GenericSetAttr((PyObject *)type, name, value); + if (res == 0) { + res = update_slot(type, name); + } + Py_DECREF(name); + return res; } extern void @@ -6929,7 +6956,7 @@ init_slotdefs(void) /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyUnicode_InternFromString(p->name); - if (!p->name_strobj) + if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) Py_FatalError("Out of memory interning slotdef names"); } slotdefs_initialized = 1; @@ -6954,6 +6981,9 @@ update_slot(PyTypeObject *type, PyObject *name) slotdef **pp; int offset; + assert(PyUnicode_CheckExact(name)); + assert(PyUnicode_CHECK_INTERNED(name)); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. This could possibly be unified with the update_subclasses() recursion below, but carefully: @@ -6964,7 +6994,6 @@ update_slot(PyTypeObject *type, PyObject *name) init_slotdefs(); pp = ptrs; for (p = slotdefs; p->name; p++) { - /* XXX assume name is interned! */ if (p->name_strobj == name) *pp++ = p; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454b495dee..e9fc6580383dde 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1779,6 +1779,7 @@ unicode_dealloc(PyObject *unicode) case SSTATE_INTERNED_IMMORTAL: Py_FatalError("Immortal interned string died."); + /* fall through */ default: Py_FatalError("Inconsistent interned string state."); @@ -3017,6 +3018,37 @@ PyUnicode_AsWideCharString(PyObject *unicode, return buffer; } +wchar_t* +_PyUnicode_AsWideCharString(PyObject *unicode) +{ + const wchar_t *wstr; + wchar_t *buffer; + Py_ssize_t buflen; + + if (unicode == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + wstr = PyUnicode_AsUnicodeAndSize(unicode, &buflen); + if (wstr == NULL) { + return NULL; + } + if (wcslen(wstr) != (size_t)buflen) { + PyErr_SetString(PyExc_ValueError, + "embedded null character"); + return NULL; + } + + buffer = PyMem_NEW(wchar_t, buflen + 1); + if (buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + memcpy(buffer, wstr, (buflen + 1) * sizeof(wchar_t)); + return buffer; +} + #endif /* HAVE_WCHAR_H */ PyObject * @@ -3902,6 +3934,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) PyObject *output = NULL; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); + *(PyObject**)addr = NULL; return 1; } @@ -4132,6 +4165,20 @@ PyUnicode_AsUnicode(PyObject *unicode) return PyUnicode_AsUnicodeAndSize(unicode, NULL); } +const Py_UNICODE * +_PyUnicode_AsUnicode(PyObject *unicode) +{ + Py_ssize_t size; + const Py_UNICODE *wstr; + + wstr = PyUnicode_AsUnicodeAndSize(unicode, &size); + if (wstr && wcslen(wstr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + return wstr; +} + Py_ssize_t PyUnicode_GetSize(PyObject *unicode) @@ -4164,10 +4211,13 @@ PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) void *data; int kind; - if (!PyUnicode_Check(unicode) || PyUnicode_READY(unicode) == -1) { + if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return (Py_UCS4)-1; } + if (PyUnicode_READY(unicode) == -1) { + return (Py_UCS4)-1; + } if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) { PyErr_SetString(PyExc_IndexError, "string index out of range"); return (Py_UCS4)-1; @@ -5467,13 +5517,12 @@ _PyUnicode_EncodeUTF32(PyObject *str, /* four bytes are reserved for each surrogate */ if (moreunits > 1) { Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v); - Py_ssize_t morebytes = 4 * (moreunits - 1); - if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 4) { /* integer overflow */ PyErr_NoMemory(); goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 4 * (moreunits - 1)) < 0) goto error; out = (uint32_t*) PyBytes_AS_STRING(v) + outpos; } @@ -5819,13 +5868,12 @@ _PyUnicode_EncodeUTF16(PyObject *str, /* two bytes are reserved for each surrogate */ if (moreunits > 1) { Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v); - Py_ssize_t morebytes = 2 * (moreunits - 1); - if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 2) { /* integer overflow */ PyErr_NoMemory(); goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 2 * (moreunits - 1)) < 0) goto error; out = (unsigned short*) PyBytes_AS_STRING(v) + outpos; } @@ -6505,6 +6553,10 @@ _PyUnicode_DecodeUnicodeInternal(const char *s, 1)) return NULL; + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } if (size == 0) _Py_RETURN_UNICODE_EMPTY(); @@ -6765,7 +6817,7 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_REPLACE: memset(str, '?', collend - collstart); str += (collend - collstart); - /* fall through ignore error handler */ + /* fall through */ case _Py_ERROR_IGNORE: pos = collend; break; @@ -6804,7 +6856,7 @@ unicode_encode_ucs1(PyObject *unicode, break; collstart = pos; assert(collstart != collend); - /* fallback to general error handling */ + /* fall through */ default: rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, @@ -7306,6 +7358,10 @@ decode_code_page_stateful(int code_page, PyErr_SetString(PyExc_ValueError, "invalid code page number"); return NULL; } + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } if (consumed) *consumed = 0; @@ -11654,10 +11710,13 @@ unicode_getitem(PyObject *self, Py_ssize_t index) enum PyUnicode_Kind kind; Py_UCS4 ch; - if (!PyUnicode_Check(self) || PyUnicode_READY(self) == -1) { + if (!PyUnicode_Check(self)) { PyErr_BadArgument(); return NULL; } + if (PyUnicode_READY(self) == -1) { + return NULL; + } if (index < 0 || index >= PyUnicode_GET_LENGTH(self)) { PyErr_SetString(PyExc_IndexError, "string index out of range"); return NULL; @@ -11701,7 +11760,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12821,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) @@ -13915,10 +13982,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b23525552b8..9ca386da2563cb 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -24,6 +24,8 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; self->wr_object = ob; + self->wr_prev = NULL; + self->wr_next = NULL; Py_XINCREF(callback); self->wr_callback = callback; } diff --git a/PC/_findvs.cpp b/PC/_findvs.cpp new file mode 100644 index 00000000000000..dd7c1fcb723670 --- /dev/null +++ b/PC/_findvs.cpp @@ -0,0 +1,263 @@ +// +// Helper library for location Visual Studio installations +// using the COM-based query API. +// +// Copyright (c) Microsoft Corporation +// Licensed to PSF under a contributor agreement +// + +// Version history +// 2017-05: Initial contribution (Steve Dower) + +#include +#include +#include "external\include\Setup.Configuration.h" +#pragma comment(lib, "ole32.lib") +#pragma comment(lib, "oleaut32.lib") +#pragma comment(lib, "version.lib") +#pragma comment(lib, "Microsoft.VisualStudio.Setup.Configuration.Native.lib") + +#include + +static PyObject *error_from_hr(HRESULT hr) +{ + if (FAILED(hr)) + PyErr_Format(PyExc_OSError, "Error %08x", hr); + assert(PyErr_Occurred()); + return nullptr; +} + +static PyObject *get_install_name(ISetupInstance2 *inst) +{ + HRESULT hr; + BSTR name; + PyObject *str = nullptr; + if (FAILED(hr = inst->GetDisplayName(LOCALE_USER_DEFAULT, &name))) + goto error; + str = PyUnicode_FromWideChar(name, SysStringLen(name)); + SysFreeString(name); + return str; +error: + + return error_from_hr(hr); +} + +static PyObject *get_install_version(ISetupInstance *inst) +{ + HRESULT hr; + BSTR ver; + PyObject *str = nullptr; + if (FAILED(hr = inst->GetInstallationVersion(&ver))) + goto error; + str = PyUnicode_FromWideChar(ver, SysStringLen(ver)); + SysFreeString(ver); + return str; +error: + + return error_from_hr(hr); +} + +static PyObject *get_install_path(ISetupInstance *inst) +{ + HRESULT hr; + BSTR path; + PyObject *str = nullptr; + if (FAILED(hr = inst->GetInstallationPath(&path))) + goto error; + str = PyUnicode_FromWideChar(path, SysStringLen(path)); + SysFreeString(path); + return str; +error: + + return error_from_hr(hr); +} + +static PyObject *get_installed_packages(ISetupInstance2 *inst) +{ + HRESULT hr; + PyObject *res = nullptr; + LPSAFEARRAY sa_packages = nullptr; + LONG ub = 0; + IUnknown **packages = nullptr; + PyObject *str = nullptr; + + if (FAILED(hr = inst->GetPackages(&sa_packages)) || + FAILED(hr = SafeArrayAccessData(sa_packages, (void**)&packages)) || + FAILED(SafeArrayGetUBound(sa_packages, 1, &ub)) || + !(res = PyList_New(0))) + goto error; + + for (LONG i = 0; i < ub; ++i) { + ISetupPackageReference *package = nullptr; + BSTR id = nullptr; + PyObject *str = nullptr; + + if (FAILED(hr = packages[i]->QueryInterface(&package)) || + FAILED(hr = package->GetId(&id))) + goto iter_error; + + str = PyUnicode_FromWideChar(id, SysStringLen(id)); + SysFreeString(id); + + if (!str || PyList_Append(res, str) < 0) + goto iter_error; + + Py_CLEAR(str); + package->Release(); + continue; + + iter_error: + if (package) package->Release(); + Py_XDECREF(str); + + goto error; + } + + SafeArrayUnaccessData(sa_packages); + SafeArrayDestroy(sa_packages); + + return res; +error: + if (sa_packages && packages) SafeArrayUnaccessData(sa_packages); + if (sa_packages) SafeArrayDestroy(sa_packages); + Py_XDECREF(res); + + return error_from_hr(hr); +} + +static PyObject *find_all_instances() +{ + ISetupConfiguration *sc = nullptr; + ISetupConfiguration2 *sc2 = nullptr; + IEnumSetupInstances *enm = nullptr; + ISetupInstance *inst = nullptr; + ISetupInstance2 *inst2 = nullptr; + PyObject *res = nullptr; + ULONG fetched; + HRESULT hr; + + if (!(res = PyList_New(0))) + goto error; + + if (FAILED(hr = CoCreateInstance( + __uuidof(SetupConfiguration), + NULL, + CLSCTX_INPROC_SERVER, + __uuidof(ISetupConfiguration), + (LPVOID*)&sc + )) && hr != REGDB_E_CLASSNOTREG) + goto error; + + // If the class is not registered, there are no VS instances installed + if (hr == REGDB_E_CLASSNOTREG) + return res; + + if (FAILED(hr = sc->QueryInterface(&sc2)) || + FAILED(hr = sc2->EnumAllInstances(&enm))) + goto error; + + while (SUCCEEDED(enm->Next(1, &inst, &fetched)) && fetched) { + PyObject *name = nullptr; + PyObject *version = nullptr; + PyObject *path = nullptr; + PyObject *packages = nullptr; + PyObject *tuple = nullptr; + + if (FAILED(hr = inst->QueryInterface(&inst2)) || + !(name = get_install_name(inst2)) || + !(version = get_install_version(inst)) || + !(path = get_install_path(inst)) || + !(packages = get_installed_packages(inst2)) || + !(tuple = PyTuple_Pack(4, name, version, path, packages)) || + PyList_Append(res, tuple) < 0) + goto iter_error; + + Py_DECREF(tuple); + Py_DECREF(packages); + Py_DECREF(path); + Py_DECREF(version); + Py_DECREF(name); + continue; + iter_error: + if (inst2) inst2->Release(); + Py_XDECREF(tuple); + Py_XDECREF(packages); + Py_XDECREF(path); + Py_XDECREF(version); + Py_XDECREF(name); + goto error; + } + + enm->Release(); + sc2->Release(); + sc->Release(); + return res; + +error: + if (enm) enm->Release(); + if (sc2) sc2->Release(); + if (sc) sc->Release(); + Py_XDECREF(res); + + return error_from_hr(hr); +} + +PyDoc_STRVAR(findvs_findall_doc, "findall()\ +\ +Finds all installed versions of Visual Studio.\ +\ +This function will initialize COM temporarily. To avoid impact on other parts\ +of your application, use a new thread to make this call."); + +static PyObject *findvs_findall(PyObject *self, PyObject *args, PyObject *kwargs) +{ + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (hr == RPC_E_CHANGED_MODE) + hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + if (FAILED(hr)) + return error_from_hr(hr); + PyObject *res = find_all_instances(); + CoUninitialize(); + return res; +} + +// List of functions to add to findvs in exec_findvs(). +static PyMethodDef findvs_functions[] = { + { "findall", (PyCFunction)findvs_findall, METH_VARARGS | METH_KEYWORDS, findvs_findall_doc }, + { NULL, NULL, 0, NULL } +}; + +// Initialize findvs. May be called multiple times, so avoid +// using static state. +static int exec_findvs(PyObject *module) +{ + PyModule_AddFunctions(module, findvs_functions); + + return 0; // success +} + +PyDoc_STRVAR(findvs_doc, "The _findvs helper module"); + +static PyModuleDef_Slot findvs_slots[] = { + { Py_mod_exec, exec_findvs }, + { 0, NULL } +}; + +static PyModuleDef findvs_def = { + PyModuleDef_HEAD_INIT, + "_findvs", + findvs_doc, + 0, // m_size + NULL, // m_methods + findvs_slots, + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free +}; + +extern "C" { + PyMODINIT_FUNC PyInit__findvs(void) + { + return PyModuleDef_Init(&findvs_def); + } +} \ No newline at end of file diff --git a/PC/_msi.c b/PC/_msi.c index 789b04f5a25177..15aa7d8fecf324 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -600,8 +600,12 @@ summary_setproperty(msiobj* si, PyObject *args) return NULL; if (PyUnicode_Check(data)) { + const WCHAR *value = _PyUnicode_AsUnicode(data); + if (value == NULL) { + return NULL; + } status = MsiSummaryInfoSetPropertyW(si->h, field, VT_LPSTR, - 0, NULL, PyUnicode_AsUnicode(data)); + 0, NULL, value); } else if (PyLong_CheckExact(data)) { long value = PyLong_AsLong(data); if (value == -1 && PyErr_Occurred()) { diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj b/PC/bdist_wininst/bdist_wininst.vcxproj index 4dc5009b58e6ac..9bbfda28ccbe2b 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj +++ b/PC/bdist_wininst/bdist_wininst.vcxproj @@ -58,6 +58,10 @@ $(PySourcePath)lib\distutils\command\ false wininst-$(VisualStudioVersion) + wininst-14.0 + wininst-12.0 + wininst-11.0 + wininst-10.0 $(TargetName)-amd64 .exe diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index 25f565ce0a16d8..d5cf75206dd810 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -1,22 +1,19 @@ -@echo off -setlocal - -set D=%~dp0 -set PCBUILD=%~dp0..\..\PCBuild\ - - -echo Building Lib\distutils\command\wininst-xx.0.exe - -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 -if errorlevel 1 goto :eof - - -echo Building Lib\distutils\command\wininst-xx.0-amd64.exe - -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +@echo off +setlocal + +set D=%~dp0 +set PCBUILD=%~dp0..\..\PCBuild\ + + +echo Building Lib\distutils\command\wininst-xx.0.exe + +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) + +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +if errorlevel 1 goto :eof + + +echo Building Lib\distutils\command\wininst-xx.0-amd64.exe + +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PC/config.c b/PC/config.c index 43d9e208cca6b8..f631d73268f092 100644 --- a/PC/config.c +++ b/PC/config.c @@ -1,7 +1,7 @@ /* Module configuration */ /* This file contains the table of built-in modules. - See create_builtin() in import.c. */ + See create_builtin() in import.c. */ #include "Python.h" @@ -69,6 +69,7 @@ extern PyObject* _PyWarnings_Init(void); extern PyObject* PyInit__string(void); extern PyObject* PyInit__stat(void); extern PyObject* PyInit__opcode(void); +extern PyObject* PyInit__findvs(void); /* tools/freeze/makeconfig.py marker for additional "extern" */ /* -- ADDMODULE MARKER 1 -- */ @@ -165,6 +166,8 @@ struct _inittab _PyImport_Inittab[] = { {"_stat", PyInit__stat}, {"_opcode", PyInit__opcode}, + {"_findvs", PyInit__findvs}, + /* Sentinel */ {0, 0} }; diff --git a/PC/external/Externals.txt b/PC/external/Externals.txt new file mode 100644 index 00000000000000..618fe16fd4cfec --- /dev/null +++ b/PC/external/Externals.txt @@ -0,0 +1,3 @@ +The files in this folder are from the Microsoft.VisualStudio.Setup.Configuration.Native package on Nuget. + +They are licensed under the MIT license. diff --git a/PC/external/include/Setup.Configuration.h b/PC/external/include/Setup.Configuration.h new file mode 100644 index 00000000000000..1fb31878d7af72 --- /dev/null +++ b/PC/external/include/Setup.Configuration.h @@ -0,0 +1,827 @@ +// The MIT License(MIT) +// Copyright(C) Microsoft Corporation.All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and / or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// + +#pragma once + +// Constants +// +#ifndef E_NOTFOUND +#define E_NOTFOUND HRESULT_FROM_WIN32(ERROR_NOT_FOUND) +#endif + +#ifndef E_FILENOTFOUND +#define E_FILENOTFOUND HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) +#endif + +#ifndef E_NOTSUPPORTED +#define E_NOTSUPPORTED HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED) +#endif + +// Enumerations +// +///

    +/// The state of an instance. +/// +enum InstanceState +{ + /// + /// The instance state has not been determined. + /// + eNone = 0, + + /// + /// The instance installation path exists. + /// + eLocal = 1, + + /// + /// A product is registered to the instance. + /// + eRegistered = 2, + + /// + /// No reboot is required for the instance. + /// + eNoRebootRequired = 4, + + /// + /// No errors were reported for the instance. + /// + eNoErrors = 8, + + /// + /// The instance represents a complete install. + /// + eComplete = MAXUINT, +}; + +// Forward interface declarations +// +#ifndef __ISetupInstance_FWD_DEFINED__ +#define __ISetupInstance_FWD_DEFINED__ +typedef struct ISetupInstance ISetupInstance; +#endif + +#ifndef __ISetupInstance2_FWD_DEFINED__ +#define __ISetupInstance2_FWD_DEFINED__ +typedef struct ISetupInstance2 ISetupInstance2; +#endif + +#ifndef __ISetupLocalizedProperties_FWD_DEFINED__ +#define __ISetupLocalizedProperties_FWD_DEFINED__ +typedef struct ISetupLocalizedProperties ISetupLocalizedProperties; +#endif + +#ifndef __IEnumSetupInstances_FWD_DEFINED__ +#define __IEnumSetupInstances_FWD_DEFINED__ +typedef struct IEnumSetupInstances IEnumSetupInstances; +#endif + +#ifndef __ISetupConfiguration_FWD_DEFINED__ +#define __ISetupConfiguration_FWD_DEFINED__ +typedef struct ISetupConfiguration ISetupConfiguration; +#endif + +#ifndef __ISetupConfiguration2_FWD_DEFINED__ +#define __ISetupConfiguration2_FWD_DEFINED__ +typedef struct ISetupConfiguration2 ISetupConfiguration2; +#endif + +#ifndef __ISetupPackageReference_FWD_DEFINED__ +#define __ISetupPackageReference_FWD_DEFINED__ +typedef struct ISetupPackageReference ISetupPackageReference; +#endif + +#ifndef __ISetupHelper_FWD_DEFINED__ +#define __ISetupHelper_FWD_DEFINED__ +typedef struct ISetupHelper ISetupHelper; +#endif + +#ifndef __ISetupErrorState_FWD_DEFINED__ +#define __ISetupErrorState_FWD_DEFINED__ +typedef struct ISetupErrorState ISetupErrorState; +#endif + +#ifndef __ISetupErrorState2_FWD_DEFINED__ +#define __ISetupErrorState2_FWD_DEFINED__ +typedef struct ISetupErrorState2 ISetupErrorState2; +#endif + +#ifndef __ISetupFailedPackageReference_FWD_DEFINED__ +#define __ISetupFailedPackageReference_FWD_DEFINED__ +typedef struct ISetupFailedPackageReference ISetupFailedPackageReference; +#endif + +#ifndef __ISetupFailedPackageReference2_FWD_DEFINED__ +#define __ISetupFailedPackageReference2_FWD_DEFINED__ +typedef struct ISetupFailedPackageReference2 ISetupFailedPackageReference2; +#endif + +#ifndef __ISetupPropertyStore_FWD_DEFINED__ +#define __ISetupPropertyStore_FWD_DEFINED__ +typedef struct ISetupPropertyStore ISetupPropertyStore; +#endif + +#ifndef __ISetupLocalizedPropertyStore_FWD_DEFINED__ +#define __ISetupLocalizedPropertyStore_FWD_DEFINED__ +typedef struct ISetupLocalizedPropertyStore ISetupLocalizedPropertyStore; +#endif + +// Forward class declarations +// +#ifndef __SetupConfiguration_FWD_DEFINED__ +#define __SetupConfiguration_FWD_DEFINED__ + +#ifdef __cplusplus +typedef class SetupConfiguration SetupConfiguration; +#endif + +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// Interface definitions +// +EXTERN_C const IID IID_ISetupInstance; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Information about an instance of a product. +/// +struct DECLSPEC_UUID("B41463C3-8866-43B5-BC33-2B0676F7F42E") DECLSPEC_NOVTABLE ISetupInstance : public IUnknown +{ + /// + /// Gets the instance identifier (should match the name of the parent instance directory). + /// + /// The instance identifier. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetInstanceId)( + _Out_ BSTR* pbstrInstanceId + ) = 0; + + /// + /// Gets the local date and time when the installation was originally installed. + /// + /// The local date and time when the installation was originally installed. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetInstallDate)( + _Out_ LPFILETIME pInstallDate + ) = 0; + + /// + /// Gets the unique name of the installation, often indicating the branch and other information used for telemetry. + /// + /// The unique name of the installation, often indicating the branch and other information used for telemetry. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetInstallationName)( + _Out_ BSTR* pbstrInstallationName + ) = 0; + + /// + /// Gets the path to the installation root of the product. + /// + /// The path to the installation root of the product. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetInstallationPath)( + _Out_ BSTR* pbstrInstallationPath + ) = 0; + + /// + /// Gets the version of the product installed in this instance. + /// + /// The version of the product installed in this instance. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetInstallationVersion)( + _Out_ BSTR* pbstrInstallationVersion + ) = 0; + + /// + /// Gets the display name (title) of the product installed in this instance. + /// + /// The LCID for the display name. + /// The display name (title) of the product installed in this instance. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetDisplayName)( + _In_ LCID lcid, + _Out_ BSTR* pbstrDisplayName + ) = 0; + + /// + /// Gets the description of the product installed in this instance. + /// + /// The LCID for the description. + /// The description of the product installed in this instance. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(GetDescription)( + _In_ LCID lcid, + _Out_ BSTR* pbstrDescription + ) = 0; + + /// + /// Resolves the optional relative path to the root path of the instance. + /// + /// A relative path within the instance to resolve, or NULL to get the root path. + /// The full path to the optional relative path within the instance. If the relative path is NULL, the root path will always terminate in a backslash. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the property is not defined. + STDMETHOD(ResolvePath)( + _In_opt_z_ LPCOLESTR pwszRelativePath, + _Out_ BSTR* pbstrAbsolutePath + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupInstance2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Information about an instance of a product. +/// +struct DECLSPEC_UUID("89143C9A-05AF-49B0-B717-72E218A2185C") DECLSPEC_NOVTABLE ISetupInstance2 : public ISetupInstance +{ + /// + /// Gets the state of the instance. + /// + /// The state of the instance. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetState)( + _Out_ InstanceState* pState + ) = 0; + + /// + /// Gets an array of package references registered to the instance. + /// + /// Pointer to an array of . + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the packages property is not defined. + STDMETHOD(GetPackages)( + _Out_ LPSAFEARRAY* ppsaPackages + ) = 0; + + /// + /// Gets a pointer to the that represents the registered product. + /// + /// Pointer to an instance of . This may be NULL if does not return . + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist and E_NOTFOUND if the packages property is not defined. + STDMETHOD(GetProduct)( + _Outptr_result_maybenull_ ISetupPackageReference** ppPackage + ) = 0; + + /// + /// Gets the relative path to the product application, if available. + /// + /// The relative path to the product application, if available. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetProductPath)( + _Outptr_result_maybenull_ BSTR* pbstrProductPath + ) = 0; + + /// + /// Gets the error state of the instance, if available. + /// + /// The error state of the instance, if available. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetErrors)( + _Outptr_result_maybenull_ ISetupErrorState** ppErrorState + ) = 0; + + /// + /// Gets a value indicating whether the instance can be launched. + /// + /// Whether the instance can be launched. + /// Standard HRESULT indicating success or failure. + /// + /// An instance could have had errors during install but still be launched. Some features may not work correctly, but others will. + /// + STDMETHOD(IsLaunchable)( + _Out_ VARIANT_BOOL* pfIsLaunchable + ) = 0; + + /// + /// Gets a value indicating whether the instance is complete. + /// + /// Whether the instance is complete. + /// Standard HRESULT indicating success or failure. + /// + /// An instance is complete if it had no errors during install, resume, or repair. + /// + STDMETHOD(IsComplete)( + _Out_ VARIANT_BOOL* pfIsComplete + ) = 0; + + /// + /// Gets product-specific properties. + /// + /// A pointer to an instance of . This may be NULL if no properties are defined. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetProperties)( + _Outptr_result_maybenull_ ISetupPropertyStore** ppProperties + ) = 0; + + /// + /// Gets the directory path to the setup engine that installed the instance. + /// + /// The directory path to the setup engine that installed the instance. + /// Standard HRESULT indicating success or failure, including E_FILENOTFOUND if the instance state does not exist. + STDMETHOD(GetEnginePath)( + _Outptr_result_maybenull_ BSTR* pbstrEnginePath + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupLocalizedProperties; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Provides localized properties of an instance of a product. +/// +struct DECLSPEC_UUID("F4BD7382-FE27-4AB4-B974-9905B2A148B0") DECLSPEC_NOVTABLE ISetupLocalizedProperties : public IUnknown +{ + /// + /// Gets localized product-specific properties. + /// + /// A pointer to an instance of . This may be NULL if no properties are defined. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetLocalizedProperties)( + _Outptr_result_maybenull_ ISetupLocalizedPropertyStore** ppLocalizedProperties + ) = 0; + + /// + /// Gets localized channel-specific properties. + /// + /// A pointer to an instance of . This may be NULL if no channel properties are defined. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetLocalizedChannelProperties)( + _Outptr_result_maybenull_ ISetupLocalizedPropertyStore** ppLocalizedChannelProperties + ) = 0; +}; +#endif + +EXTERN_C const IID IID_IEnumSetupInstances; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// An enumerator of installed objects. +/// +struct DECLSPEC_UUID("6380BCFF-41D3-4B2E-8B2E-BF8A6810C848") DECLSPEC_NOVTABLE IEnumSetupInstances : public IUnknown +{ + /// + /// Retrieves the next set of product instances in the enumeration sequence. + /// + /// The number of product instances to retrieve. + /// A pointer to an array of . + /// A pointer to the number of product instances retrieved. If is 1 this parameter may be NULL. + /// S_OK if the number of elements were fetched, S_FALSE if nothing was fetched (at end of enumeration), E_INVALIDARG if is greater than 1 and pceltFetched is NULL, or E_OUTOFMEMORY if an could not be allocated. + STDMETHOD(Next)( + _In_ ULONG celt, + _Out_writes_to_(celt, *pceltFetched) ISetupInstance** rgelt, + _Out_opt_ _Deref_out_range_(0, celt) ULONG* pceltFetched + ) = 0; + + /// + /// Skips the next set of product instances in the enumeration sequence. + /// + /// The number of product instances to skip. + /// S_OK if the number of elements could be skipped; otherwise, S_FALSE; + STDMETHOD(Skip)( + _In_ ULONG celt + ) = 0; + + /// + /// Resets the enumeration sequence to the beginning. + /// + /// Always returns S_OK; + STDMETHOD(Reset)(void) = 0; + + /// + /// Creates a new enumeration object in the same state as the current enumeration object: the new object points to the same place in the enumeration sequence. + /// + /// A pointer to a pointer to a new interface. If the method fails, this parameter is undefined. + /// S_OK if a clone was returned; otherwise, E_OUTOFMEMORY. + STDMETHOD(Clone)( + _Deref_out_opt_ IEnumSetupInstances** ppenum + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupConfiguration; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Gets information about product instances installed on the machine. +/// +struct DECLSPEC_UUID("42843719-DB4C-46C2-8E7C-64F1816EFD5B") DECLSPEC_NOVTABLE ISetupConfiguration : public IUnknown +{ + /// + /// Enumerates all launchable product instances installed. + /// + /// An enumeration of completed, installed product instances. + /// Standard HRESULT indicating success or failure. + STDMETHOD(EnumInstances)( + _Out_ IEnumSetupInstances** ppEnumInstances + ) = 0; + + /// + /// Gets the instance for the current process path. + /// + /// The instance for the current process path. + /// + /// The instance for the current process path, or E_NOTFOUND if not found. + /// The may indicate the instance is invalid. + /// + /// + /// The returned instance may not be launchable. + /// +STDMETHOD(GetInstanceForCurrentProcess)( + _Out_ ISetupInstance** ppInstance + ) = 0; + + /// + /// Gets the instance for the given path. + /// + /// The instance for the given path. + /// + /// The instance for the given path, or E_NOTFOUND if not found. + /// The may indicate the instance is invalid. + /// + /// + /// The returned instance may not be launchable. + /// +STDMETHOD(GetInstanceForPath)( + _In_z_ LPCWSTR wzPath, + _Out_ ISetupInstance** ppInstance + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupConfiguration2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Gets information about product instances. +/// +struct DECLSPEC_UUID("26AAB78C-4A60-49D6-AF3B-3C35BC93365D") DECLSPEC_NOVTABLE ISetupConfiguration2 : public ISetupConfiguration +{ + /// + /// Enumerates all product instances. + /// + /// An enumeration of all product instances. + /// Standard HRESULT indicating success or failure. + STDMETHOD(EnumAllInstances)( + _Out_ IEnumSetupInstances** ppEnumInstances + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupPackageReference; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// A reference to a package. +/// +struct DECLSPEC_UUID("da8d8a16-b2b6-4487-a2f1-594ccccd6bf5") DECLSPEC_NOVTABLE ISetupPackageReference : public IUnknown +{ + /// + /// Gets the general package identifier. + /// + /// The general package identifier. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetId)( + _Out_ BSTR* pbstrId + ) = 0; + + /// + /// Gets the version of the package. + /// + /// The version of the package. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetVersion)( + _Out_ BSTR* pbstrVersion + ) = 0; + + /// + /// Gets the target process architecture of the package. + /// + /// The target process architecture of the package. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetChip)( + _Out_ BSTR* pbstrChip + ) = 0; + + /// + /// Gets the language and optional region identifier. + /// + /// The language and optional region identifier. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetLanguage)( + _Out_ BSTR* pbstrLanguage + ) = 0; + + /// + /// Gets the build branch of the package. + /// + /// The build branch of the package. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetBranch)( + _Out_ BSTR* pbstrBranch + ) = 0; + + /// + /// Gets the type of the package. + /// + /// The type of the package. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetType)( + _Out_ BSTR* pbstrType + ) = 0; + + /// + /// Gets the unique identifier consisting of all defined tokens. + /// + /// The unique identifier consisting of all defined tokens. + /// Standard HRESULT indicating success or failure, including E_UNEXPECTED if no Id was defined (required). + STDMETHOD(GetUniqueId)( + _Out_ BSTR* pbstrUniqueId + ) = 0; + + /// + /// Gets a value indicating whether the package refers to an external extension. + /// + /// A value indicating whether the package refers to an external extension. + /// Standard HRESULT indicating success or failure, including E_UNEXPECTED if no Id was defined (required). + STDMETHOD(GetIsExtension)( + _Out_ VARIANT_BOOL* pfIsExtension + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupHelper; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Helper functions. +/// +/// +/// You can query for this interface from the class. +/// +struct DECLSPEC_UUID("42b21b78-6192-463e-87bf-d577838f1d5c") DECLSPEC_NOVTABLE ISetupHelper : public IUnknown +{ + /// + /// Parses a dotted quad version string into a 64-bit unsigned integer. + /// + /// The dotted quad version string to parse, e.g. 1.2.3.4. + /// A 64-bit unsigned integer representing the version. You can compare this to other versions. + /// Standard HRESULT indicating success or failure, including E_INVALIDARG if the version is not valid. + STDMETHOD(ParseVersion)( + _In_ LPCOLESTR pwszVersion, + _Out_ PULONGLONG pullVersion + ) = 0; + + /// + /// Parses a dotted quad version string into a 64-bit unsigned integer. + /// + /// The string containing 1 or 2 dotted quad version strings to parse, e.g. [1.0,) that means 1.0.0.0 or newer. + /// A 64-bit unsigned integer representing the minimum version, which may be 0. You can compare this to other versions. + /// A 64-bit unsigned integer representing the maximum version, which may be MAXULONGLONG. You can compare this to other versions. + /// Standard HRESULT indicating success or failure, including E_INVALIDARG if the version range is not valid. + STDMETHOD(ParseVersionRange)( + _In_ LPCOLESTR pwszVersionRange, + _Out_ PULONGLONG pullMinVersion, + _Out_ PULONGLONG pullMaxVersion + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupErrorState; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Information about the error state of an instance. +/// +struct DECLSPEC_UUID("46DCCD94-A287-476A-851E-DFBC2FFDBC20") DECLSPEC_NOVTABLE ISetupErrorState : public IUnknown +{ + /// + /// Gets an array of failed package references. + /// + /// Pointer to an array of , if packages have failed. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetFailedPackages)( + _Outptr_result_maybenull_ LPSAFEARRAY* ppsaFailedPackages + ) = 0; + + /// + /// Gets an array of skipped package references. + /// + /// Pointer to an array of , if packages have been skipped. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetSkippedPackages)( + _Outptr_result_maybenull_ LPSAFEARRAY* ppsaSkippedPackages + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupErrorState2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Information about the error state of an instance. +/// +struct DECLSPEC_UUID("9871385B-CA69-48F2-BC1F-7A37CBF0B1EF") DECLSPEC_NOVTABLE ISetupErrorState2 : public ISetupErrorState +{ + /// + /// Gets the path to the error log. + /// + /// The path to the error log. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetErrorLogFilePath)( + _Outptr_result_maybenull_ BSTR* pbstrErrorLogFilePath + ) = 0; +}; +#endif + +EXTERN_C const IID IID_ISetupFailedPackageReference; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// A reference to a failed package. +/// +struct DECLSPEC_UUID("E73559CD-7003-4022-B134-27DC650B280F") DECLSPEC_NOVTABLE ISetupFailedPackageReference : public ISetupPackageReference +{ +}; + +#endif + +EXTERN_C const IID IID_ISetupFailedPackageReference2; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// A reference to a failed package. +/// +struct DECLSPEC_UUID("0FAD873E-E874-42E3-B268-4FE2F096B9CA") DECLSPEC_NOVTABLE ISetupFailedPackageReference2 : public ISetupFailedPackageReference +{ + /// + /// Gets the path to the optional package log. + /// + /// The path to the optional package log. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetLogFilePath)( + _Outptr_result_maybenull_ BSTR* pbstrLogFilePath + ) = 0; + + /// + /// Gets the description of the package failure. + /// + /// The description of the package failure. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetDescription)( + _Outptr_result_maybenull_ BSTR* pbstrDescription + ) = 0; + + /// + /// Gets the signature to use for feedback reporting. + /// + /// The signature to use for feedback reporting. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetSignature)( + _Outptr_result_maybenull_ BSTR* pbstrSignature + ) = 0; + + /// + /// Gets the array of details for this package failure. + /// + /// Pointer to an array of details as BSTRs. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetDetails)( + _Out_ LPSAFEARRAY* ppsaDetails + ) = 0; + + /// + /// Gets an array of packages affected by this package failure. + /// + /// Pointer to an array of for packages affected by this package failure. This may be NULL. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetAffectedPackages)( + _Out_ LPSAFEARRAY* ppsaAffectedPackages + ) = 0; +}; + +#endif + +EXTERN_C const IID IID_ISetupPropertyStore; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Provides named properties. +/// +/// +/// You can get this from an , , or derivative. +/// +struct DECLSPEC_UUID("C601C175-A3BE-44BC-91F6-4568D230FC83") DECLSPEC_NOVTABLE ISetupPropertyStore : public IUnknown +{ + /// + /// Gets an array of property names in this property store. + /// + /// Pointer to an array of property names as BSTRs. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetNames)( + _Out_ LPSAFEARRAY* ppsaNames + ) = 0; + + /// + /// Gets the value of a named property in this property store. + /// + /// The name of the property to get. + /// The value of the property. + /// Standard HRESULT indicating success or failure, including E_NOTFOUND if the property is not defined or E_NOTSUPPORTED if the property type is not supported. + STDMETHOD(GetValue)( + _In_ LPCOLESTR pwszName, + _Out_ LPVARIANT pvtValue + ) = 0; +}; + +#endif + +EXTERN_C const IID IID_ISetupLocalizedPropertyStore; + +#if defined(__cplusplus) && !defined(CINTERFACE) +/// +/// Provides localized named properties. +/// +/// +/// You can get this from an . +/// +struct DECLSPEC_UUID("5BB53126-E0D5-43DF-80F1-6B161E5C6F6C") DECLSPEC_NOVTABLE ISetupLocalizedPropertyStore : public IUnknown +{ + /// + /// Gets an array of property names in this property store. + /// + /// The LCID for the property names. + /// Pointer to an array of property names as BSTRs. + /// Standard HRESULT indicating success or failure. + STDMETHOD(GetNames)( + _In_ LCID lcid, + _Out_ LPSAFEARRAY* ppsaNames + ) = 0; + + /// + /// Gets the value of a named property in this property store. + /// + /// The name of the property to get. + /// The LCID for the property. + /// The value of the property. + /// Standard HRESULT indicating success or failure, including E_NOTFOUND if the property is not defined or E_NOTSUPPORTED if the property type is not supported. + STDMETHOD(GetValue)( + _In_ LPCOLESTR pwszName, + _In_ LCID lcid, + _Out_ LPVARIANT pvtValue + ) = 0; +}; + +#endif + +// Class declarations +// +EXTERN_C const CLSID CLSID_SetupConfiguration; + +#ifdef __cplusplus +/// +/// This class implements , , and . +/// +class DECLSPEC_UUID("177F0C4A-1CD3-4DE7-A32C-71DBBB9FA36D") SetupConfiguration; +#endif + +// Function declarations +// +/// +/// Gets an that provides information about product instances installed on the machine. +/// +/// The that provides information about product instances installed on the machine. +/// Reserved for future use. +/// Standard HRESULT indicating success or failure. +STDMETHODIMP GetSetupConfiguration( + _Out_ ISetupConfiguration** ppConfiguration, + _Reserved_ LPVOID pReserved +); + +#ifdef __cplusplus +} +#endif diff --git a/PC/external/v140/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib b/PC/external/v140/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib new file mode 100644 index 00000000000000..675a501d8cb5a7 Binary files /dev/null and b/PC/external/v140/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib differ diff --git a/PC/external/v140/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib b/PC/external/v140/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib new file mode 100644 index 00000000000000..40f70b2682d35b Binary files /dev/null and b/PC/external/v140/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib differ diff --git a/PC/external/v141/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib b/PC/external/v141/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib new file mode 100644 index 00000000000000..675a501d8cb5a7 Binary files /dev/null and b/PC/external/v141/amd64/Microsoft.VisualStudio.Setup.Configuration.Native.lib differ diff --git a/PC/external/v141/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib b/PC/external/v141/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib new file mode 100644 index 00000000000000..40f70b2682d35b Binary files /dev/null and b/PC/external/v141/win32/Microsoft.VisualStudio.Setup.Configuration.Native.lib differ diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe9c19451..e7be704a9a78dd 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PC/python.manifest b/PC/python.manifest index 4e73d60df2842d..8e1bc022adfb4f 100644 --- a/PC/python.manifest +++ b/PC/python.manifest @@ -1,20 +1,20 @@ - + - - - + + + - + diff --git a/PC/winsound.c b/PC/winsound.c index 7feebcbcf4377f..0d3f04584335ce 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -101,7 +101,7 @@ winsound_PlaySound_impl(PyObject *module, PyObject *sound, int flags) Py_TYPE(sound)->tp_name); return NULL; } - wsound = PyUnicode_AsWideCharString(sound, NULL); + wsound = _PyUnicode_AsWideCharString(sound); if (wsound == NULL) { return NULL; } diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 725b5a967bebc9..0db61f208183d7 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 @@ -87,6 +87,7 @@ + @@ -103,4 +104,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index ee78295fa9494d..4597ee521b3331 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -33,6 +33,9 @@ Header Files + + Header Files + Header Files @@ -69,4 +72,4 @@ Source Files - \ No newline at end of file + diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index 7ec2692010783e..d8b159e6a1d853 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -65,7 +65,7 @@ WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions) - $(OutDir)/liblzma$(PyDebugExt).lib + $(OutDir)liblzma$(PyDebugExt).lib;%(AdditionalDependencies) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 70ab340905e4c9..5828b518b11aa9 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -1,156 +1,144 @@ -@echo off -goto Run -:Usage -echo.%~nx0 [flags and arguments] [quoted MSBuild options] -echo. -echo.Build CPython from the command line. Requires the appropriate -echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). -echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is -echo.given. -echo. -echo.After the flags recognized by this script, up to 9 arguments to be passed -echo.directly to MSBuild may be passed. If the argument contains an '=', the -echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`) -echo. -echo.Available flags: -echo. -h Display this help message -echo. -V Display version information for the current build -echo. -r Target Rebuild instead of Build -echo. -d Set the configuration to Debug -echo. -e Build external libraries fetched by get_externals.bat -echo. Extension modules that depend on external libraries will not attempt -echo. to build if this flag is not present -echo. -m Enable parallel build (enabled by default) -echo. -M Disable parallel build -echo. -v Increased output messages -echo. -k Attempt to kill any running Pythons before building (usually done -echo. automatically by the pythoncore project) -echo. --pgo Build with Profile-Guided Optimization. This flag -echo. overrides -c and -d -echo. --test-marker Enable the test marker within the build. -echo. -echo.Available flags to avoid building certain modules. -echo.These flags have no effect if '-e' is not given: -echo. --no-ssl Do not attempt to build _ssl -echo. --no-tkinter Do not attempt to build Tkinter -echo. -echo.Available arguments: -echo. -c Release ^| Debug ^| PGInstrument ^| PGUpdate -echo. Set the configuration (default: Release) -echo. -p x64 ^| Win32 -echo. Set the platform (default: Win32) -echo. -t Build ^| Rebuild ^| Clean ^| CleanAll -echo. Set the target manually -echo. --pgo-job The job to use for PGO training; implies --pgo -echo. (default: "-m test --pgo") -exit /b 127 - -:Run -setlocal -set platf=Win32 -set vs_platf=x86 -set conf=Release -set target=Build -set dir=%~dp0 -set parallel=/m -set verbose=/nologo /v:m -set kill= -set do_pgo= -set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) - -:CheckOpts -if "%~1"=="-h" goto Usage -if "%~1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts -if "%~1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts -if "%~1"=="-r" (set target=Rebuild) & shift & goto CheckOpts -if "%~1"=="-t" (set target=%2) & shift & shift & goto CheckOpts -if "%~1"=="-d" (set conf=Debug) & shift & goto CheckOpts -if "%~1"=="-m" (set parallel=/m) & shift & goto CheckOpts -if "%~1"=="-M" (set parallel=) & shift & goto CheckOpts -if "%~1"=="-v" (set verbose=/v:n) & shift & goto CheckOpts -if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts -if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts -if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts -if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts -if "%~1"=="-V" shift & goto Version -rem These use the actual property names used by MSBuild. We could just let -rem them in through the environment, but we specify them on the command line -rem anyway for visibility so set defaults after this -if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts -if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts -if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts - -if "%IncludeExternals%"=="" set IncludeExternals=false -if "%IncludeSSL%"=="" set IncludeSSL=true -if "%IncludeTkinter%"=="" set IncludeTkinter=true - -if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" - -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 - ) -) - -if not exist "%HG%" where hg > "%TEMP%\hg.loc" 2> nul && set /P HG= < "%TEMP%\hg.loc" & del "%TEMP%\hg.loc" -if exist "%HG%" set HGProperty=/p:HG="%HG%" -if not exist "%HG%" echo Cannot find Mercurial on PATH & set HGProperty= - -rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul - -if "%kill%"=="true" call :Kill - -if "%do_pgo%"=="true" ( - set conf=PGInstrument - call :Build - del /s "%dir%\*.pgc" - del /s "%dir%\..\Lib\*.pyc" - echo on - call "%dir%\..\python.bat" %pgo_job% - @echo off - call :Kill - set conf=PGUpdate - set target=Build -) -goto Build -:Kill -echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ - /p:Configuration=%conf% /p:Platform=%platf%^ - /p:KillPython=true - -@echo off -goto :eof - -:Build -rem Call on MSBuild to do the work, echo the command. -rem Passing %1-9 is not the preferred option, but argument parsing in -rem batch is, shall we say, "lackluster" -echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ - /p:Configuration=%conf% /p:Platform=%platf%^ - /p:IncludeExternals=%IncludeExternals%^ - /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ - /p:UseTestMarker=%UseTestMarker% %HGProperty%^ - %1 %2 %3 %4 %5 %6 %7 %8 %9 - -@echo off -goto :eof - -:Version -rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +@echo off +goto Run +:Usage +echo.%~nx0 [flags and arguments] [quoted MSBuild options] +echo. +echo.Build CPython from the command line. Requires the appropriate +echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). +echo. +echo.After the flags recognized by this script, up to 9 arguments to be passed +echo.directly to MSBuild may be passed. If the argument contains an '=', the +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`) +echo. +echo.Available flags: +echo. -h Display this help message +echo. -V Display version information for the current build +echo. -r Target Rebuild instead of Build +echo. -d Set the configuration to Debug +echo. -e Build external libraries fetched by get_externals.bat +echo. Extension modules that depend on external libraries will not attempt +echo. to build if this flag is not present +echo. -m Enable parallel build (enabled by default) +echo. -M Disable parallel build +echo. -v Increased output messages +echo. -k Attempt to kill any running Pythons before building (usually done +echo. automatically by the pythoncore project) +echo. --pgo Build with Profile-Guided Optimization. This flag +echo. overrides -c and -d +echo. --test-marker Enable the test marker within the build. +echo. +echo.Available flags to avoid building certain modules. +echo.These flags have no effect if '-e' is not given: +echo. --no-ssl Do not attempt to build _ssl +echo. --no-tkinter Do not attempt to build Tkinter +echo. +echo.Available arguments: +echo. -c Release ^| Debug ^| PGInstrument ^| PGUpdate +echo. Set the configuration (default: Release) +echo. -p x64 ^| Win32 +echo. Set the platform (default: Win32) +echo. -t Build ^| Rebuild ^| Clean ^| CleanAll +echo. Set the target manually +echo. --pgo-job The job to use for PGO training; implies --pgo +echo. (default: "-m test --pgo") +exit /b 127 + +:Run +setlocal +set platf=Win32 +set conf=Release +set target=Build +set dir=%~dp0 +set parallel=/m +set verbose=/nologo /v:m +set kill= +set do_pgo= +set pgo_job=-m test --pgo + +:CheckOpts +if "%~1"=="-h" goto Usage +if "%~1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-r" (set target=Rebuild) & shift & goto CheckOpts +if "%~1"=="-t" (set target=%2) & shift & shift & goto CheckOpts +if "%~1"=="-d" (set conf=Debug) & shift & goto CheckOpts +if "%~1"=="-m" (set parallel=/m) & shift & goto CheckOpts +if "%~1"=="-M" (set parallel=) & shift & goto CheckOpts +if "%~1"=="-v" (set verbose=/v:n) & shift & goto CheckOpts +if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts +if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts +if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts +if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts +if "%~1"=="-V" shift & goto Version +rem These use the actual property names used by MSBuild. We could just let +rem them in through the environment, but we specify them on the command line +rem anyway for visibility so set defaults after this +if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts +if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts + +if "%IncludeExternals%"=="" set IncludeExternals=true +if "%IncludeSSL%"=="" set IncludeSSL=true +if "%IncludeTkinter%"=="" set IncludeTkinter=true + +if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" + +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 + ) +) + +if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" +if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" +if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= + +rem Setup the environment +call "%dir%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) + +if "%kill%"=="true" call :Kill + +if "%do_pgo%"=="true" ( + set conf=PGInstrument + call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9 + del /s "%dir%\*.pgc" + del /s "%dir%\..\Lib\*.pyc" + echo on + call "%dir%\..\python.bat" %pgo_job% + @echo off + call :Kill + set conf=PGUpdate + set target=Build +) +goto Build +:Kill +echo on +%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:KillPython=true + +@echo off +goto :eof + +:Build +rem Call on MSBuild to do the work, echo the command. +rem Passing %1-9 is not the preferred option, but argument parsing in +rem batch is, shall we say, "lackluster" +echo on +%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:IncludeExternals=%IncludeExternals%^ + /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ + /p:UseTestMarker=%UseTestMarker% %GITProperty%^ + %1 %2 %3 %4 %5 %6 %7 %8 %9 + +@echo off +goto :eof + +:Version +rem Display the current build version information +%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/build_env.bat b/PCbuild/build_env.bat index 01024cff215d39..4c67ae3ac253c7 100644 --- a/PCbuild/build_env.bat +++ b/PCbuild/build_env.bat @@ -1 +1 @@ -@%comspec% /k env.bat %* +@%comspec% /k env.bat %* diff --git a/PCbuild/clean.bat b/PCbuild/clean.bat index 78517d225f9f44..6144c715f64e43 100644 --- a/PCbuild/clean.bat +++ b/PCbuild/clean.bat @@ -1,5 +1,5 @@ -@echo off -rem A batch program to clean a particular configuration, -rem just for convenience. - -call %~dp0build.bat -t Clean %* +@echo off +rem A batch program to clean a particular configuration, +rem just for convenience. + +call %~dp0build.bat -t Clean %* diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 2b2c0051ce912f..9d4c9d1c32f7a4 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -1,16 +1,16 @@ -@echo off -rem This script adds the latest available tools to the path for the current -rem command window. However, most builds of Python will ignore the version -rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of -rem tools should be the same version to avoid potential conflicts. -rem -rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or -rem 'v110', 'v120' or 'v140') to the build script. - -echo Build environments: x86, amd64, x86_amd64 -echo. -set VSTOOLS=%VS140COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS120COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS110COMNTOOLS% -if "%VSTOOLS%"=="" set VSTOOLS=%VS100COMNTOOLS% -call "%VSTOOLS%..\..\VC\vcvarsall.bat" %* +@echo off +rem This script adds the latest available tools to the path for the current +rem command window. However, most builds of Python will ignore the version +rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of +rem tools should be the same version to avoid potential conflicts. +rem +rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or +rem 'v110', 'v120' or 'v140') to the build script. + +echo Build environments: x86, amd64, x86_amd64 +echo. +set VSTOOLS=%VS140COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS120COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS110COMNTOOLS% +if "%VSTOOLS%"=="" set VSTOOLS=%VS100COMNTOOLS% +call "%VSTOOLS%..\..\VC\vcvarsall.bat" %* diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat new file mode 100644 index 00000000000000..2b7413fbcde870 --- /dev/null +++ b/PCbuild/find_msbuild.bat @@ -0,0 +1,52 @@ +@rem +@rem Searches for MSBuild.exe. This is the only tool we need to initiate +@rem a build, so we no longer search for the full VC toolset. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found + +:begin_search +@set MSBUILD= + +@rem If msbuild.exe is on the PATH, assume that the user wants that one. +@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" +@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found + +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + +@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( + @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found + + +@exit /b 1 + +:found +@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%) +@set _Py_MSBuild_Source= diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat new file mode 100644 index 00000000000000..ba3a0f5a10daa3 --- /dev/null +++ b/PCbuild/find_python.bat @@ -0,0 +1,75 @@ +@rem +@rem Searches for python.exe and may download a private copy from nuget. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found + +:begin_search +@set PYTHON= + +@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR% +@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) + +@rem If we have Python in externals, use that one +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found + +@rem If HOST_PYTHON is recent enough, use that +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found + +@rem If py.exe finds a recent enough version, use that one +@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found + +@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" +@set _Py_NUGET=%NUGET% +@set _Py_NUGET_URL=%NUGET_URL% +@set _Py_HOST_PYTHON=%HOST_PYTHON% +@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py +@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if NOT exist "%_Py_NUGET%" ( + @echo Downloading nuget... + @rem NB: Must use single quotes around NUGET here, NOT double! + @rem Otherwise, a space in the path would break things + @rem If it fails, retry with any available copy of Python + @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%' + @if errorlevel 1 ( + @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%" + ) +) +@echo Installing Python via nuget... +@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work +@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found + + +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= +@exit /b 1 + +:found +@echo Using %PYTHON% (%_Py_Python_Source%) +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py new file mode 100644 index 00000000000000..a682d3849f14c6 --- /dev/null +++ b/PCbuild/get_external.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import zipfile +from urllib.request import urlretrieve + + +def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): + repo = f'cpython-{"bin" if binary else "source"}-deps' + url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' + reporthook = None + if verbose: + reporthook = print + zip_dir.mkdir(parents=True, exist_ok=True) + filename, headers = urlretrieve( + url, + zip_dir / f'{commit_hash}.zip', + reporthook=reporthook, + ) + return filename + + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-b', '--binary', action='store_true', + help='Is the dependency in the binary repo?') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='python') + p.add_argument('-e', '--externals-dir', type=pathlib.Path, + help='Directory in which to store dependencies', + default=pathlib.Path(__file__).parent.parent / 'externals') + p.add_argument('tag', + help='tag of the dependency') + return p.parse_args() + + +def main(): + args = parse_args() + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + binary=args.binary, + verbose=args.verbose, + ) + final_name = args.externals_dir / args.tag + extract_zip(args.externals_dir, zip_path).replace(final_name) + + +if __name__ == '__main__': + main() diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index a5185becee5a96..0a86892c3500ec 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -1,104 +1,102 @@ -@echo off -setlocal -rem Simple script to fetch source for external libraries - -if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" -pushd "%~dp0..\externals" - -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ - -rem Optionally clean up first. Be warned that this can be very destructive! -if not "%1"=="" ( - for %%c in (-c --clean --clean-only) do ( - if "%1"=="%%c" goto clean - ) - goto usage -) -goto fetch - -:clean -echo.Cleaning up external libraries. -for /D %%d in ( - bzip2-* - db-* - nasm-* - openssl-* - tcl-* - tcltk* - tk-* - tix-* - sqlite-* - xz-* - ) do ( - echo.Removing %%d - rmdir /s /q %%d -) -if "%1"=="--clean-only" ( - goto end -) - -:fetch -rem Fetch current versions - -svn --version > nul 2>&1 -if ERRORLEVEL 9009 ( - echo.svn.exe must be on your PATH. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 -) - -echo.Fetching external libraries... - -set libraries= -set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2j -set libraries=%libraries% sqlite-3.14.2.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 -set libraries=%libraries% xz-5.2.2 - -for %%e in (%libraries%) do ( - if exist %%e ( - echo.%%e already exists, skipping. - ) else ( - echo.Fetching %%e... - svn export %SVNROOT%%%e - ) -) - -goto end - -:usage -echo.invalid argument: %1 -echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] -echo. -echo.Pull all sources necessary for compiling optional extension modules -echo.that rely on external libraries. Requires svn.exe to be on your PATH -echo.and pulls sources from %SVNROOT%. -echo. -echo.Use the -c or --clean option to clean up all external library sources -echo.before pulling in the current versions. -echo. -echo.Use the --clean-only option to do the same cleaning, without pulling in -echo.anything new. -echo. -echo.Only the first argument is checked, all others are ignored. -echo. -echo.**WARNING**: the cleaning options unconditionally remove any directory -echo.that is a child of -echo. %CD% -echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, -echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential -echo.to be very destructive if you are not aware of what it is doing. Use with -echo.caution! -popd -exit /b -1 - - -:end -echo Finished. -popd +@echo off +setlocal +rem Simple script to fetch source for external libraries + +if "%PCBUILD%"=="" (set PCBUILD=%~dp0) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) + +set DO_FETCH=true +set DO_CLEAN=false + +:CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts +if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts +if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts +if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean +if "x%~1" NEQ "x" goto usage + +if "%DO_CLEAN%"=="false" goto fetch +:clean +echo.Cleaning up external libraries. +if exist "%EXTERNALS_DIR%" ( + rem Sometimes this fails the first time; try it twice + rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" +) + +if "%DO_FETCH%"=="false" goto end +:fetch + +if "%ORG%"=="" (set ORG=python) + +call "%PCBUILD%find_python.bat" "%PYTHON%" + +if "%PYTHON%"=="" ( + where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 +) + +echo.Fetching external libraries... + +set libraries= +set libraries=%libraries% bzip2-1.0.6 +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k +set libraries=%libraries% sqlite-3.14.2.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 +set libraries=%libraries% xz-5.2.2 + +for %%e in (%libraries%) do ( + if exist "%EXTERNALS_DIR%\%%e" ( + echo.%%e already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%e with git... + git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e" + ) else ( + echo.Fetching %%e... + %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e + ) +) + +echo.Fetching external binaries... + +set binaries= +set binaries=%binaries% +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 + +for %%b in (%binaries%) do ( + if exist "%EXTERNALS_DIR%\%%b" ( + echo.%%b already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%b with git... + git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" + ) else ( + echo.Fetching %%b... + %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b + ) +) + +echo Finished. +goto end + +:usage +echo.Valid options: -c, --clean, --clean-only, --organization, --python, +echo.--no-tkinter, --no-openssl +echo. +echo.Pull all sources and binaries necessary for compiling optional extension +echo.modules that rely on external libraries. +echo. +echo.The --organization option determines which github organization to download +echo.from, the --python option determines which Python 3.6+ interpreter to use +echo.with PCbuild\get_external.py. +echo. +echo.Use the -c or --clean option to remove the entire externals directory. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +exit /b -1 + +:end diff --git a/PCbuild/idle.bat b/PCbuild/idle.bat index bacaaa8414a267..1978b99f6ee19d 100644 --- a/PCbuild/idle.bat +++ b/PCbuild/idle.bat @@ -1,15 +1,15 @@ -@echo off -rem start idle -rem Usage: idle [-d] -rem -d Run Debug build (python_d.exe). Else release build. - -setlocal -set exe=win32\python -PATH %PATH%;..\externals\tcltk\bin - -if "%1"=="-d" (set exe=%exe%_d) & shift - -set cmd=%exe% ../Lib/idlelib/idle.py %1 %2 %3 %4 %5 %6 %7 %8 %9 - -echo on -%cmd% +@echo off +rem start idle +rem Usage: idle [-d] +rem -d Run Debug build (python_d.exe). Else release build. + +setlocal +set exe=win32\python +PATH %PATH%;..\externals\tcltk\bin + +if "%1"=="-d" (set exe=%exe%_d) & shift + +set cmd=%exe% ../Lib/idlelib/idle.py %1 %2 %3 %4 %5 %6 %7 %8 %9 + +echo on +%cmd% diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 0e65811ae70731..736a5997037208 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,735 +1,769 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25123.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" - ProjectSection(SolutionItems) = preProject - ..\Modules\getbuildinfo.c = ..\Modules\getbuildinfo.c - readme.txt = readme.txt - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonw", "pythonw.vcxproj", "{F4229CC3-873C-49AE-9729-DD308ED4CD4A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winsound", "winsound.vcxproj", "{28B5D777-DDF2-4B6B-B34F-31D938813856}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes_test", "_ctypes_test.vcxproj", "{9EC7190A-249F-4180-A900-548FDCF3055F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_msi", "_msi.vcxproj", "{31FFC478-7B4A-43E8-9954-8D03E2187E9C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_sqlite3", "_sqlite3.vcxproj", "{13CECB97-4119-4316-9D42-8534019A5A44}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testcapi", "_testcapi.vcxproj", "{6901D91C-6E48-4BB7-9FEC-700C8131DF1D}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_tkinter", "_tkinter.vcxproj", "{4946ECAC-2E69-4BF8-A90A-F5136F5094DF}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_bz2", "_bz2.vcxproj", "{73FCD2BD-F133-46B7-8EC1-144CD82A59D5}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_lzma", "_lzma.vcxproj", "{F9D71780-F393-11E0-BE50-0800200C9A66}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bdist_wininst", "..\PC\bdist_wininst\bdist_wininst.vcxproj", "{EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sqlite3", "sqlite3.vcxproj", "{A1A295E5-463C-437F-81CA-1F32367685DA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3dll", "python3dll.vcxproj", "{885D4898-D08D-4091-9C40-C700CFE3FC5A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxproj", "{F749B822-B489-4CA5-A3AD-CE078F5F338A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pylauncher", "pylauncher.vcxproj", "{7B2727B5-5A3F-40EE-A866-43A13CD31446}" - ProjectSection(ProjectDependencies) = postProject - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} = {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.vcxproj", "{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}" - ProjectSection(ProjectDependencies) = postProject - {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testembed", "_testembed.vcxproj", "{6DAC66D9-E703-4624-BE03-49112AB5AA62}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmultiphase.vcxproj", "{16BFE6F0-22EF-40B5-B831-7E937119EF10}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tcl", "tcl.vcxproj", "{B5FD6F1D-129E-4BFF-9340-03606FAC7283}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tix", "tix.vcxproj", "{C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tk", "tk.vcxproj", "{7E85ECCF-A72C-4DA4-9E52-884508E80BA1}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay", "libeay.vcxproj", "{E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay", "ssleay.vcxproj", "{10615B24-73BF-4EFA-93AA-236916321317}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - PGInstrument|Win32 = PGInstrument|Win32 - PGInstrument|x64 = PGInstrument|x64 - PGUpdate|Win32 = PGUpdate|Win32 - PGUpdate|x64 = PGUpdate|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.ActiveCfg = Debug|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.Build.0 = Debug|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.ActiveCfg = Debug|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.Build.0 = Debug|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.ActiveCfg = Release|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.Build.0 = Release|Win32 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.ActiveCfg = Release|x64 - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.Build.0 = Release|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.ActiveCfg = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.Build.0 = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.ActiveCfg = Release|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.ActiveCfg = Debug|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.Build.0 = Debug|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.ActiveCfg = Debug|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.Build.0 = Debug|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.Build.0 = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.ActiveCfg = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.Build.0 = Release|Win32 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.ActiveCfg = Release|x64 - {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.Build.0 = Release|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.ActiveCfg = Debug|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.Build.0 = Debug|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.ActiveCfg = Debug|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.Build.0 = Debug|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.ActiveCfg = Release|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.Build.0 = Release|Win32 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.ActiveCfg = Release|x64 - {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.ActiveCfg = Debug|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.Build.0 = Debug|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.ActiveCfg = Debug|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.Build.0 = Debug|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.Build.0 = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.ActiveCfg = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.Build.0 = Release|Win32 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.ActiveCfg = Release|x64 - {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.Build.0 = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.ActiveCfg = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.Build.0 = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.ActiveCfg = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.Build.0 = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.ActiveCfg = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.Build.0 = Debug|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.ActiveCfg = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.Build.0 = Debug|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.ActiveCfg = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.Build.0 = Release|Win32 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.ActiveCfg = Release|x64 - {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.Build.0 = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.ActiveCfg = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.Build.0 = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.ActiveCfg = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.Build.0 = Release|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.ActiveCfg = Debug|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.Build.0 = Debug|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.ActiveCfg = Debug|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.Build.0 = Debug|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.ActiveCfg = Release|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.Build.0 = Release|Win32 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.ActiveCfg = Release|x64 - {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.Build.0 = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.ActiveCfg = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.Build.0 = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.ActiveCfg = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.Build.0 = Release|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.ActiveCfg = Debug|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.Build.0 = Debug|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.ActiveCfg = Debug|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.Build.0 = Debug|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.ActiveCfg = Release|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.Build.0 = Release|Win32 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.ActiveCfg = Release|x64 - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.ActiveCfg = Debug|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.Build.0 = Debug|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.ActiveCfg = Debug|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.Build.0 = Debug|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.Build.0 = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.ActiveCfg = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.Build.0 = Release|Win32 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.ActiveCfg = Release|x64 - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.Build.0 = Release|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.ActiveCfg = Debug|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.Build.0 = Debug|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.ActiveCfg = Debug|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.Build.0 = Debug|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.ActiveCfg = Release|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.Build.0 = Release|Win32 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.ActiveCfg = Release|x64 - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.Build.0 = Release|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.ActiveCfg = Debug|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.Build.0 = Debug|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.ActiveCfg = Debug|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.Build.0 = Debug|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.ActiveCfg = Release|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.Build.0 = Release|Win32 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.ActiveCfg = Release|x64 - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.Build.0 = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.ActiveCfg = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.Build.0 = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.ActiveCfg = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.Build.0 = Release|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.ActiveCfg = Debug|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.Build.0 = Debug|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.ActiveCfg = Debug|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.Build.0 = Debug|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.ActiveCfg = Release|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.Build.0 = Release|Win32 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.ActiveCfg = Release|x64 - {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.Build.0 = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.ActiveCfg = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.Build.0 = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.ActiveCfg = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.Build.0 = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.ActiveCfg = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.Build.0 = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.ActiveCfg = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.Build.0 = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|Win32.ActiveCfg = Debug|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|x64.ActiveCfg = Release|x64 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|Win32.ActiveCfg = Release|Win32 - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|x64.ActiveCfg = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.ActiveCfg = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.Build.0 = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.ActiveCfg = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.Build.0 = Release|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.ActiveCfg = Debug|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.Build.0 = Debug|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.ActiveCfg = Debug|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.Build.0 = Debug|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.ActiveCfg = Release|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.Build.0 = Release|Win32 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.ActiveCfg = Release|x64 - {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.Build.0 = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.ActiveCfg = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.Build.0 = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.ActiveCfg = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.Build.0 = Release|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.ActiveCfg = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.Build.0 = Debug|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.ActiveCfg = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.Build.0 = Debug|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.ActiveCfg = Release|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.Build.0 = Release|Win32 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.ActiveCfg = Release|x64 - {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.Build.0 = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|Win32.ActiveCfg = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.Build.0 = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.ActiveCfg = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.Build.0 = Release|Win32 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.ActiveCfg = Release|x64 - {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.ActiveCfg = Debug|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.Build.0 = Debug|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.ActiveCfg = Debug|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.Build.0 = Debug|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.Build.0 = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.ActiveCfg = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.Build.0 = Release|Win32 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.ActiveCfg = Release|x64 - {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.ActiveCfg = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.Build.0 = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.ActiveCfg = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.Build.0 = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.ActiveCfg = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.Build.0 = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.ActiveCfg = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.Build.0 = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.ActiveCfg = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.Build.0 = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.ActiveCfg = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.Build.0 = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.ActiveCfg = Debug|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.Build.0 = Debug|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.ActiveCfg = Debug|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.Build.0 = Debug|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.Build.0 = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.Build.0 = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.ActiveCfg = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.Build.0 = Release|Win32 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.ActiveCfg = Release|x64 - {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.Build.0 = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.ActiveCfg = Debug|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.Build.0 = Debug|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.ActiveCfg = Debug|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.Build.0 = Debug|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.Build.0 = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.Build.0 = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.ActiveCfg = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.Build.0 = Release|Win32 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.ActiveCfg = Release|x64 - {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.ActiveCfg = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.Build.0 = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.ActiveCfg = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.Build.0 = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.ActiveCfg = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.Build.0 = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.ActiveCfg = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.Build.0 = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.ActiveCfg = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.Build.0 = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.ActiveCfg = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.Build.0 = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.ActiveCfg = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.Build.0 = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.ActiveCfg = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.Build.0 = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.ActiveCfg = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.Build.0 = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.ActiveCfg = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.Build.0 = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.Build.0 = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.Build.0 = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.ActiveCfg = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.ActiveCfg = Debug|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.Build.0 = Debug|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.ActiveCfg = Debug|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.Build.0 = Debug|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.Build.0 = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.ActiveCfg = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 - {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25123.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" + ProjectSection(SolutionItems) = preProject + ..\Modules\getbuildinfo.c = ..\Modules\getbuildinfo.c + readme.txt = readme.txt + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonw", "pythonw.vcxproj", "{F4229CC3-873C-49AE-9729-DD308ED4CD4A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winsound", "winsound.vcxproj", "{28B5D777-DDF2-4B6B-B34F-31D938813856}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes_test", "_ctypes_test.vcxproj", "{9EC7190A-249F-4180-A900-548FDCF3055F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_msi", "_msi.vcxproj", "{31FFC478-7B4A-43E8-9954-8D03E2187E9C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_sqlite3", "_sqlite3.vcxproj", "{13CECB97-4119-4316-9D42-8534019A5A44}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testcapi", "_testcapi.vcxproj", "{6901D91C-6E48-4BB7-9FEC-700C8131DF1D}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_tkinter", "_tkinter.vcxproj", "{4946ECAC-2E69-4BF8-A90A-F5136F5094DF}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_bz2", "_bz2.vcxproj", "{73FCD2BD-F133-46B7-8EC1-144CD82A59D5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_lzma", "_lzma.vcxproj", "{F9D71780-F393-11E0-BE50-0800200C9A66}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bdist_wininst", "..\PC\bdist_wininst\bdist_wininst.vcxproj", "{EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sqlite3", "sqlite3.vcxproj", "{A1A295E5-463C-437F-81CA-1F32367685DA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3dll", "python3dll.vcxproj", "{885D4898-D08D-4091-9C40-C700CFE3FC5A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxproj", "{F749B822-B489-4CA5-A3AD-CE078F5F338A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pylauncher", "pylauncher.vcxproj", "{7B2727B5-5A3F-40EE-A866-43A13CD31446}" + ProjectSection(ProjectDependencies) = postProject + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} = {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.vcxproj", "{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}" + ProjectSection(ProjectDependencies) = postProject + {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testembed", "_testembed.vcxproj", "{6DAC66D9-E703-4624-BE03-49112AB5AA62}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmultiphase.vcxproj", "{16BFE6F0-22EF-40B5-B831-7E937119EF10}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tcl", "tcl.vcxproj", "{B5FD6F1D-129E-4BFF-9340-03606FAC7283}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tix", "tix.vcxproj", "{C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tk", "tk.vcxproj", "{7E85ECCF-A72C-4DA4-9E52-884508E80BA1}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay", "libeay.vcxproj", "{E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay", "ssleay.vcxproj", "{10615B24-73BF-4EFA-93AA-236916321317}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + PGInstrument|Win32 = PGInstrument|Win32 + PGInstrument|x64 = PGInstrument|x64 + PGUpdate|Win32 = PGUpdate|Win32 + PGUpdate|x64 = PGUpdate|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.ActiveCfg = Debug|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|Win32.Build.0 = Debug|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.ActiveCfg = Debug|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Debug|x64.Build.0 = Debug|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.ActiveCfg = Release|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|Win32.Build.0 = Release|Win32 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.ActiveCfg = Release|x64 + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9}.Release|x64.Build.0 = Release|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.ActiveCfg = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|x64.Build.0 = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.ActiveCfg = Release|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.ActiveCfg = Debug|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|Win32.Build.0 = Debug|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.ActiveCfg = Debug|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Debug|x64.Build.0 = Debug|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGInstrument|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.PGUpdate|x64.Build.0 = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.ActiveCfg = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|Win32.Build.0 = Release|Win32 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.ActiveCfg = Release|x64 + {F4229CC3-873C-49AE-9729-DD308ED4CD4A}.Release|x64.Build.0 = Release|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.ActiveCfg = Debug|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|Win32.Build.0 = Debug|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.ActiveCfg = Debug|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Debug|x64.Build.0 = Debug|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.ActiveCfg = Release|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|Win32.Build.0 = Release|Win32 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.ActiveCfg = Release|x64 + {28B5D777-DDF2-4B6B-B34F-31D938813856}.Release|x64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|x64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311617}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|x64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|x64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311618}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.ActiveCfg = Debug|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|Win32.Build.0 = Debug|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.ActiveCfg = Debug|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Debug|x64.Build.0 = Debug|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGInstrument|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.PGUpdate|x64.Build.0 = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.ActiveCfg = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|Win32.Build.0 = Release|Win32 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.ActiveCfg = Release|x64 + {9EC7190A-249F-4180-A900-548FDCF3055F}.Release|x64.Build.0 = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.ActiveCfg = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|x64.Build.0 = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.ActiveCfg = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|x64.Build.0 = Release|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.ActiveCfg = Debug|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|Win32.Build.0 = Debug|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.ActiveCfg = Debug|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Debug|x64.Build.0 = Debug|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.ActiveCfg = Release|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|Win32.Build.0 = Release|Win32 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.ActiveCfg = Release|x64 + {31FFC478-7B4A-43E8-9954-8D03E2187E9C}.Release|x64.Build.0 = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.ActiveCfg = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|x64.Build.0 = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.ActiveCfg = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|x64.Build.0 = Release|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.ActiveCfg = Debug|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|Win32.Build.0 = Debug|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.ActiveCfg = Debug|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Debug|x64.Build.0 = Debug|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.ActiveCfg = Release|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|Win32.Build.0 = Release|Win32 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.ActiveCfg = Release|x64 + {13CECB97-4119-4316-9D42-8534019A5A44}.Release|x64.Build.0 = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.ActiveCfg = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|x64.Build.0 = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.ActiveCfg = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|x64.Build.0 = Release|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.ActiveCfg = Debug|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|Win32.Build.0 = Debug|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.ActiveCfg = Debug|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Debug|x64.Build.0 = Debug|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.ActiveCfg = Release|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|Win32.Build.0 = Release|Win32 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.ActiveCfg = Release|x64 + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D}.Release|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.ActiveCfg = Debug|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|Win32.Build.0 = Debug|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.ActiveCfg = Debug|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Debug|x64.Build.0 = Debug|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGInstrument|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.PGUpdate|x64.Build.0 = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.ActiveCfg = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|Win32.Build.0 = Release|Win32 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.ActiveCfg = Release|x64 + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}.Release|x64.Build.0 = Release|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.ActiveCfg = Debug|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|Win32.Build.0 = Debug|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.ActiveCfg = Debug|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Debug|x64.Build.0 = Debug|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.ActiveCfg = Release|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|Win32.Build.0 = Release|Win32 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.ActiveCfg = Release|x64 + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF}.Release|x64.Build.0 = Release|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.ActiveCfg = Debug|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|Win32.Build.0 = Debug|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.ActiveCfg = Debug|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Debug|x64.Build.0 = Debug|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.ActiveCfg = Release|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|Win32.Build.0 = Release|Win32 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.ActiveCfg = Release|x64 + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5}.Release|x64.Build.0 = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.ActiveCfg = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|x64.Build.0 = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.ActiveCfg = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|x64.Build.0 = Release|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.ActiveCfg = Debug|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|Win32.Build.0 = Debug|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.ActiveCfg = Debug|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Debug|x64.Build.0 = Debug|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.ActiveCfg = Release|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|Win32.Build.0 = Release|Win32 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.ActiveCfg = Release|x64 + {F9D71780-F393-11E0-BE50-0800200C9A66}.Release|x64.Build.0 = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.ActiveCfg = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|x64.Build.0 = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.ActiveCfg = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|x64.Build.0 = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.ActiveCfg = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|x64.Build.0 = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.ActiveCfg = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|x64.Build.0 = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|Win32.ActiveCfg = Debug|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Debug|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGInstrument|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.PGUpdate|x64.ActiveCfg = Release|x64 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|Win32.ActiveCfg = Release|Win32 + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C}.Release|x64.ActiveCfg = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.ActiveCfg = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|x64.Build.0 = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.ActiveCfg = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|x64.Build.0 = Release|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.ActiveCfg = Debug|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|Win32.Build.0 = Debug|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.ActiveCfg = Debug|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Debug|x64.Build.0 = Debug|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.ActiveCfg = Release|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|Win32.Build.0 = Release|Win32 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.ActiveCfg = Release|x64 + {A1A295E5-463C-437F-81CA-1F32367685DA}.Release|x64.Build.0 = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.ActiveCfg = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|x64.Build.0 = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.ActiveCfg = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|x64.Build.0 = Release|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Debug|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGInstrument|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.ActiveCfg = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|Win32.Build.0 = Debug|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.ActiveCfg = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.PGUpdate|x64.Build.0 = Debug|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.ActiveCfg = Release|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.Build.0 = Release|Win32 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.ActiveCfg = Release|x64 + {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.Build.0 = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|Win32.ActiveCfg = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.PGUpdate|x64.Build.0 = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.ActiveCfg = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|Win32.Build.0 = Release|Win32 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.ActiveCfg = Release|x64 + {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Release|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.ActiveCfg = Debug|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|Win32.Build.0 = Debug|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.ActiveCfg = Debug|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Debug|x64.Build.0 = Debug|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGInstrument|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.PGUpdate|x64.Build.0 = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.ActiveCfg = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.Build.0 = Release|Win32 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.ActiveCfg = Release|x64 + {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.ActiveCfg = Debug|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.Build.0 = Debug|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.ActiveCfg = Debug|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.Build.0 = Debug|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.Build.0 = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.ActiveCfg = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.Build.0 = Release|Win32 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.ActiveCfg = Release|x64 + {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.ActiveCfg = Debug|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.Build.0 = Debug|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.ActiveCfg = Debug|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.Build.0 = Debug|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.Build.0 = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.ActiveCfg = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.ActiveCfg = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|x64.Build.0 = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.ActiveCfg = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|x64.Build.0 = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.ActiveCfg = Debug|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|Win32.Build.0 = Debug|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.ActiveCfg = Debug|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Debug|x64.Build.0 = Debug|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGInstrument|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|Win32.Build.0 = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.PGUpdate|x64.Build.0 = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.ActiveCfg = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|Win32.Build.0 = Release|Win32 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.ActiveCfg = Release|x64 + {6DAC66D9-E703-4624-BE03-49112AB5AA62}.Release|x64.Build.0 = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.ActiveCfg = Debug|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|Win32.Build.0 = Debug|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.ActiveCfg = Debug|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Debug|x64.Build.0 = Debug|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGInstrument|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|Win32.Build.0 = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.PGUpdate|x64.Build.0 = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.ActiveCfg = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.Build.0 = Release|Win32 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.ActiveCfg = Release|x64 + {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.ActiveCfg = Debug|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.Build.0 = Debug|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.ActiveCfg = Debug|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.Build.0 = Debug|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.Build.0 = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.ActiveCfg = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.Build.0 = Release|Win32 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.ActiveCfg = Release|x64 + {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.ActiveCfg = Debug|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.Build.0 = Debug|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.ActiveCfg = Debug|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.Build.0 = Debug|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.Build.0 = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.ActiveCfg = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.Build.0 = Release|Win32 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.ActiveCfg = Release|x64 + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.ActiveCfg = Debug|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.Build.0 = Debug|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.ActiveCfg = Debug|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.Build.0 = Debug|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.Build.0 = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.ActiveCfg = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.Build.0 = Release|Win32 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.ActiveCfg = Release|x64 + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.ActiveCfg = Debug|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.Build.0 = Debug|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.ActiveCfg = Debug|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.Build.0 = Debug|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.Build.0 = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.ActiveCfg = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.Build.0 = Release|Win32 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.ActiveCfg = Release|x64 + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.ActiveCfg = Debug|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.Build.0 = Debug|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.ActiveCfg = Debug|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.Build.0 = Debug|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.Build.0 = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.ActiveCfg = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.Build.0 = Release|Win32 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.ActiveCfg = Release|x64 + {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.Build.0 = Release|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.Build.0 = Debug|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.ActiveCfg = Release|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.ActiveCfg = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|Win32.Build.0 = Debug|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.ActiveCfg = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|x64.Build.0 = Debug|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGInstrument|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.PGUpdate|x64.Build.0 = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.ActiveCfg = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 + {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/PCbuild/prepare_ssl.bat b/PCbuild/prepare_ssl.bat index 2f41ae847da5ae..6d9ac87a727076 100644 --- a/PCbuild/prepare_ssl.bat +++ b/PCbuild/prepare_ssl.bat @@ -1,12 +1,12 @@ -@echo off -if not defined HOST_PYTHON ( - if "%1" EQU "Debug" ( - shift - set HOST_PYTHON=python_d.exe - if not exist python36_d.dll exit 1 - ) ELSE ( - set HOST_PYTHON=python.exe - if not exist python36.dll exit 1 - ) -) -%HOST_PYTHON% "%~dp0prepare_ssl.py" %1 +@echo off +if not defined HOST_PYTHON ( + if "%1" EQU "Debug" ( + shift + set HOST_PYTHON=python_d.exe + if not exist python36_d.dll exit 1 + ) ELSE ( + set HOST_PYTHON=python.exe + if not exist python36.dll exit 1 + ) +) +%HOST_PYTHON% "%~dp0prepare_ssl.py" %1 diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 99c7286b2bdfde..51ca69e95d705e 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -59,7 +59,7 @@ $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) - PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions) @@ -68,6 +68,7 @@ + @@ -84,4 +85,4 @@ - \ No newline at end of file + diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index abf362fb0bfa8a..cb02847980c634 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -20,6 +20,9 @@ Source Files + + Source Files + Source Files @@ -30,4 +33,4 @@ Source Files - \ No newline at end of file + diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 7012170e0c7aa4..5a2da998c3799d 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -6,7 +6,7 @@ $(BuildPath) $(OutDir)\ $(MSBuildThisFileDirectory)obj\ - $(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -147,13 +147,28 @@ foreach (System.Diagnostics.Process p in System.Diagnostics.Process.GetProcesses Targets="CleanAll" /> + + + <_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" /> + <_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" /> + <_CopyFiles Include="@(_PGCFiles);@(_PGDFile)" Condition="Exists(%(FullPath))" /> + + + + + + - $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86 + $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\$(DefaultWindowsSDKVersion)\x86 + $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" - <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" diff --git a/PCbuild/python.props b/PCbuild/python.props index dde94f733a13ae..c26f642a9533c1 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -10,6 +10,7 @@ We set BasePlatformToolset for ICC's benefit, it's otherwise ignored. --> + v141 v140 v120 v110 @@ -39,13 +40,14 @@ $(BuildPath64) $(PySourcePath)PCBuild\$(ArchName)\ $(BuildPath)\ + $(BuildPath)instrumented\ $([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`)) $(ExternalsDir)sqlite-3.14.2.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2j\ + $(ExternalsDir)openssl-1.0.2k\ $(opensslDir)include32 $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ @@ -62,7 +64,24 @@ $(BuildPath)python$(PyDebugExt).exe - + + + + 10.0.15063.0 + 10.0.15063.0 + 10.0.14393.0 + 10.0.14393.0 + 10.0.10586.0 + 10.0.10586.0 + 10.0.10240.0 + 10.0.10240.0 + +