Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions Lib/multiprocessing/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,12 @@ def arbitrary_address(family):
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
# NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
# mktemp-style ToC vs ToU concern is not important; bind() surfaces
# the extremely unlikely collision as EADDRINUSE.
# NOTE: util.get_temp_dir() is a 0o700 per-process directory.
# A mktemp-style ToC vs ToU concern is not important as bind()
# surfaces the extremely unlikely collision as EADDRINUSE.
suffix = os.urandom(util._TMPSOCK_SUFFIXLEN // 2).hex()
return os.path.join(util.get_temp_dir(),
f'sock-{os.urandom(6).hex()}')
f"{util._TMPSOCK_PREFIX}{suffix}")
elif family == 'AF_PIPE':
return (r'\\.\pipe\pyc-%d-%d-%s' %
(os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
Expand Down
55 changes: 41 additions & 14 deletions Lib/multiprocessing/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,39 @@ def is_abstract_socket_namespace(address):
# On Windows platforms, we do not create AF_UNIX sockets.
_SUN_PATH_MAX = None if os.name == 'nt' else 92

# The temporary socket file path (without NULL terminator) is
#
# TEMPDIR/{_TMPPYMP_PREFIX}{S1}/{_TMPSOCK_PREFIX}{S2}
#
# where
#
# - S1 is a random suffix of length 8 generated by get_temp_dir().
# This is used to generate the temporary sub-directory that will
# contains the socket file. Update _TMPPYMP_SUFFIXLEN accordingly
# if the random suffix length chosen by tempfile.mkdtemp() changes.
Comment thread
picnixz marked this conversation as resolved.
#
# - S2 is a random suffix of length 2N generated by os.urandom(N).hex()
# in multiprocessing.connection.arbitrary_address(). The value of N
# is not fixed and may vary across Python versions as to minimize
# collisions and path lengths.
#
_TMPPYMP_PREFIX = "pymp-"
_TMPPYMP_PREFIXLEN = len(_TMPPYMP_PREFIX)
_TMPPYMP_SUFFIXLEN = 8
_TMPSOCK_PREFIX = "sock-"
_TMPSOCK_PREFIXLEN = len(_TMPSOCK_PREFIX)
_TMPSOCK_SUFFIXLEN = 12

# Length of the socket filepath from the chosen temporary directory,
# including a leading path separator and the path separator between
# the temporary subdirectory and the socket filename.
_SUN_PATH_LEN_RESERVED = (
len(os.path.sep) + _TMPPYMP_PREFIXLEN + _TMPPYMP_SUFFIXLEN +
len(os.path.sep) + _TMPSOCK_PREFIXLEN + _TMPSOCK_SUFFIXLEN
)
assert _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX


def _remove_temp_dir(rmtree, tempdir):
rmtree(tempdir)

Expand All @@ -152,31 +185,24 @@ def _remove_temp_dir(rmtree, tempdir):
if current_process is not None:
current_process._config['tempdir'] = None


def _get_base_temp_dir(tempfile):
"""Get a temporary directory where socket files will be created.

To prevent additional imports, pass a pre-imported 'tempfile' module.
"""
if os.name == 'nt':
return None
# Most of the time, the default temporary directory is /tmp. Thus,
# listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do
# not have a path length exceeding SUN_PATH_MAX.
# Most of the time, the default temporary directory is /tmp. As such,
# the path length of listener sockets files does not exceed SUN_PATH_MAX.
#
# If users specify their own temporary directory, we may be unable
# to create those files. Therefore, we fall back to the system-wide
# temporary directory /tmp, assumed to exist on POSIX systems.
#
# See https://github.com/python/cpython/issues/132124.
base_tempdir = tempfile.gettempdir()
# Files created in a temporary directory are suffixed by a string
# generated by tempfile._RandomNameSequence, which, by design,
# is 8 characters long.
#
# Thus, the socket file path length (without NULL terminator) will be:
#
# len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX')
sun_path_len = len(base_tempdir) + 14 + 14
sun_path_len = len(base_tempdir) + _SUN_PATH_LEN_RESERVED
# Strict inequality to account for the NULL terminator.
# See https://github.com/python/cpython/issues/140734.
if sun_path_len < _SUN_PATH_MAX:
Expand Down Expand Up @@ -204,17 +230,18 @@ def _get_base_temp_dir(tempfile):
# not be able to write socket files out there.
return base_tempdir
warn("Ignoring user-defined temporary directory: %s", base_tempdir)
# at most max(map(len, dirlist)) + 14 + 14 = 36 characters
assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX
# The following assertion must be satisfied with the chosen constants.
assert len(base_system_tempdir) + _SUN_PATH_LEN_RESERVED < _SUN_PATH_MAX
return base_system_tempdir


def get_temp_dir():
# get name of a temp directory which will be automatically cleaned up
tempdir = process.current_process()._config.get('tempdir')
if tempdir is None:
import shutil, tempfile
base_tempdir = _get_base_temp_dir(tempfile)
tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
tempdir = tempfile.mkdtemp(prefix=_TMPPYMP_PREFIX, dir=base_tempdir)
info('created temp directory %s', tempdir)
# keep a strong reference to shutil.rmtree(), since the finalizer
# can be called late during Python shutdown
Expand Down
35 changes: 35 additions & 0 deletions Lib/test/test_multiprocessing_forkserver/test_misc.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,42 @@
import os
import re
import unittest
from test.support import os_helper
from multiprocessing.util import _SUN_PATH_MAX
from test._test_multiprocessing import install_tests_in_module_dict
from test.support import script_helper

install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True)


class TestForkServerConfiguration(unittest.TestCase):
def test_respect_sun_path_max(self):
# Ensure that the calculation for temporary filepath lengths is correct.
# See https://github.com/python/cpython/issues/149527.

cmd = '''if 1:
from multiprocessing.connection import arbitrary_address
from multiprocessing.util import get_temp_dir
if __name__ == "__main__":
print(get_temp_dir())
print(arbitrary_address("AF_UNIX"))
'''
with os_helper.temp_dir() as root:
self.assertLess(len(root), _SUN_PATH_MAX)
_, out, _ = script_helper.assert_python_ok('-c', cmd, TMPDIR=root)
res = out.decode().strip().splitlines()
self.assertEqual(len(res), 2)

temp_pymp = res[0]
self.assertLess(len(temp_pymp), _SUN_PATH_MAX)
temp_pymp_regex = os.path.join(re.escape(root), r"pymp-\w{8}")
self.assertRegex(temp_pymp, temp_pymp_regex)

temp_sock = res[1]
self.assertLess(len(temp_sock), _SUN_PATH_MAX)
temp_sock_regex = os.path.join(temp_pymp_regex, r"sock-[0-9a-fA-F]{12}")
self.assertRegex(temp_sock, temp_sock_regex)


if __name__ == '__main__':
unittest.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
:mod:`multiprocessing`: fix creation of temporary socket files following
:gh:`137335` (`PR#148578 <https://github.com/python/cpython/pull/148578>`__
and corresponding backports). Patch by Bénédikt Tran.
Loading