From e116414a68038c1c52e983fabc2661f074bc0387 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 31 Aug 2026 21:04:04 +0200 Subject: [PATCH 1/5] [3.15] gh-156002: Bound zipfile decompression for bzip2/LZMA/Zstandard (GH-156003) (GH-156362) Patch by @tonghuaroot. zipfile.ZipExtFile._read1() bounds the output of each decompress() call for DEFLATE members by passing a max_length to zlib, but for bzip2, LZMA, and Zstandard members it called decompress() with no bound. A whole compressed chunk was therefore expanded into a single allocation before the data[:self._left] clip ran, so a consumer that deliberately reads in small chunks to limit memory (for example zf.open(name).read(8192)) was silently unprotected for non-DEFLATE members. A small, spec-conformant archive member declaring a large uncompressed size could drive multi-GB peak memory. _read1() now passes a per-call bound to the non-DEFLATE decompress() (mirroring the DEFLATE branch) and drains the decompressor's internal buffer across calls by checking needs_input before reading more compressed input. zipfile's LZMADecompressor wrapper forwards max_length and exposes needs_input so the bound also holds for LZMA members. (cherry picked from commit f897dbf2f36a5935700b7c2d94d4681d2136b7d4) (cherry picked from commit 1b424c0178a01e155fd0267dc28a8fc1159b33a8) Co-authored-by: Petr Viktorin Co-authored-by: tonghuaroot --- Lib/test/test_zipfile.py | 42 +++++++++++++++++++ Lib/zipfile.py | 38 ++++++++++++++--- ...-08-18-13-54-05.gh-issue-156002.CcWXPP.rst | 4 ++ 3 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 55e9792b36aa6c..de97985cb2684b 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2198,6 +2198,48 @@ def tearDown(self): unlink(TESTFN2) +class AbstractBoundedDecompressTests: + # ZipExtFile._read1() bounds the output of each decompress() call so that a + # small member declaring a large uncompressed size cannot expand into one + # unbounded read. + def test_read1_output_is_bounded(self): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: + zf.writestr("big", b"\0" * (4 * 1024 * 1024)) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + with zf.open("big") as f: + self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) + + +class StoredBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_STORED + + +@requires_zlib() +class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + + +@requires_bz2() +class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + + +@requires_lzma() +class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +@requires_zstd() +class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, + unittest.TestCase): + compression = zipfile.ZIP_ZSTANDARD + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 38bf08064cfa41..8352dc9ab47fbe 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -640,7 +640,16 @@ def __init__(self): self._unconsumed = b'' self.eof = False - def decompress(self, data): + @property + def _needs_input(self): + # While the LZMA properties header is still being buffered, more input + # is required; afterwards defer to the wrapped decompressor so a bounded + # decompress() call can be drained across reads. + if self._decomp is None: + return True + return self._decomp.needs_input + + def decompress(self, data, max_length=-1): if self._decomp is None: self._unconsumed += data if len(self._unconsumed) <= 4: @@ -656,7 +665,7 @@ def decompress(self, data): data = self._unconsumed[4 + psize:] del self._unconsumed - result = self._decomp.decompress(data) + result = self._decomp.decompress(data, max_length) self.eof = self._decomp.eof return result @@ -716,6 +725,13 @@ def _get_compressor(compress_type, compresslevel=None): return None +def _decompressor_needs_input(decompressor): + # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA + # wrapper keeps it private (_needs_input) to avoid adding public API. + needs_input = getattr(decompressor, "needs_input", None) + return decompressor._needs_input if needs_input is None else needs_input + + def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1012,8 +1028,15 @@ def _read1(self, n): data = self._decompressor.unconsumed_tail if n > len(data): data += self._read2(n - len(data)) - else: + elif self._compress_type == ZIP_STORED: data = self._read2(n) + else: + # bzip2/lzma/zstd: a bounded decompress() call may leave input + # buffered inside the decompressor; drain that before reading more. + if _decompressor_needs_input(self._decompressor): + data = self._read2(n) + else: + data = b'' if self._compress_type == ZIP_STORED: self._eof = self._compress_left <= 0 @@ -1026,8 +1049,13 @@ def _read1(self, n): if self._eof: data += self._decompressor.flush() else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 + # Bound the output of a single decompress() call (mirroring the + # DEFLATE path above) so that a small compressed member cannot + # expand into one unbounded read. + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + _decompressor_needs_input(self._decompressor)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst new file mode 100644 index 00000000000000..4e49ad5ce8fa00 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst @@ -0,0 +1,4 @@ +Bound the amount of data :mod:`zipfile` decompresses per read for members +compressed with bzip2, LZMA, or Zstandard, matching the existing limit for +deflate. A small archive member could previously expand into an unbounded +allocation even when read in small chunks. From 3696a46be8c36cf708c1f88efa03fd8331f3733b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 3 Sep 2026 16:39:39 +0200 Subject: [PATCH 2/5] Remove zstd test (3.14+) --- Lib/test/test_zipfile.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index de97985cb2684b..6c4ba365eb0eea 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2234,12 +2234,6 @@ class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, compression = zipfile.ZIP_LZMA -@requires_zstd() -class ZstdBoundedDecompressTests(AbstractBoundedDecompressTests, - unittest.TestCase): - compression = zipfile.ZIP_ZSTANDARD - - class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" From c6b8aabcb5c73aef402ad5d34257a7440dd09d06 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Sep 2026 06:52:44 -0700 Subject: [PATCH 3/5] gh-156002: Keep reading through monkey-patched zipfile decompressors (GH-157180) (GH-157557) (cherry picked from commit f507e6946a3194e83e1d7b8ee6e14567175e46de) Co-authored-by: Petr Viktorin Co-authored-by: rasmusfaber --- Lib/test/test_zipfile.py | 68 +++++++++++++++++++ Lib/zipfile.py | 19 +++--- ...-09-08-13-06-29.gh-issue-156002.vmOC8T.rst | 5 ++ 3 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 6c4ba365eb0eea..1d069c156ffd40 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2234,6 +2234,74 @@ class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, compression = zipfile.ZIP_LZMA + +class MonkeypatchedDecompressorTests(unittest.TestCase): + # Some third-party projects monkey-patch _get_decompressor() to add + # additional compression schemes. This can break at any time as the + # internal compressor objects change. + # To protect users, we try to keep this case working. + # See also: GH-156002 and GH-113767. + COMPRESSION = 99 + + class Compressor: + """Compressor with only the original BZ2Compressor API""" + def compress(self, data): + return data.swapcase() + + def flush(self): + return b'' + + class Decompressor: + """Decompressor with only the 3.3+ BZ2Decompressor API""" + eof = False + + def decompress(self, data): + return data.swapcase() + + def setUp(self): + orig_check_compression = zipfile._check_compression + orig_get_compressor = zipfile._get_compressor + orig_get_decompressor = zipfile._get_decompressor + + def check_compression(compression): + if compression != self.COMPRESSION: + orig_check_compression(compression) + + def get_compressor(compress_type, compresslevel=None): + if compress_type == self.COMPRESSION: + return self.Compressor() + return orig_get_compressor(compress_type, compresslevel) + + def get_decompressor(compress_type): + if compress_type == self.COMPRESSION: + return self.Decompressor() + return orig_get_decompressor(compress_type) + + self.enterContext(mock.patch.object( + zipfile, '_check_compression', check_compression)) + self.enterContext(mock.patch.object( + zipfile, '_get_compressor', get_compressor)) + self.enterContext(mock.patch.object( + zipfile, '_get_decompressor', get_decompressor)) + + def test_roundtrip_monkeypatched_decompressor(self): + data = bytes(range(256)) * 8 + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: + zf.writestr("member", data) + self.assertIn(data.swapcase(), buf.getvalue()) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + self.assertEqual(zf.read("member"), data) + with zf.open("member") as f: + self.assertEqual(f.read(100), data[:100]) + self.assertEqual(f.read1(100), data[100:200]) + f.seek(-100, os.SEEK_END) + self.assertEqual(f.read(), data[-100:]) + # Rewinding past the read buffer re-creates the decompressor. + f.seek(0) + self.assertEqual(f.read(), data) + + class AbstractBadCrcTests: def test_testzip_with_bad_crc(self): """Tests that files with bad CRCs return their name from testzip.""" diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 8352dc9ab47fbe..33beef20955154 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -641,7 +641,7 @@ def __init__(self): self.eof = False @property - def _needs_input(self): + def needs_input(self): # While the LZMA properties header is still being buffered, more input # is required; afterwards defer to the wrapped decompressor so a bounded # decompress() call can be drained across reads. @@ -725,13 +725,6 @@ def _get_compressor(compress_type, compresslevel=None): return None -def _decompressor_needs_input(decompressor): - # bz2/zstd expose the stdlib decompressor's public needs_input; the LZMA - # wrapper keeps it private (_needs_input) to avoid adding public API. - needs_input = getattr(decompressor, "needs_input", None) - return decompressor._needs_input if needs_input is None else needs_input - - def _get_decompressor(compress_type): _check_compression(compress_type) if compress_type == ZIP_STORED: @@ -1033,7 +1026,7 @@ def _read1(self, n): else: # bzip2/lzma/zstd: a bounded decompress() call may leave input # buffered inside the decompressor; drain that before reading more. - if _decompressor_needs_input(self._decompressor): + if getattr(self._decompressor, "needs_input", True): data = self._read2(n) else: data = b'' @@ -1052,10 +1045,14 @@ def _read1(self, n): # Bound the output of a single decompress() call (mirroring the # DEFLATE path above) so that a small compressed member cannot # expand into one unbounded read. - data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + try: + data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) + except TypeError: + # See MonkeypatchedDecompressorTests in test_core.py + data = self._decompressor.decompress(data) self._eof = (self._decompressor.eof or self._compress_left <= 0 and - _decompressor_needs_input(self._decompressor)) + getattr(self._decompressor, "needs_input", True)) data = data[:self._left] self._left -= len(data) diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst new file mode 100644 index 00000000000000..a21386803cca0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst @@ -0,0 +1,5 @@ +:mod:`zipfile` again reads members through a third-party decompressor +installed by monkey-patching the private ``_get_decompressor()`` to return an +object that only implements old BZ2Decompressor API from Python 3.3. +Note that decompressors without ``needs_input`` and two-argument +``decompress()`` are vulnerable to :cve:`2026-15310`. From 14bfd82b3aae6218263d3f4057d292f64200a247 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 16 Sep 2026 18:08:46 +0200 Subject: [PATCH 4/5] Avoid unittest's enterContext; it doesn't exist yet --- Lib/test/test_zipfile.py | 45 +++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 1d069c156ffd40..fea72705d7fd19 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2258,7 +2258,7 @@ class Decompressor: def decompress(self, data): return data.swapcase() - def setUp(self): + def test_roundtrip_monkeypatched_decompressor(self): orig_check_compression = zipfile._check_compression orig_get_compressor = zipfile._get_compressor orig_get_decompressor = zipfile._get_decompressor @@ -2277,29 +2277,26 @@ def get_decompressor(compress_type): return self.Decompressor() return orig_get_decompressor(compress_type) - self.enterContext(mock.patch.object( - zipfile, '_check_compression', check_compression)) - self.enterContext(mock.patch.object( - zipfile, '_get_compressor', get_compressor)) - self.enterContext(mock.patch.object( - zipfile, '_get_decompressor', get_decompressor)) - - def test_roundtrip_monkeypatched_decompressor(self): - data = bytes(range(256)) * 8 - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: - zf.writestr("member", data) - self.assertIn(data.swapcase(), buf.getvalue()) - with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: - self.assertEqual(zf.read("member"), data) - with zf.open("member") as f: - self.assertEqual(f.read(100), data[:100]) - self.assertEqual(f.read1(100), data[100:200]) - f.seek(-100, os.SEEK_END) - self.assertEqual(f.read(), data[-100:]) - # Rewinding past the read buffer re-creates the decompressor. - f.seek(0) - self.assertEqual(f.read(), data) + with ( + mock.patch.object(zipfile, '_check_compression', check_compression), + mock.patch.object(zipfile, '_get_compressor', get_compressor), + mock.patch.object(zipfile, '_get_decompressor', get_decompressor), + ): + data = bytes(range(256)) * 8 + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: + zf.writestr("member", data) + self.assertIn(data.swapcase(), buf.getvalue()) + with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: + self.assertEqual(zf.read("member"), data) + with zf.open("member") as f: + self.assertEqual(f.read(100), data[:100]) + self.assertEqual(f.read1(100), data[100:200]) + f.seek(-100, os.SEEK_END) + self.assertEqual(f.read(), data[-100:]) + # Rewinding past the read buffer re-creates the decompressor. + f.seek(0) + self.assertEqual(f.read(), data) class AbstractBadCrcTests: From 42b6c97424c4d9b4472640eac7d49026a733edd4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 16 Sep 2026 18:10:58 +0200 Subject: [PATCH 5/5] Don't use the :cve: RST role, it doesn't exist yet --- .../next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst index a21386803cca0f..689967bff72fe7 100644 --- a/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst +++ b/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst @@ -2,4 +2,4 @@ installed by monkey-patching the private ``_get_decompressor()`` to return an object that only implements old BZ2Decompressor API from Python 3.3. Note that decompressors without ``needs_input`` and two-argument -``decompress()`` are vulnerable to :cve:`2026-15310`. +``decompress()`` are vulnerable to CVE 2026-15310.