mirror of
https://github.com/python/cpython.git
synced 2025-07-07 19:35:27 +00:00
gh-132983: Introduce compression
package and move _compression
module (GH-133018)
* Introduces `compression` package for https://peps.python.org/pep-0784/ This commit introduces the `compression` package, specified in PEP 784 to re-export the `lzma`, `bz2`, `gzip`, and `zlib` modules. Introduction of `compression.zstd` will be completed in a future commit once the `_zstd` module is merged. This commit also moves the `_compression` private module to `compression._common._streams`. * Re-exports existing module docstrings.
This commit is contained in:
parent
6d53b75283
commit
20be6ba61a
12 changed files with 42 additions and 22 deletions
|
@ -10,9 +10,9 @@ __all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
|
||||||
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
|
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
|
||||||
|
|
||||||
from builtins import open as _builtin_open
|
from builtins import open as _builtin_open
|
||||||
|
from compression._common import _streams
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
import _compression
|
|
||||||
|
|
||||||
from _bz2 import BZ2Compressor, BZ2Decompressor
|
from _bz2 import BZ2Compressor, BZ2Decompressor
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ _MODE_READ = 1
|
||||||
_MODE_WRITE = 3
|
_MODE_WRITE = 3
|
||||||
|
|
||||||
|
|
||||||
class BZ2File(_compression.BaseStream):
|
class BZ2File(_streams.BaseStream):
|
||||||
|
|
||||||
"""A file object providing transparent bzip2 (de)compression.
|
"""A file object providing transparent bzip2 (de)compression.
|
||||||
|
|
||||||
|
@ -88,7 +88,7 @@ class BZ2File(_compression.BaseStream):
|
||||||
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
||||||
|
|
||||||
if self._mode == _MODE_READ:
|
if self._mode == _MODE_READ:
|
||||||
raw = _compression.DecompressReader(self._fp,
|
raw = _streams.DecompressReader(self._fp,
|
||||||
BZ2Decompressor, trailing_error=OSError)
|
BZ2Decompressor, trailing_error=OSError)
|
||||||
self._buffer = io.BufferedReader(raw)
|
self._buffer = io.BufferedReader(raw)
|
||||||
else:
|
else:
|
||||||
|
@ -248,7 +248,7 @@ class BZ2File(_compression.BaseStream):
|
||||||
|
|
||||||
Line separators are not added between the written byte strings.
|
Line separators are not added between the written byte strings.
|
||||||
"""
|
"""
|
||||||
return _compression.BaseStream.writelines(self, seq)
|
return _streams.BaseStream.writelines(self, seq)
|
||||||
|
|
||||||
def seek(self, offset, whence=io.SEEK_SET):
|
def seek(self, offset, whence=io.SEEK_SET):
|
||||||
"""Change the file position.
|
"""Change the file position.
|
||||||
|
|
0
Lib/compression/__init__.py
Normal file
0
Lib/compression/__init__.py
Normal file
|
@ -1,4 +1,4 @@
|
||||||
"""Internal classes used by the gzip, lzma and bz2 modules"""
|
"""Internal classes used by compression modules"""
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import sys
|
import sys
|
5
Lib/compression/bz2/__init__.py
Normal file
5
Lib/compression/bz2/__init__.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
import bz2
|
||||||
|
__doc__ = bz2.__doc__
|
||||||
|
del bz2
|
||||||
|
|
||||||
|
from bz2 import *
|
5
Lib/compression/gzip/__init__.py
Normal file
5
Lib/compression/gzip/__init__.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
import gzip
|
||||||
|
__doc__ = gzip.__doc__
|
||||||
|
del gzip
|
||||||
|
|
||||||
|
from gzip import *
|
5
Lib/compression/lzma/__init__.py
Normal file
5
Lib/compression/lzma/__init__.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
import lzma
|
||||||
|
__doc__ = lzma.__doc__
|
||||||
|
del lzma
|
||||||
|
|
||||||
|
from lzma import *
|
5
Lib/compression/zlib/__init__.py
Normal file
5
Lib/compression/zlib/__init__.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
import zlib
|
||||||
|
__doc__ = zlib.__doc__
|
||||||
|
del zlib
|
||||||
|
|
||||||
|
from zlib import *
|
|
@ -5,7 +5,6 @@ but random access is not allowed."""
|
||||||
|
|
||||||
# based on Andrew Kuchling's minigzip.py distributed with the zlib module
|
# based on Andrew Kuchling's minigzip.py distributed with the zlib module
|
||||||
|
|
||||||
import _compression
|
|
||||||
import builtins
|
import builtins
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
|
@ -14,6 +13,7 @@ import sys
|
||||||
import time
|
import time
|
||||||
import weakref
|
import weakref
|
||||||
import zlib
|
import zlib
|
||||||
|
from compression._common import _streams
|
||||||
|
|
||||||
__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
|
__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
|
||||||
|
|
||||||
|
@ -144,7 +144,7 @@ class _WriteBufferStream(io.RawIOBase):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
class GzipFile(_compression.BaseStream):
|
class GzipFile(_streams.BaseStream):
|
||||||
"""The GzipFile class simulates most of the methods of a file object with
|
"""The GzipFile class simulates most of the methods of a file object with
|
||||||
the exception of the truncate() method.
|
the exception of the truncate() method.
|
||||||
|
|
||||||
|
@ -523,7 +523,7 @@ def _read_gzip_header(fp):
|
||||||
return last_mtime
|
return last_mtime
|
||||||
|
|
||||||
|
|
||||||
class _GzipReader(_compression.DecompressReader):
|
class _GzipReader(_streams.DecompressReader):
|
||||||
def __init__(self, fp):
|
def __init__(self, fp):
|
||||||
super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor,
|
super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor,
|
||||||
wbits=-zlib.MAX_WBITS)
|
wbits=-zlib.MAX_WBITS)
|
||||||
|
|
|
@ -24,9 +24,9 @@ __all__ = [
|
||||||
import builtins
|
import builtins
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
|
from compression._common import _streams
|
||||||
from _lzma import *
|
from _lzma import *
|
||||||
from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401
|
from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401
|
||||||
import _compression
|
|
||||||
|
|
||||||
|
|
||||||
# Value 0 no longer used
|
# Value 0 no longer used
|
||||||
|
@ -35,7 +35,7 @@ _MODE_READ = 1
|
||||||
_MODE_WRITE = 3
|
_MODE_WRITE = 3
|
||||||
|
|
||||||
|
|
||||||
class LZMAFile(_compression.BaseStream):
|
class LZMAFile(_streams.BaseStream):
|
||||||
|
|
||||||
"""A file object providing transparent LZMA (de)compression.
|
"""A file object providing transparent LZMA (de)compression.
|
||||||
|
|
||||||
|
@ -127,7 +127,7 @@ class LZMAFile(_compression.BaseStream):
|
||||||
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
raise TypeError("filename must be a str, bytes, file or PathLike object")
|
||||||
|
|
||||||
if self._mode == _MODE_READ:
|
if self._mode == _MODE_READ:
|
||||||
raw = _compression.DecompressReader(self._fp, LZMADecompressor,
|
raw = _streams.DecompressReader(self._fp, LZMADecompressor,
|
||||||
trailing_error=LZMAError, format=format, filters=filters)
|
trailing_error=LZMAError, format=format, filters=filters)
|
||||||
self._buffer = io.BufferedReader(raw)
|
self._buffer = io.BufferedReader(raw)
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,7 @@ import threading
|
||||||
from test.support import import_helper
|
from test.support import import_helper
|
||||||
from test.support import threading_helper
|
from test.support import threading_helper
|
||||||
from test.support.os_helper import unlink, FakePath
|
from test.support.os_helper import unlink, FakePath
|
||||||
import _compression
|
from compression._common import _streams
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
@ -126,15 +126,15 @@ class BZ2FileTest(BaseTest):
|
||||||
def testReadMonkeyMultiStream(self):
|
def testReadMonkeyMultiStream(self):
|
||||||
# Test BZ2File.read() on a multi-stream archive where a stream
|
# Test BZ2File.read() on a multi-stream archive where a stream
|
||||||
# boundary coincides with the end of the raw read buffer.
|
# boundary coincides with the end of the raw read buffer.
|
||||||
buffer_size = _compression.BUFFER_SIZE
|
buffer_size = _streams.BUFFER_SIZE
|
||||||
_compression.BUFFER_SIZE = len(self.DATA)
|
_streams.BUFFER_SIZE = len(self.DATA)
|
||||||
try:
|
try:
|
||||||
self.createTempFile(streams=5)
|
self.createTempFile(streams=5)
|
||||||
with BZ2File(self.filename) as bz2f:
|
with BZ2File(self.filename) as bz2f:
|
||||||
self.assertRaises(TypeError, bz2f.read, float())
|
self.assertRaises(TypeError, bz2f.read, float())
|
||||||
self.assertEqual(bz2f.read(), self.TEXT * 5)
|
self.assertEqual(bz2f.read(), self.TEXT * 5)
|
||||||
finally:
|
finally:
|
||||||
_compression.BUFFER_SIZE = buffer_size
|
_streams.BUFFER_SIZE = buffer_size
|
||||||
|
|
||||||
def testReadTrailingJunk(self):
|
def testReadTrailingJunk(self):
|
||||||
self.createTempFile(suffix=self.BAD_DATA)
|
self.createTempFile(suffix=self.BAD_DATA)
|
||||||
|
@ -742,7 +742,7 @@ class BZ2FileTest(BaseTest):
|
||||||
def testDecompressLimited(self):
|
def testDecompressLimited(self):
|
||||||
"""Decompressed data buffering should be limited"""
|
"""Decompressed data buffering should be limited"""
|
||||||
bomb = bz2.compress(b'\0' * int(2e6), compresslevel=9)
|
bomb = bz2.compress(b'\0' * int(2e6), compresslevel=9)
|
||||||
self.assertLess(len(bomb), _compression.BUFFER_SIZE)
|
self.assertLess(len(bomb), _streams.BUFFER_SIZE)
|
||||||
|
|
||||||
decomp = BZ2File(BytesIO(bomb))
|
decomp = BZ2File(BytesIO(bomb))
|
||||||
self.assertEqual(decomp.read(1), b'\0')
|
self.assertEqual(decomp.read(1), b'\0')
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import _compression
|
|
||||||
import array
|
import array
|
||||||
from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE
|
from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE
|
||||||
import os
|
import os
|
||||||
|
@ -7,6 +6,7 @@ import random
|
||||||
import sys
|
import sys
|
||||||
from test import support
|
from test import support
|
||||||
import unittest
|
import unittest
|
||||||
|
from compression._common import _streams
|
||||||
|
|
||||||
from test.support import _4G, bigmemtest
|
from test.support import _4G, bigmemtest
|
||||||
from test.support.import_helper import import_module
|
from test.support.import_helper import import_module
|
||||||
|
@ -861,13 +861,13 @@ class FileTestCase(unittest.TestCase):
|
||||||
def test_read_multistream_buffer_size_aligned(self):
|
def test_read_multistream_buffer_size_aligned(self):
|
||||||
# Test the case where a stream boundary coincides with the end
|
# Test the case where a stream boundary coincides with the end
|
||||||
# of the raw read buffer.
|
# of the raw read buffer.
|
||||||
saved_buffer_size = _compression.BUFFER_SIZE
|
saved_buffer_size = _streams.BUFFER_SIZE
|
||||||
_compression.BUFFER_SIZE = len(COMPRESSED_XZ)
|
_streams.BUFFER_SIZE = len(COMPRESSED_XZ)
|
||||||
try:
|
try:
|
||||||
with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
|
with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
|
||||||
self.assertEqual(f.read(), INPUT * 5)
|
self.assertEqual(f.read(), INPUT * 5)
|
||||||
finally:
|
finally:
|
||||||
_compression.BUFFER_SIZE = saved_buffer_size
|
_streams.BUFFER_SIZE = saved_buffer_size
|
||||||
|
|
||||||
def test_read_trailing_junk(self):
|
def test_read_trailing_junk(self):
|
||||||
with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f:
|
with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f:
|
||||||
|
@ -1066,7 +1066,7 @@ class FileTestCase(unittest.TestCase):
|
||||||
def test_decompress_limited(self):
|
def test_decompress_limited(self):
|
||||||
"""Decompressed data buffering should be limited"""
|
"""Decompressed data buffering should be limited"""
|
||||||
bomb = lzma.compress(b'\0' * int(2e6), preset=6)
|
bomb = lzma.compress(b'\0' * int(2e6), preset=6)
|
||||||
self.assertLess(len(bomb), _compression.BUFFER_SIZE)
|
self.assertLess(len(bomb), _streams.BUFFER_SIZE)
|
||||||
|
|
||||||
decomp = LZMAFile(BytesIO(bomb))
|
decomp = LZMAFile(BytesIO(bomb))
|
||||||
self.assertEqual(decomp.read(1), b'\0')
|
self.assertEqual(decomp.read(1), b'\0')
|
||||||
|
|
2
Python/stdlib_module_names.h
generated
2
Python/stdlib_module_names.h
generated
|
@ -24,7 +24,6 @@ static const char* _Py_stdlib_module_names[] = {
|
||||||
"_collections_abc",
|
"_collections_abc",
|
||||||
"_colorize",
|
"_colorize",
|
||||||
"_compat_pickle",
|
"_compat_pickle",
|
||||||
"_compression",
|
|
||||||
"_contextvars",
|
"_contextvars",
|
||||||
"_csv",
|
"_csv",
|
||||||
"_ctypes",
|
"_ctypes",
|
||||||
|
@ -128,6 +127,7 @@ static const char* _Py_stdlib_module_names[] = {
|
||||||
"collections",
|
"collections",
|
||||||
"colorsys",
|
"colorsys",
|
||||||
"compileall",
|
"compileall",
|
||||||
|
"compression",
|
||||||
"concurrent",
|
"concurrent",
|
||||||
"configparser",
|
"configparser",
|
||||||
"contextlib",
|
"contextlib",
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue