From ff1d2f4cc5a165b2d4dcc4b5abe6335cb3626e53 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 May 2011 13:43:23 +0200 Subject: [PATCH 1/3] Backport commit 33543b4e0e5d from Python 3.2: #10801: In zipfile, support different encodings for the header and the filenames. Patch by MvL, test by Eli Bendersky. --- Lib/test/test_zipfile.py | 29 ++++++++++++++++++++++++++--- Lib/zipfile.py | 8 +++++++- Misc/NEWS | 3 +++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 34d0fbcb722..825fba1c199 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -3,7 +3,13 @@ try: import zlib except ImportError: zlib = None -import zipfile, os, unittest, sys, shutil, struct, io +import io +import os +import shutil +import struct +import sys +import unittest +import zipfile from tempfile import TemporaryFile from random import randint, random @@ -14,6 +20,7 @@ from test.support import TESTFN, run_unittest, findfile TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" FIXEDTEST_SIZE = 1000 +DATAFILES_DIR = 'zipfile_datafiles' SMALL_TEST_DATA = [('_ziptest1', '1q2w3e4r5t'), ('ziptest2dir/_ziptest2', 'qawsedrftg'), @@ -387,9 +394,25 @@ class TestsWithSourceFile(unittest.TestCase): orig_zip.writestr(zinfo, data) orig_zip.close() + def test_unicode_filenames(self): + if __name__ == '__main__': + myfile = sys.argv[0] + else: + myfile = __file__ + + mydir = os.path.dirname(myfile) or os.curdir + fname = os.path.join(mydir, 'zip_cp437_header.zip') + + print(fname) + zipfp = zipfile.ZipFile(fname) + try: + zipfp.extractall() + finally: + zipfp.close() + def tearDown(self): - os.remove(TESTFN) - os.remove(TESTFN2) + support.unlink(TESTFN) + support.unlink(TESTFN2) class TestZip64InSmallFiles(unittest.TestCase): # These tests test the ZIP64 functionality without using large files, diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 2ec630605f6..a382383e177 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -928,7 +928,13 @@ class ZipFile: if fheader[_FH_EXTRA_FIELD_LENGTH]: zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) - if fname != zinfo.orig_filename.encode("utf-8"): + if zinfo.flag_bits & 0x800: + # UTF-8 filename + fname_str = fname.decode("utf-8") + else: + fname_str = fname.decode("cp437") + + if fname_str != zinfo.orig_filename: raise BadZipfile( 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) diff --git a/Misc/NEWS b/Misc/NEWS index 8f994b9cee7..40317f33777 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -72,6 +72,9 @@ Core and Builtins Library ------- +- Issue #10801: In zipfile, support different encodings for the header and + the filenames. + - Issue #10154, #10090: change the normalization of UTF-8 to "UTF-8" instead of "UTF8" in the locale module as the latter is not supported MacOSX and OpenBSD. From ae8856fe363c65c66f3ae61b57ea2af886c61e32 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 May 2011 13:48:41 +0200 Subject: [PATCH 2/3] Ooops, add the missing file of the backport of commit 33543b4e0e5d from Python 3.2 (#10801: In zipfile, support different encodings for the header and the filenames. Patch by MvL, test by Eli Bendersky.) --- Lib/test/zip_cp437_header.zip | Bin 0 -> 270 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Lib/test/zip_cp437_header.zip diff --git a/Lib/test/zip_cp437_header.zip b/Lib/test/zip_cp437_header.zip new file mode 100644 index 0000000000000000000000000000000000000000..f7c6cf170422c48ce1337a4c05843fe66a4b0bbf GIT binary patch literal 270 zcmWIWW@Zs#U}E54IGj1(Hjhth3p)b?11k`V0&!YqPHJ9aZfbmaW=Tf;#QZkBl8Tbz zujP-QGBE_;)G30lGrzP1MI$4V2m@|Qfx3Y}0Zig>LV!1JBS5x8fC7-R0%8 Date: Fri, 20 May 2011 00:16:09 +0200 Subject: [PATCH 3/3] Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore to be able to unload the module. --- Lib/test/test_zipimport.py | 17 -------------- Misc/NEWS | 3 +++ Modules/zipimport.c | 47 +++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index ebf6ad4df2f..71d5a510eff 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -19,11 +19,6 @@ import io from traceback import extract_tb, extract_stack, print_tb raise_src = 'def do_raise(): raise TypeError\n' -# so we only run testAFakeZlib once if this test is run repeatedly -# which happens when we look for ref leaks -test_imported = False - - def make_pyc(co, mtime): data = marshal.dumps(co) if type(mtime) is type(0.0): @@ -453,19 +448,7 @@ class BadFileZipImportTestCase(unittest.TestCase): zipimport._zip_directory_cache.clear() -def cleanup(): - # this is necessary if test is run repeated (like when finding leaks) - global test_imported - if test_imported: - zipimport._zip_directory_cache.clear() - if hasattr(UncompressedZipImportTestCase, 'testAFakeZlib'): - delattr(UncompressedZipImportTestCase, 'testAFakeZlib') - if hasattr(CompressedZipImportTestCase, 'testAFakeZlib'): - delattr(CompressedZipImportTestCase, 'testAFakeZlib') - test_imported = True - def test_main(): - cleanup() try: support.run_unittest( UncompressedZipImportTestCase, diff --git a/Misc/NEWS b/Misc/NEWS index 40317f33777..9ae7979776c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -72,6 +72,9 @@ Core and Builtins Library ------- +- Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore + to be able to unload the module. + - Issue #10801: In zipfile, support different encodings for the header and the filenames. diff --git a/Modules/zipimport.c b/Modules/zipimport.c index fb5f42b58a0..e874498c7d5 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -785,35 +785,33 @@ error: /* Return the zlib.decompress function object, or NULL if zlib couldn't be imported. The function is cached when found, so subsequent calls - don't import zlib again. Returns a *borrowed* reference. - XXX This makes zlib.decompress immortal. */ + don't import zlib again. */ static PyObject * get_decompress_func(void) { - static PyObject *decompress = NULL; + static int importing_zlib = 0; + PyObject *zlib; + PyObject *decompress; - if (decompress == NULL) { - PyObject *zlib; - static int importing_zlib = 0; - - if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; - let's avoid a stack overflow. */ - return NULL; - importing_zlib = 1; - zlib = PyImport_ImportModuleNoBlock("zlib"); - importing_zlib = 0; - if (zlib != NULL) { - decompress = PyObject_GetAttrString(zlib, - "decompress"); - Py_DECREF(zlib); - } - else - PyErr_Clear(); - if (Py_VerboseFlag) - PySys_WriteStderr("# zipimport: zlib %s\n", - zlib != NULL ? "available": "UNAVAILABLE"); + if (importing_zlib != 0) + /* Someone has a zlib.py[co] in their Zip file; + let's avoid a stack overflow. */ + return NULL; + importing_zlib = 1; + zlib = PyImport_ImportModuleNoBlock("zlib"); + importing_zlib = 0; + if (zlib != NULL) { + decompress = PyObject_GetAttrString(zlib, + "decompress"); + Py_DECREF(zlib); } + else { + PyErr_Clear(); + decompress = NULL; + } + if (Py_VerboseFlag) + PySys_WriteStderr("# zipimport: zlib %s\n", + zlib != NULL ? "available": "UNAVAILABLE"); return decompress; } @@ -904,6 +902,7 @@ get_data(char *archive, PyObject *toc_entry) goto error; } data = PyObject_CallFunction(decompress, "Oi", raw_data, -15); + Py_DECREF(decompress); error: Py_DECREF(raw_data); return data;