From e6f8e0a035f4cbeffb331d90c295ab5894852c39 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 6 May 2025 00:58:47 +0100 Subject: [PATCH] GH-132983: Build ``_zstd`` on Windows (#133366) --- Misc/externals.spdx.json | 21 ++ Modules/Setup | 2 +- Modules/Setup.stdlib.in | 2 +- .../_zstd/clinic/{zdict.c.h => zstddict.c.h} | 0 Modules/_zstd/{zdict.c => zstddict.c} | 2 +- PCbuild/_zstd.vcxproj | 176 +++++++++++++++ PCbuild/_zstd.vcxproj.filters | 206 ++++++++++++++++++ PCbuild/get_externals.bat | 1 + PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 35 +++ PCbuild/python.props | 1 + PCbuild/readme.txt | 4 + PCbuild/regen.targets | 3 +- Tools/msi/freethreaded/freethreaded_files.wxs | 2 +- Tools/msi/lib/lib_files.wxs | 2 +- configure | 2 +- configure.ac | 2 +- 17 files changed, 454 insertions(+), 9 deletions(-) rename Modules/_zstd/clinic/{zdict.c.h => zstddict.c.h} (100%) rename Modules/_zstd/{zdict.c => zstddict.c} (99%) create mode 100644 PCbuild/_zstd.vcxproj create mode 100644 PCbuild/_zstd.vcxproj.filters diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 13be10ca039..69f3beec82e 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -190,6 +190,27 @@ "name": "zlib-ng", "primaryPackagePurpose": "SOURCE", "versionInfo": "2.2.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-zstd", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:facebook:zstandard:1.5.7:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "zstd", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.5.7" } ], "spdxVersion": "SPDX-2.3" diff --git a/Modules/Setup b/Modules/Setup index 81f364abd47..a066982df1a 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -200,8 +200,8 @@ PYTHONPATH=$(COREPYTHONPATH) #_dbm _dbmmodule.c -lgdbm_compat -DUSE_GDBM_COMPAT #_gdbm _gdbmmodule.c -lgdbm #_lzma _lzmamodule.c -llzma -#_zstd _zstd/_zstdmodule.c -lzstd -I$(srcdir)/Modules/_zstd #_uuid _uuidmodule.c -luuid +#_zstd _zstd/_zstdmodule.c -lzstd -I$(srcdir)/Modules/_zstd #zlib zlibmodule.c -lz # The readline module also supports libeditline (-leditline). diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8b676ca69b0..3a38a60a152 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -69,7 +69,7 @@ @MODULE_BINASCII_TRUE@binascii binascii.c @MODULE__BZ2_TRUE@_bz2 _bz2module.c @MODULE__LZMA_TRUE@_lzma _lzmamodule.c -@MODULE__ZSTD_TRUE@_zstd _zstd/_zstdmodule.c _zstd/zdict.c _zstd/compressor.c _zstd/decompressor.c +@MODULE__ZSTD_TRUE@_zstd _zstd/_zstdmodule.c _zstd/zstddict.c _zstd/compressor.c _zstd/decompressor.c @MODULE_ZLIB_TRUE@zlib zlibmodule.c # dbm/gdbm diff --git a/Modules/_zstd/clinic/zdict.c.h b/Modules/_zstd/clinic/zstddict.c.h similarity index 100% rename from Modules/_zstd/clinic/zdict.c.h rename to Modules/_zstd/clinic/zstddict.c.h diff --git a/Modules/_zstd/zdict.c b/Modules/_zstd/zstddict.c similarity index 99% rename from Modules/_zstd/zdict.c rename to Modules/_zstd/zstddict.c index 28ab964a6ca..a19224c4a64 100644 --- a/Modules/_zstd/zdict.c +++ b/Modules/_zstd/zstddict.c @@ -136,7 +136,7 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, } #define clinic_state() (get_zstd_state(type)) -#include "clinic/zdict.c.h" +#include "clinic/zstddict.c.h" #undef clinic_state PyDoc_STRVAR(ZstdDict_dictid_doc, diff --git a/PCbuild/_zstd.vcxproj b/PCbuild/_zstd.vcxproj new file mode 100644 index 00000000000..b53f93a6af8 --- /dev/null +++ b/PCbuild/_zstd.vcxproj @@ -0,0 +1,176 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {07029b86-f3e9-443e-86fb-78aa6d47fed1} + _zstd + Win32Proj + + + + + DynamicLibrary + NotSet + + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + WIN32;ZSTD_MULTITHREAD=1;%(PreprocessorDefinitions) + $(zstdDir)lib\;$(zstdDir)lib\common;$(zstdDir)lib\compress;$(zstdDir)lib\decompress;$(zstdDir)lib\dictBuilder;%(AdditionalIncludeDirectories) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_zstd.vcxproj.filters b/PCbuild/_zstd.vcxproj.filters new file mode 100644 index 00000000000..d4d36063c85 --- /dev/null +++ b/PCbuild/_zstd.vcxproj.filters @@ -0,0 +1,206 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + + + {971714e1-ff37-4240-87bf-a36f6afe764f} + + + {deb43fb6-fa77-4c93-a333-85785fe5f68d} + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + Source Files\zstd + + + + + Header Files + + + Header Files + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + Header Files\zstd + + + + + Resource Files + + + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 49ace616793..e29054f5734 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -61,6 +61,7 @@ if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15. if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-ng-2.2.4 +set libraries=%libraries% zstd-1.5.7 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index da603eed1b3..7a5327bf016 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -69,7 +69,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 45e46095c99..7296ea75301 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -11,6 +11,7 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", "{B11D750F-CD1F-4A96-85CE-E69A5C5259F9}" ProjectSection(ProjectDependencies) = postProject {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} = {01FDF29A-40A1-46DF-84F5-85EBBD2A2410} + {07029B86-F3E9-443E-86FB-78AA6D47FED1} = {07029B86-F3E9-443E-86FB-78AA6D47FED1} {0E9791DB-593A-465F-98BC-681011311617} = {0E9791DB-593A-465F-98BC-681011311617} {0E9791DB-593A-465F-98BC-681011311618} = {0E9791DB-593A-465F-98BC-681011311618} {12728250-16EC-4DC6-94D7-E21DD88947F8} = {12728250-16EC-4DC6-94D7-E21DD88947F8} @@ -165,6 +166,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib-ng", "zlib-ng.vcxproj" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_remote_debugging", "_remote_debugging.vcxproj", "{4D7C112F-3083-4D9E-9754-9341C14D9B39}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_zstd", "_zstd.vcxproj", "{07029B86-F3E9-443E-86FB-78AA6D47FED1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1750,6 +1753,38 @@ Global {4D7C112F-3083-4D9E-9754-9341C14D9B39}.Release|Win32.Build.0 = Release|Win32 {4D7C112F-3083-4D9E-9754-9341C14D9B39}.Release|x64.ActiveCfg = Release|x64 {4D7C112F-3083-4D9E-9754-9341C14D9B39}.Release|x64.Build.0 = Release|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|ARM.ActiveCfg = Debug|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|ARM.Build.0 = Debug|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|ARM64.Build.0 = Debug|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|Win32.ActiveCfg = Debug|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|Win32.Build.0 = Debug|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|x64.ActiveCfg = Debug|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Debug|x64.Build.0 = Debug|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|ARM.ActiveCfg = Release|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|ARM.Build.0 = Release|ARM + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|ARM64.ActiveCfg = Release|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|ARM64.Build.0 = Release|ARM64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|Win32.ActiveCfg = Release|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|Win32.Build.0 = Release|Win32 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|x64.ActiveCfg = Release|x64 + {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/python.props b/PCbuild/python.props index 7cb9ad8e65b..ddc7696d276 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -87,6 +87,7 @@ $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ $(ExternalsDir)\zlib-ng-2.2.4\ + $(ExternalsDir)\zstd-1.5.7\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3bf215d907c..01e19aabdec 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -262,6 +262,10 @@ zlib-ng Sources for zlib-ng are imported unmodified into our source repository at https://github.com/python/cpython-source-deps. +_zstd + Python wrapper for version 1.5.7 of the Zstandard compression library + Homepage: + https://facebook.github.io/zstd/ Getting External Sources diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index e7822a126c6..3ad17737807 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -157,7 +157,8 @@ <_LicenseSources Include="$(PySourcePath)LICENSE; $(PySourcePath)PC\crtlicense.txt; $(bz2Dir)LICENSE; - $(libffiDir)LICENSE;" /> + $(libffiDir)LICENSE; + $(zstdDir)\LICENSE;" /> <_LicenseSources Include="$(opensslOutDir)LICENSE.txt" Condition="Exists('$(opensslOutDir)LICENSE.txt')" /> <_LicenseSources Include="$(opensslOutDir)LICENSE" Condition="!Exists('$(opensslOutDir)LICENSE.txt')" /> <_LicenseSources Include="$(tcltkDir)tcllicense.terms; diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index b3ce28e7aed..86d9a8b83f6 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ - + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index b8e16b5fe23..8439518bcbd 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@  - + diff --git a/configure b/configure index a1069509406..383b8b53b57 100755 --- a/configure +++ b/configure @@ -29769,9 +29769,9 @@ SRCDIRS="\ Modules/_testinternalcapi \ Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ + Modules/_zstd \ Modules/cjkcodecs \ Modules/expat \ - Modules/_zstd \ Objects \ Objects/mimalloc \ Objects/mimalloc/prim \ diff --git a/configure.ac b/configure.ac index 4efd0328691..2c77f3ae9fe 100644 --- a/configure.ac +++ b/configure.ac @@ -7119,9 +7119,9 @@ SRCDIRS="\ Modules/_testinternalcapi \ Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ + Modules/_zstd \ Modules/cjkcodecs \ Modules/expat \ - Modules/_zstd \ Objects \ Objects/mimalloc \ Objects/mimalloc/prim \