bpo-34977: Add Windows App Store package (GH-11027)

Also adds the PC/layout script for generating layouts on Windows.
This commit is contained in:
Steve Dower 2018-12-10 18:52:57 -08:00 committed by GitHub
parent 1c3de541e6
commit 0cd6391fd8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 2928 additions and 405 deletions

View file

@ -37,6 +37,7 @@ set BUILDX64=
set TARGET=Rebuild
set TESTTARGETDIR=
set PGO=-m test -q --pgo
set BUILDMSI=1
set BUILDNUGET=1
set BUILDZIP=1
@ -61,6 +62,7 @@ if "%1" EQU "--pgo" (set PGO=%~2) && shift && shift && goto CheckOpts
if "%1" EQU "--skip-pgo" (set PGO=) && shift && goto CheckOpts
if "%1" EQU "--skip-nuget" (set BUILDNUGET=) && shift && goto CheckOpts
if "%1" EQU "--skip-zip" (set BUILDZIP=) && shift && goto CheckOpts
if "%1" EQU "--skip-msi" (set BUILDMSI=) && shift && goto CheckOpts
if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1
@ -174,10 +176,12 @@ if "%OUTDIR_PLAT%" EQU "win32" (
)
set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI%
%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
if errorlevel 1 exit /B
%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
if errorlevel 1 exit /B
if defined BUILDMSI (
%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
if errorlevel 1 exit /B
%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
if errorlevel 1 exit /B
)
if defined BUILDZIP (
%MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us"
@ -214,6 +218,7 @@ echo --skip-build (-B) Do not build Python (just do the installers)
echo --skip-doc (-D) Do not build documentation
echo --pgo Specify PGO command for x64 installers
echo --skip-pgo Build x64 installers without using PGO
echo --skip-msi Do not build executable/MSI packages
echo --skip-nuget Do not build Nuget packages
echo --skip-zip Do not build embeddable package
echo --download Specify the full download URL for MSIs

71
Tools/msi/make_appx.ps1 Normal file
View file

@ -0,0 +1,71 @@
<#
.Synopsis
Compiles and signs an APPX package
.Description
Given the file listing, ensures all the contents are signed
and builds and signs the final package.
.Parameter mapfile
The location on disk of the text mapping file.
.Parameter msix
The path and name to store the APPX/MSIX.
.Parameter sign
When set, signs the APPX/MSIX. Packages to be published to
the store should not be signed.
.Parameter description
Description to embed in the signature (optional).
.Parameter certname
The name of the certificate to sign with (optional).
.Parameter certsha1
The SHA1 hash of the certificate to sign with (optional).
#>
param(
[Parameter(Mandatory=$true)][string]$layout,
[Parameter(Mandatory=$true)][string]$msix,
[switch]$sign,
[string]$description,
[string]$certname,
[string]$certsha1,
[string]$certfile
)
$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
Set-Alias makeappx (Find-Tool "makeappx.exe") -Scope Script
Set-Alias makepri (Find-Tool "makepri.exe") -Scope Script
$msixdir = Split-Path $msix -Parent
if ($msixdir) {
$msixdir = (mkdir -Force $msixdir).FullName
} else {
$msixdir = Get-Location
}
$msix = Join-Path $msixdir (Split-Path $msix -Leaf)
pushd $layout
try {
if (Test-Path resources.pri) {
del resources.pri
}
$name = ([xml](gc AppxManifest.xml)).Package.Identity.Name
makepri new /pr . /mn AppxManifest.xml /in $name /cf _resources.xml /of _resources.pri /mf appx /o
if (-not $? -or -not (Test-Path _resources.map.txt)) {
throw "makepri step failed"
}
$lines = gc _resources.map.txt
$lines | ?{ -not ($_ -match '"_resources[\w\.]+?"') } | Out-File _resources.map.txt -Encoding utf8
makeappx pack /f _resources.map.txt /m AppxManifest.xml /o /p $msix
if (-not $?) {
throw "makeappx step failed"
}
} finally {
popd
}
if ($sign) {
Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files $msix
if (-not $?) {
throw "Package signing failed"
}
}

34
Tools/msi/make_cat.ps1 Normal file
View file

@ -0,0 +1,34 @@
<#
.Synopsis
Compiles and signs a catalog file.
.Description
Given the CDF definition file, builds and signs a catalog.
.Parameter catalog
The path to the catalog definition file to compile and
sign. It is assumed that the .cat file will be the same
name with a new extension.
.Parameter description
The description to add to the signature (optional).
.Parameter certname
The name of the certificate to sign with (optional).
.Parameter certsha1
The SHA1 hash of the certificate to sign with (optional).
#>
param(
[Parameter(Mandatory=$true)][string]$catalog,
[string]$description,
[string]$certname,
[string]$certsha1,
[string]$certfile
)
$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
Set-Alias MakeCat (Find-Tool "makecat.exe") -Scope Script
MakeCat $catalog
if (-not $?) {
throw "Catalog compilation failed"
}
Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files @($catalog -replace 'cdf$', 'cat')

View file

@ -15,11 +15,12 @@
<TargetExt>.zip</TargetExt>
<TargetPath>$(OutputPath)\$(TargetName)$(TargetExt)</TargetPath>
<CleanCommand>rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)"</CleanCommand>
<Arguments>"$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py"</Arguments>
<Arguments>$(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -b "$(BuildPath.TrimEnd(`\`))"</Arguments>
<Environment>set DOC_FILENAME=python$(PythonVersion).chm</Environment>
<Arguments>"$(PythonExe)" "$(PySourcePath)PC\layout"</Arguments>
<Arguments>$(Arguments) -b "$(BuildPath.TrimEnd(`\`))" -s "$(PySourcePath.TrimEnd(`\`))"</Arguments>
<Arguments>$(Arguments) -t "$(IntermediateOutputPath)\zip_$(ArchName)"</Arguments>
<Arguments>$(Arguments) --zip "$(TargetPath)"</Arguments>
<Arguments>$(Arguments) --precompile --zip-lib --include-underpth --include-stable --flat-dlls</Arguments>
<Environment>$(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib</Environment>
<Environment Condition="Exists($(CRTRedist))">$(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform)</Environment>
</PropertyGroup>
<Target Name="_Build">

View file

@ -1,250 +0,0 @@
import argparse
import py_compile
import re
import sys
import shutil
import stat
import os
import tempfile
from itertools import chain
from pathlib import Path
from zipfile import ZipFile, ZIP_DEFLATED
TKTCL_RE = re.compile(r'^(_?tk|tcl).+\.(pyd|dll)', re.IGNORECASE)
DEBUG_RE = re.compile(r'_d\.(pyd|dll|exe|pdb|lib)$', re.IGNORECASE)
PYTHON_DLL_RE = re.compile(r'python\d\d?\.dll$', re.IGNORECASE)
DEBUG_FILES = {
'_ctypes_test',
'_testbuffer',
'_testcapi',
'_testconsole',
'_testimportmultiple',
'_testmultiphase',
'xxlimited',
'python3_dstub',
}
EXCLUDE_FROM_LIBRARY = {
'__pycache__',
'idlelib',
'pydoc_data',
'site-packages',
'tkinter',
'turtledemo',
}
EXCLUDE_FROM_EMBEDDABLE_LIBRARY = {
'ensurepip',
'venv',
}
EXCLUDE_FILE_FROM_LIBRARY = {
'bdist_wininst.py',
}
EXCLUDE_FILE_FROM_LIBS = {
'liblzma',
'python3stub',
}
EXCLUDED_FILES = {
'pyshellext',
}
def is_not_debug(p):
if DEBUG_RE.search(p.name):
return False
if TKTCL_RE.search(p.name):
return False
return p.stem.lower() not in DEBUG_FILES and p.stem.lower() not in EXCLUDED_FILES
def is_not_debug_or_python(p):
return is_not_debug(p) and not PYTHON_DLL_RE.search(p.name)
def include_in_lib(p):
name = p.name.lower()
if p.is_dir():
if name in EXCLUDE_FROM_LIBRARY:
return False
if name == 'test' and p.parts[-2].lower() == 'lib':
return False
if name in {'test', 'tests'} and p.parts[-3].lower() == 'lib':
return False
return True
if name in EXCLUDE_FILE_FROM_LIBRARY:
return False
suffix = p.suffix.lower()
return suffix not in {'.pyc', '.pyo', '.exe'}
def include_in_embeddable_lib(p):
if p.is_dir() and p.name.lower() in EXCLUDE_FROM_EMBEDDABLE_LIBRARY:
return False
return include_in_lib(p)
def include_in_libs(p):
if not is_not_debug(p):
return False
return p.stem.lower() not in EXCLUDE_FILE_FROM_LIBS
def include_in_tools(p):
if p.is_dir() and p.name.lower() in {'scripts', 'i18n', 'pynche', 'demo', 'parser'}:
return True
return p.suffix.lower() in {'.py', '.pyw', '.txt'}
BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)
FULL_LAYOUT = [
('/', '$build', 'python.exe', is_not_debug),
('/', '$build', 'pythonw.exe', is_not_debug),
('/', '$build', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
('/', '$build', '{}.dll'.format(BASE_NAME), is_not_debug),
('DLLs/', '$build', '*.pyd', is_not_debug),
('DLLs/', '$build', '*.dll', is_not_debug_or_python),
('include/', 'include', '*.h', None),
('include/', 'PC', 'pyconfig.h', None),
('Lib/', 'Lib', '**/*', include_in_lib),
('libs/', '$build', '*.lib', include_in_libs),
('Tools/', 'Tools', '**/*', include_in_tools),
]
EMBED_LAYOUT = [
('/', '$build', 'python*.exe', is_not_debug),
('/', '$build', '*.pyd', is_not_debug),
('/', '$build', '*.dll', is_not_debug),
('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_embeddable_lib),
]
if os.getenv('DOC_FILENAME'):
FULL_LAYOUT.append(('Doc/', 'Doc/build/htmlhelp', os.getenv('DOC_FILENAME'), None))
if os.getenv('VCREDIST_PATH'):
FULL_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))
EMBED_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))
def copy_to_layout(target, rel_sources):
count = 0
if target.suffix.lower() == '.zip':
if target.exists():
target.unlink()
with ZipFile(str(target), 'w', ZIP_DEFLATED) as f:
with tempfile.TemporaryDirectory() as tmpdir:
for s, rel in rel_sources:
if rel.suffix.lower() == '.py':
pyc = Path(tmpdir) / rel.with_suffix('.pyc').name
try:
py_compile.compile(str(s), str(pyc), str(rel), doraise=True, optimize=2)
except py_compile.PyCompileError:
f.write(str(s), str(rel))
else:
f.write(str(pyc), str(rel.with_suffix('.pyc')))
else:
f.write(str(s), str(rel))
count += 1
else:
for s, rel in rel_sources:
dest = target / rel
try:
dest.parent.mkdir(parents=True)
except FileExistsError:
pass
if dest.is_file():
dest.chmod(stat.S_IWRITE)
shutil.copy(str(s), str(dest))
if dest.is_file():
dest.chmod(stat.S_IWRITE)
count += 1
return count
def rglob(root, pattern, condition):
dirs = [root]
recurse = pattern[:3] in {'**/', '**\\'}
while dirs:
d = dirs.pop(0)
for f in d.glob(pattern[3:] if recurse else pattern):
if recurse and f.is_dir() and (not condition or condition(f)):
dirs.append(f)
elif f.is_file() and (not condition or condition(f)):
yield f, f.relative_to(root)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--source', metavar='dir', help='The directory containing the repository root', type=Path)
parser.add_argument('-o', '--out', metavar='file', help='The name of the output archive', type=Path, default=None)
parser.add_argument('-t', '--temp', metavar='dir', help='A directory to temporarily extract files into', type=Path, default=None)
parser.add_argument('-e', '--embed', help='Create an embedding layout', action='store_true', default=False)
parser.add_argument('-b', '--build', help='Specify the build directory', type=Path, default=None)
ns = parser.parse_args()
source = ns.source or (Path(__file__).resolve().parent.parent.parent)
out = ns.out
build = ns.build or Path(sys.exec_prefix)
assert isinstance(source, Path)
assert not out or isinstance(out, Path)
assert isinstance(build, Path)
if ns.temp:
temp = ns.temp
delete_temp = False
else:
temp = Path(tempfile.mkdtemp())
delete_temp = True
if out:
try:
out.parent.mkdir(parents=True)
except FileExistsError:
pass
try:
temp.mkdir(parents=True)
except FileExistsError:
pass
layout = EMBED_LAYOUT if ns.embed else FULL_LAYOUT
try:
for t, s, p, c in layout:
if s == '$build':
fs = build
else:
fs = source / s
files = rglob(fs, p, c)
extra_files = []
if s == 'Lib' and p == '**/*':
extra_files.append((
source / 'tools' / 'msi' / 'distutils.command.bdist_wininst.py',
Path('distutils') / 'command' / 'bdist_wininst.py'
))
copied = copy_to_layout(temp / t.rstrip('/'), chain(files, extra_files))
print('Copied {} files'.format(copied))
if ns.embed:
with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
print(BASE_NAME + '.zip', file=f)
print('.', file=f)
print('', file=f)
print('# Uncomment to run site.main() automatically', file=f)
print('#import site', file=f)
if out:
total = copy_to_layout(out, rglob(temp, '**/*', None))
print('Wrote {} files to {}'.format(total, out))
finally:
if delete_temp:
shutil.rmtree(temp, True)
if __name__ == "__main__":
sys.exit(int(main() or 0))

43
Tools/msi/sdktools.psm1 Normal file
View file

@ -0,0 +1,43 @@
function Find-Tool {
param([string]$toolname)
$kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
$tool = (gci -r "$kitroot\Bin\*\x64\$toolname" | sort FullName -Desc | select -First 1)
if (-not $tool) {
throw "$toolname is not available"
}
Write-Host "Found $toolname at $($tool.FullName)"
return $tool.FullName
}
Set-Alias SignTool (Find-Tool "signtool.exe") -Scope Script
function Sign-File {
param([string]$certname, [string]$certsha1, [string]$certfile, [string]$description, [string[]]$files)
if (-not $description) {
$description = $env:SigningDescription;
if (-not $description) {
$description = "Python";
}
}
if (-not $certname) {
$certname = $env:SigningCertificate;
}
if (-not $certfile) {
$certfile = $env:SigningCertificateFile;
}
foreach ($a in $files) {
if ($certsha1) {
SignTool sign /sha1 $certsha1 /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
} elseif ($certname) {
SignTool sign /n $certname /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
} elseif ($certfile) {
SignTool sign /f $certfile /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
} else {
SignTool sign /a /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
}
}
}

34
Tools/msi/sign_build.ps1 Normal file
View file

@ -0,0 +1,34 @@
<#
.Synopsis
Recursively signs the contents of a directory.
.Description
Given the file patterns, code signs the contents.
.Parameter root
The root directory to sign.
.Parameter patterns
The file patterns to sign
.Parameter description
The description to add to the signature (optional).
.Parameter certname
The name of the certificate to sign with (optional).
.Parameter certsha1
The SHA1 hash of the certificate to sign with (optional).
#>
param(
[Parameter(Mandatory=$true)][string]$root,
[string[]]$patterns=@("*.exe", "*.dll", "*.pyd"),
[string]$description,
[string]$certname,
[string]$certsha1,
[string]$certfile
)
$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
pushd $root
try {
Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files (gci -r $patterns)
} finally {
popd
}