mirror of
https://github.com/python/cpython.git
synced 2025-08-04 17:08:35 +00:00
gh-112302: Move pip SBOM discovery to release-tools (#115360)
This commit is contained in:
Seth Michael Larson
GitHub
parent
49e8fdc1df
commit
889cc43cb1
2 changed files with 0 additions and 921 deletions
|
|
@ -53,8 +53,6 @@ class PackageFiles(typing.NamedTuple):
|
|||
# values to 'exclude' if we create new files within tracked
|
||||
# directories that aren't sourced from third-party packages.
|
||||
PACKAGE_TO_FILES = {
|
||||
# NOTE: pip's entry in this structure is automatically generated in
|
||||
# the 'discover_pip_sbom_package()' function below.
|
||||
"mpdecimal": PackageFiles(
|
||||
include=["Modules/_decimal/libmpdec/**"]
|
||||
),
|
||||
|
|
@ -127,264 +125,15 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
|
|||
return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])
|
||||
|
||||
|
||||
def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None:
|
||||
"""
|
||||
Fetches the SHA256 checksum and download location from PyPI.
|
||||
If we're given a filename then we match with that, otherwise we use wheels.
|
||||
"""
|
||||
# Get pip's download location from PyPI. Check that the checksum is correct too.
|
||||
try:
|
||||
raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read()
|
||||
release_metadata = json.loads(raw_text)
|
||||
url: dict[str, typing.Any]
|
||||
|
||||
# Look for a matching artifact filename and then check
|
||||
# its remote checksum to the local one.
|
||||
for url in release_metadata["urls"]:
|
||||
# pip can only use Python-only dependencies, so there's
|
||||
# no risk of picking the 'incorrect' wheel here.
|
||||
if (
|
||||
(filename is None and url["packagetype"] == "bdist_wheel")
|
||||
or (filename is not None and url["filename"] == filename)
|
||||
):
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"No matching filename on PyPI for '{filename}'")
|
||||
|
||||
# Successfully found the download URL for the matching artifact.
|
||||
download_url = url["url"]
|
||||
checksum_sha256 = url["digests"]["sha256"]
|
||||
return download_url, checksum_sha256
|
||||
|
||||
except (OSError, ValueError) as e:
|
||||
# Fail if we're running in CI where we should have an internet connection.
|
||||
error_if(
|
||||
"CI" in os.environ,
|
||||
f"Couldn't fetch metadata for project '{project}' from PyPI: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def find_ensurepip_pip_wheel() -> pathlib.Path | None:
|
||||
"""Try to find the pip wheel bundled in ensurepip. If missing return None"""
|
||||
|
||||
ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled"
|
||||
|
||||
pip_wheels = []
|
||||
try:
|
||||
for wheel_filename in os.listdir(ensurepip_bundled_dir):
|
||||
if wheel_filename.startswith("pip-"):
|
||||
pip_wheels.append(wheel_filename)
|
||||
else:
|
||||
print(f"Unexpected wheel in ensurepip: '{wheel_filename}'")
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore this error, likely caused by downstream distributors
|
||||
# deleting the 'ensurepip/_bundled' directory.
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
if len(pip_wheels) == 0:
|
||||
return None
|
||||
elif len(pip_wheels) > 1:
|
||||
print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'")
|
||||
sys.exit(1)
|
||||
# Otherwise return the one pip wheel.
|
||||
return ensurepip_bundled_dir / pip_wheels[0]
|
||||
|
||||
|
||||
def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None:
|
||||
"""
|
||||
Removes pip and its dependencies from the SBOM data
|
||||
if the pip wheel is removed from ensurepip. This is done
|
||||
by redistributors of Python and pip.
|
||||
"""
|
||||
|
||||
# If there's a wheel we don't remove anything.
|
||||
if find_ensurepip_pip_wheel() is not None:
|
||||
return
|
||||
|
||||
# Otherwise we traverse the relationships
|
||||
# to find dependent packages to remove.
|
||||
sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
|
||||
sbom_spdx_ids_to_remove = {sbom_pip_spdx_id}
|
||||
|
||||
# Find all package SPDXIDs that pip depends on.
|
||||
for sbom_relationship in sbom_data["relationships"]:
|
||||
if (
|
||||
sbom_relationship["relationshipType"] == "DEPENDS_ON"
|
||||
and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id
|
||||
):
|
||||
sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"])
|
||||
|
||||
# Remove all the packages and relationships.
|
||||
sbom_data["packages"] = [
|
||||
sbom_package for sbom_package in sbom_data["packages"]
|
||||
if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove
|
||||
]
|
||||
sbom_data["relationships"] = [
|
||||
sbom_relationship for sbom_relationship in sbom_data["relationships"]
|
||||
if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove
|
||||
]
|
||||
|
||||
|
||||
def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
|
||||
"""pip is a part of a packaging ecosystem (Python, surprise!) so it's actually
|
||||
automatable to discover the metadata we need like the version and checksums
|
||||
so let's do that on behalf of our friends at the PyPA. This function also
|
||||
discovers vendored packages within pip and fetches their metadata.
|
||||
"""
|
||||
global PACKAGE_TO_FILES
|
||||
|
||||
pip_wheel_filepath = find_ensurepip_pip_wheel()
|
||||
if pip_wheel_filepath is None:
|
||||
return # There's no pip wheel, nothing to discover.
|
||||
|
||||
# Add the wheel filename to the list of files so the SBOM file
|
||||
# and relationship generator can work its magic on the wheel too.
|
||||
PACKAGE_TO_FILES["pip"] = PackageFiles(
|
||||
include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))]
|
||||
)
|
||||
|
||||
# Wheel filename format puts the version right after the project name.
|
||||
pip_version = pip_wheel_filepath.name.split("-")[1]
|
||||
pip_checksum_sha256 = hashlib.sha256(
|
||||
pip_wheel_filepath.read_bytes()
|
||||
).hexdigest()
|
||||
|
||||
pip_metadata = fetch_package_metadata_from_pypi(
|
||||
project="pip",
|
||||
version=pip_version,
|
||||
filename=pip_wheel_filepath.name,
|
||||
)
|
||||
# We couldn't fetch any metadata from PyPI,
|
||||
# so we give up on verifying if we're not in CI.
|
||||
if pip_metadata is None:
|
||||
return
|
||||
|
||||
pip_download_url, pip_actual_sha256 = pip_metadata
|
||||
if pip_actual_sha256 != pip_checksum_sha256:
|
||||
raise ValueError("Unexpected")
|
||||
|
||||
# Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies.
|
||||
with zipfile.ZipFile(pip_wheel_filepath) as whl:
|
||||
vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode()
|
||||
|
||||
# With this version regex we're assuming that pip isn't using pre-releases.
|
||||
# If any version doesn't match we get a failure below, so we're safe doing this.
|
||||
version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$")
|
||||
sbom_pip_dependency_spdx_ids = set()
|
||||
for line in vendor_txt_data.splitlines():
|
||||
line = line.partition("#")[0].strip() # Strip comments and whitespace.
|
||||
if not line: # Skip empty lines.
|
||||
continue
|
||||
|
||||
# Non-empty lines we must be able to match.
|
||||
match = version_pin_re.match(line)
|
||||
error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'")
|
||||
assert match is not None # Make mypy happy.
|
||||
|
||||
# Parse out and normalize the project name.
|
||||
project_name, project_version = match.groups()
|
||||
project_name = project_name.lower()
|
||||
|
||||
# At this point if pip's metadata fetch succeeded we should
|
||||
# expect this request to also succeed.
|
||||
project_metadata = (
|
||||
fetch_package_metadata_from_pypi(project_name, project_version)
|
||||
)
|
||||
assert project_metadata is not None
|
||||
project_download_url, project_checksum_sha256 = project_metadata
|
||||
|
||||
# Update our SBOM data with what we received from PyPI.
|
||||
# Don't overwrite any existing values.
|
||||
sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}")
|
||||
sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id)
|
||||
for package in sbom_data["packages"]:
|
||||
if package["SPDXID"] != sbom_project_spdx_id:
|
||||
continue
|
||||
|
||||
# Only thing missing from this blob is the `licenseConcluded`,
|
||||
# that needs to be triaged by human maintainers if the list changes.
|
||||
package.update({
|
||||
"SPDXID": sbom_project_spdx_id,
|
||||
"name": project_name,
|
||||
"versionInfo": project_version,
|
||||
"downloadLocation": project_download_url,
|
||||
"checksums": [
|
||||
{"algorithm": "SHA256", "checksumValue": project_checksum_sha256}
|
||||
],
|
||||
"externalRefs": [
|
||||
{
|
||||
"referenceCategory": "PACKAGE_MANAGER",
|
||||
"referenceLocator": f"pkg:pypi/{project_name}@{project_version}",
|
||||
"referenceType": "purl",
|
||||
},
|
||||
],
|
||||
"primaryPackagePurpose": "SOURCE"
|
||||
})
|
||||
break
|
||||
|
||||
PACKAGE_TO_FILES[project_name] = PackageFiles(include=None)
|
||||
|
||||
# Remove pip from the existing SBOM packages if it's there
|
||||
# and then overwrite its entry with our own generated one.
|
||||
sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
|
||||
sbom_data["packages"] = [
|
||||
sbom_package
|
||||
for sbom_package in sbom_data["packages"]
|
||||
if sbom_package["name"] != "pip"
|
||||
]
|
||||
sbom_data["packages"].append(
|
||||
{
|
||||
"SPDXID": sbom_pip_spdx_id,
|
||||
"name": "pip",
|
||||
"versionInfo": pip_version,
|
||||
"originator": "Organization: Python Packaging Authority",
|
||||
"licenseConcluded": "NOASSERTION",
|
||||
"downloadLocation": pip_download_url,
|
||||
"checksums": [
|
||||
{"algorithm": "SHA256", "checksumValue": pip_checksum_sha256}
|
||||
],
|
||||
"externalRefs": [
|
||||
{
|
||||
"referenceCategory": "SECURITY",
|
||||
"referenceLocator": f"cpe:2.3:a:pypa:pip:{pip_version}:*:*:*:*:*:*:*",
|
||||
"referenceType": "cpe23Type",
|
||||
},
|
||||
{
|
||||
"referenceCategory": "PACKAGE_MANAGER",
|
||||
"referenceLocator": f"pkg:pypi/pip@{pip_version}",
|
||||
"referenceType": "purl",
|
||||
},
|
||||
],
|
||||
"primaryPackagePurpose": "SOURCE",
|
||||
}
|
||||
)
|
||||
for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids):
|
||||
sbom_data["relationships"].append({
|
||||
"spdxElementId": sbom_pip_spdx_id,
|
||||
"relatedSpdxElement": sbom_dep_spdx_id,
|
||||
"relationshipType": "DEPENDS_ON"
|
||||
})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
|
||||
sbom_data = json.loads(sbom_path.read_bytes())
|
||||
|
||||
# Check if pip should be removed if the wheel is missing.
|
||||
# We can't reset the SBOM relationship data until checking this.
|
||||
maybe_remove_pip_and_deps_from_sbom(sbom_data)
|
||||
|
||||
# We regenerate all of this information. Package information
|
||||
# should be preserved though since that is edited by humans.
|
||||
sbom_data["files"] = []
|
||||
sbom_data["relationships"] = []
|
||||
|
||||
# Insert pip's SBOM metadata from the wheel.
|
||||
discover_pip_sbom_package(sbom_data)
|
||||
|
||||
# Ensure all packages in this tool are represented also in the SBOM file.
|
||||
actual_names = {package["name"] for package in sbom_data["packages"]}
|
||||
expected_names = set(PACKAGE_TO_FILES)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue