GH-107956: install build-details.json (PEP 739) (#130069)

Lib/sysconfig/__init__.py

@@ -666,34 +666,34 @@ def get_platform():
 
     # Set for cross builds explicitly
     if "_PYTHON_HOST_PLATFORM" in os.environ:
-        return os.environ["_PYTHON_HOST_PLATFORM"]
+        osname, _, machine = os.environ["_PYTHON_HOST_PLATFORM"].partition('-')
+        release = None
+    else:
+        # Try to distinguish various flavours of Unix
+        osname, host, release, version, machine = os.uname()
 
-    # Try to distinguish various flavours of Unix
+        # Convert the OS name to lowercase, remove '/' characters, and translate
-    osname, host, release, version, machine = os.uname()
+        # spaces (for "Power Macintosh")
+        osname = osname.lower().replace('/', '')
+        machine = machine.replace(' ', '_')
+        machine = machine.replace('/', '-')
 
-    # Convert the OS name to lowercase, remove '/' characters, and translate
+    if osname == "android" or sys.platform == "android":
-    # spaces (for "Power Macintosh")
+        osname = "android"
-    osname = osname.lower().replace('/', '')
+        release = get_config_var("ANDROID_API_LEVEL")
-    machine = machine.replace(' ', '_')
-    machine = machine.replace('/', '-')
 
-    if osname[:5] == "linux":
+        # Wheel tags use the ABI names from Android's own tools.
-        if sys.platform == "android":
+        machine = {
-            osname = "android"
+            "x86_64": "x86_64",
-            release = get_config_var("ANDROID_API_LEVEL")
+            "i686": "x86",
-
+            "aarch64": "arm64_v8a",
-            # Wheel tags use the ABI names from Android's own tools.
+            "armv7l": "armeabi_v7a",
-            machine = {
+        }[machine]
-                "x86_64": "x86_64",
+    elif osname == "linux":
-                "i686": "x86",
+        # At least on Linux/Intel, 'machine' is the processor --
-                "aarch64": "arm64_v8a",
+        # i386, etc.
-                "armv7l": "armeabi_v7a",
+        # XXX what about Alpha, SPARC, etc?
-            }[machine]
+        return  f"{osname}-{machine}"
-        else:
-            # At least on Linux/Intel, 'machine' is the processor --
-            # i386, etc.
-            # XXX what about Alpha, SPARC, etc?
-            return  f"{osname}-{machine}"
     elif osname[:5] == "sunos":
         if release[0] >= "5":           # SunOS 5 == Solaris 2
             osname = "solaris"
@@ -725,7 +725,7 @@ def get_platform():
                                                 get_config_vars(),
                                                 osname, release, machine)
 
-    return f"{osname}-{release}-{machine}"
+    return '-'.join(map(str, filter(None, (osname, release, machine))))
 
 
 def get_python_version():

Lib/test/test_build_details.py

@@ -0,0 +1,128 @@
+import json
+import os
+import sys
+import sysconfig
+import string
+import unittest
+
+from test.support import is_android, is_apple_mobile, is_emscripten, is_wasi
+
+
+class FormatTestsBase:
+    @property
+    def contents(self):
+        """Install details file contents. Should be overriden by subclasses."""
+        raise NotImplementedError
+
+    @property
+    def data(self):
+        """Parsed install details file data, as a Python object."""
+        return json.loads(self.contents)
+
+    def key(self, name):
+        """Helper to fetch subsection entries.
+
+        It takes the entry name, allowing the usage of a dot to separate the
+        different subsection names (eg. specifying 'a.b.c' as the key will
+        return the value of self.data['a']['b']['c']).
+        """
+        value = self.data
+        for part in name.split('.'):
+            value = value[part]
+        return value
+
+    def test_parse(self):
+        self.data
+
+    def test_top_level_container(self):
+        self.assertIsInstance(self.data, dict)
+        for key, value in self.data.items():
+            with self.subTest(key=key):
+                if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'):
+                    self.assertIsInstance(value, str)
+                elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'):
+                    self.assertIsInstance(value, dict)
+
+    def test_base_prefix(self):
+        self.assertIsInstance(self.key('base_prefix'), str)
+
+    def test_base_interpreter(self):
+        """Test the base_interpreter entry.
+
+        The generic test wants the key to be missing. If your implementation
+        provides a value for it, you should override this test.
+        """
+        with self.assertRaises(KeyError):
+            self.key('base_interpreter')
+
+    def test_platform(self):
+        self.assertEqual(self.key('platform'), sysconfig.get_platform())
+
+    def test_language_version(self):
+        allowed_characters = string.digits + string.ascii_letters + '.'
+        value = self.key('language.version')
+
+        self.assertLessEqual(set(value), set(allowed_characters))
+        self.assertTrue(sys.version.startswith(value))
+
+    def test_language_version_info(self):
+        value = self.key('language.version_info')
+
+        self.assertEqual(len(value), sys.version_info.n_fields)
+        for part_name, part_value in value.items():
+            with self.subTest(part=part_name):
+                self.assertEqual(part_value, getattr(sys.version_info, part_name))
+
+    def test_implementation(self):
+        for key, value in self.key('implementation').items():
+            with self.subTest(part=key):
+                if key == 'version':
+                    self.assertEqual(len(value), len(sys.implementation.version))
+                    for part_name, part_value in value.items():
+                        self.assertEqual(getattr(sys.implementation.version, part_name), part_value)
+                else:
+                    self.assertEqual(getattr(sys.implementation, key), value)
+
+
+needs_installed_python = unittest.skipIf(
+    sysconfig.is_python_build(),
+    'This test can only run in an installed Python',
+)
+
+
+@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now')
+@unittest.skipIf(is_wasi or is_emscripten, 'Feature not available on WebAssembly builds')
+class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase):
+    """Test CPython's install details file implementation."""
+
+    @property
+    def location(self):
+        if sysconfig.is_python_build():
+            projectdir = sysconfig.get_config_var('projectbase')
+            with open(os.path.join(projectdir, 'pybuilddir.txt')) as f:
+                dirname = os.path.join(projectdir, f.read())
+        else:
+            dirname = sysconfig.get_path('stdlib')
+        return os.path.join(dirname, 'build-details.json')
+
+    @property
+    def contents(self):
+        with open(self.location, 'r') as f:
+            return f.read()
+
+    @needs_installed_python
+    def test_location(self):
+        self.assertTrue(os.path.isfile(self.location))
+
+    # Override generic format tests with tests for our specific implemenation.
+
+    @needs_installed_python
+    @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS run tests via a custom testbed method that changes sys.executable')
+    def test_base_interpreter(self):
+        value = self.key('base_interpreter')
+
+        self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable))
+
+
+if __name__ == '__main__':
+    unittest.main()

Makefile.pre.in

@@ -728,7 +728,7 @@ list-targets:
 
 .PHONY: build_all
 build_all:	check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \
-		gdbhooks Programs/_testembed scripts checksharedmods rundsymutil
+		gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json
 
 .PHONY: build_wasm
 build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
@@ -934,6 +934,9 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS)
 		exit 1 ; \
 	fi
 
+build-details.json: pybuilddir.txt
+	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json
+
 # Build static library
 $(LIBRARY): $(LIBRARY_OBJS)
 	-rm -f $@
@@ -2644,6 +2647,7 @@ libinstall:	all $(srcdir)/Modules/xxmodule.c
 	done
 	$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \
 	$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \
+	$(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \
 	$(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt
 	@ # If app store compliance has been configured, apply the patch to the
 	@ # installed library code. The patch has been previously validated against

Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst

@@ -0,0 +1,2 @@
+A ``build-details.json`` file is now install in the platform-independent
+standard library directory (:pep:`739` implementation).

Tools/build/generate-build-details.py

@@ -0,0 +1,192 @@
+"""Generate build-details.json (see PEP 739)."""
+
+# Script initially imported from:
+# https://github.com/FFY00/python-instrospection/blob/main/python_introspection/scripts/generate-build-details.py
+
+import argparse
+import collections
+import importlib.machinery
+import json
+import os
+import sys
+import sysconfig
+import traceback
+import warnings
+
+
+if False:  # TYPE_CHECKING
+    pass
+
+
+def version_info_to_dict(obj):  # (object) -> dict[str, Any]
+    field_names = ('major', 'minor', 'micro', 'releaselevel', 'serial')
+    return {field: getattr(obj, field) for field in field_names}
+
+
+def get_dict_key(container, key):  # (dict[str, Any], str) -> dict[str, Any]
+    for part in key.split('.'):
+        container = container[part]
+    return container
+
+
+def generate_data(schema_version):
+    """Generate the build-details.json data (PEP 739).
+
+    :param schema_version: The schema version of the data we want to generate.
+    """
+
+    if schema_version != '1.0':
+        raise ValueError(f'Unsupported schema_version: {schema_version}')
+
+    data = collections.defaultdict(lambda: collections.defaultdict(dict))
+
+    data['schema_version'] = schema_version
+
+    data['base_prefix'] = sysconfig.get_config_var('installed_base')
+    #data['base_interpreter'] = sys._base_executable
+    data['base_interpreter'] = os.path.join(
+        sysconfig.get_path('scripts'),
+        'python' + sysconfig.get_config_var('VERSION'),
+    )
+    data['platform'] = sysconfig.get_platform()
+
+    data['language']['version'] = sysconfig.get_python_version()
+    data['language']['version_info'] = version_info_to_dict(sys.version_info)
+
+    data['implementation'] = vars(sys.implementation)
+    data['implementation']['version'] = version_info_to_dict(sys.implementation.version)
+    # Fix cross-compilation
+    if '_multiarch' in data['implementation']:
+        data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH')
+
+    data['abi']['flags'] = list(sys.abiflags)
+
+    data['suffixes']['source'] = importlib.machinery.SOURCE_SUFFIXES
+    data['suffixes']['bytecode'] = importlib.machinery.BYTECODE_SUFFIXES
+    #data['suffixes']['optimized_bytecode'] = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES
+    #data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES
+    data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES
+
+    LIBDIR = sysconfig.get_config_var('LIBDIR')
+    LDLIBRARY = sysconfig.get_config_var('LDLIBRARY')
+    LIBRARY = sysconfig.get_config_var('LIBRARY')
+    PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY')
+    LIBPYTHON = sysconfig.get_config_var('LIBPYTHON')
+    LIBPC = sysconfig.get_config_var('LIBPC')
+    INCLUDEDIR = sysconfig.get_config_var('INCLUDEDIR')
+
+    if os.name == 'posix':
+        # On POSIX, LIBRARY is always the static library, while LDLIBRARY is the
+        # dynamic library if enabled, otherwise it's the static library.
+        # If LIBRARY != LDLIBRARY, support for the dynamic library is enabled.
+        has_dynamic_library = LDLIBRARY != LIBRARY
+        has_static_library = sysconfig.get_config_var('STATIC_LIBPYTHON')
+    elif os.name == 'nt':
+        # Windows can only use a dynamic library or a static library.
+        # If it's using a dynamic library, sys.dllhandle will be set.
+        # Static builds on Windows are not really well supported, though.
+        # More context: https://github.com/python/cpython/issues/110234
+        has_dynamic_library = hasattr(sys, 'dllhandle')
+        has_static_library = not has_dynamic_library
+    else:
+        raise NotADirectoryError(f'Unknown platform: {os.name}')
+
+    # On POSIX, EXT_SUFFIX is set regardless if extension modules are supported
+    # or not, and on Windows older versions of CPython only set EXT_SUFFIX when
+    # extension modules are supported, but newer versions of CPython set it
+    # regardless.
+    #
+    # We only want to set abi.extension_suffix and stable_abi_suffix if
+    # extension modules are supported.
+    if has_dynamic_library:
+        data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX')
+
+        # EXTENSION_SUFFIXES has been constant for a long time, and currently we
+        # don't have a better information source to find the  stable ABI suffix.
+        for suffix in importlib.machinery.EXTENSION_SUFFIXES:
+            if suffix.startswith('.abi'):
+                data['abi']['stable_abi_suffix'] = suffix
+                break
+
+        data['libpython']['dynamic'] = os.path.join(LIBDIR, LDLIBRARY)
+        # FIXME: Not sure if windows has a different dll for the stable ABI, and
+        #        even if it does, currently we don't have a way to get its name.
+        if PY3LIBRARY:
+            data['libpython']['dynamic_stableabi'] = os.path.join(LIBDIR, PY3LIBRARY)
+
+        # Os POSIX, this is defined by the LIBPYTHON Makefile variable not being
+        # empty. On Windows, don't link extensions — LIBPYTHON won't be defined,
+        data['libpython']['link_extensions'] = bool(LIBPYTHON)
+
+    if has_static_library:
+        data['libpython']['static'] = os.path.join(LIBDIR, LIBRARY)
+
+    data['c_api']['include'] = INCLUDEDIR
+    if LIBPC:
+        data['c_api']['pkgconfig_path'] = LIBPC
+
+    return data
+
+
+def make_paths_relative(data, config_path=None):  # (dict[str, Any], str | None) -> None
+    # Make base_prefix relative to the config_path directory
+    if config_path:
+        data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path))
+    # Update path values to make them relative to base_prefix
+    PATH_KEYS = [
+        'base_interpreter',
+        'libpython.dynamic',
+        'libpython.dynamic_stableabi',
+        'libpython.static',
+        'c_api.headers',
+        'c_api.pkgconfig_path',
+    ]
+    for entry in PATH_KEYS:
+        parent, _, child = entry.rpartition('.')
+        # Get the key container object
+        try:
+            container = data
+            for part in parent.split('.'):
+                container = container[part]
+            current_path = container[child]
+        except KeyError:
+            continue
+        # Get the relative path
+        new_path = os.path.relpath(current_path, data['base_prefix'])
+        # Join '.' so that the path is formated as './path' instead of 'path'
+        new_path = os.path.join('.', new_path)
+        container[child] = new_path
+
+
+def main():  # () -> None
+    parser = argparse.ArgumentParser(exit_on_error=False)
+    parser.add_argument('location')
+    parser.add_argument(
+        '--schema-version',
+        default='1.0',
+        help='Schema version of the build-details.json file to generate.',
+    )
+    parser.add_argument(
+        '--relative-paths',
+        action='store_true',
+        help='Whether to specify paths as absolute, or as relative paths to ``base_prefix``.',
+    )
+    parser.add_argument(
+        '--config-file-path',
+        default=None,
+        help='If specified, ``base_prefix`` will be set as a relative path to the given config file path.',
+    )
+
+    args = parser.parse_args()
+
+    data = generate_data(args.schema_version)
+    if args.relative_paths:
+        make_paths_relative(data, args.config_file_path)
+
+    json_output = json.dumps(data, indent=2)
+    with open(args.location, 'w') as f:
+        print(json_output, file=f)
+
+
+if __name__ == '__main__':
+    main()