bpo-45573: Use Makefile's dependencies in setup.py (GH-29559)

Makefile.pre.in

@@ -2460,7 +2460,7 @@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/uni
 MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-dispatch.c $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2-kat.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b-test.c $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2bp-test.c $(srcdir)/Modules/_blake2/impl/blake2bp.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s-test.c $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/impl/blake2sp-test.c $(srcdir)/Modules/_blake2/impl/blake2sp.c $(srcdir)/Modules/hashlib.h
 MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h
 MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h $(LIBMPDEC_HEADERS) @LIBMPDEC_INTERNAL@
-MODULE__ELEMENTTREE_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
+MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c $(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
 MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
 MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
 MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h

setup.py

@@ -361,7 +361,7 @@ def find_module_file(module, dirlist):
         return module
     if len(dirs) > 1:
         log.info(f"WARNING: multiple copies of {module} found")
-    return os.path.join(dirs[0], module)
+    return os.path.abspath(os.path.join(dirs[0], module))
 
 
 def parse_cflags(flags):
@@ -454,7 +454,13 @@ class PyBuildExt(build_ext):
     def update_sources_depends(self):
         # Fix up the autodetected modules, prefixing all the source files
         # with Modules/.
-        moddirlist = [os.path.join(self.srcdir, 'Modules')]
+        # Add dependencies from MODULE_{name}_DEPS variable
+        moddirlist = [
+            # files in Modules/ directory
+            os.path.join(self.srcdir, 'Modules'),
+            # files relative to build base, e.g. libmpdec.a, libexpat.a
+            os.getcwd()
+        ]
 
         # Fix up the paths for scripts, too
         self.distribution.scripts = [os.path.join(self.srcdir, filename)
@@ -470,11 +476,16 @@ class PyBuildExt(build_ext):
         for ext in self.extensions:
             ext.sources = [ find_module_file(filename, moddirlist)
                             for filename in ext.sources ]
-            if ext.depends is not None:
+            # Update dependencies from Makefile
-                ext.depends = [find_module_file(filename, moddirlist)
+            makedeps = sysconfig.get_config_var(f"MODULE_{ext.name.upper()}_DEPS")
-                               for filename in ext.depends]
+            if makedeps:
-            else:
+                # remove backslashes from line break continuations
-                ext.depends = []
+                ext.depends.extend(
+                    dep for dep in makedeps.split() if dep != "\\"
+                )
+            ext.depends = [
+                find_module_file(filename, moddirlist) for filename in ext.depends
+            ]
             # re-compile extensions if a header file has been changed
             ext.depends.extend(headers)
 
@@ -966,12 +977,10 @@ class PyBuildExt(build_ext):
 
         # math library functions, e.g. sin()
         self.add(Extension('math',  ['mathmodule.c'],
-                           depends=['_math.h'],
                            libraries=['m']))
 
         # complex math library functions
         self.add(Extension('cmath', ['cmathmodule.c'],
-                           depends=['_math.h'],
                            libraries=['m']))
 
         # time libraries: librt may be needed for clock_gettime()
@@ -1003,8 +1012,7 @@ class PyBuildExt(build_ext):
         # profiler (_lsprof is for cProfile.py)
         self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c']))
         # static Unicode character database
-        self.add(Extension('unicodedata', ['unicodedata.c'],
+        self.add(Extension('unicodedata', ['unicodedata.c']))
-                           depends=['unicodedata_db.h', 'unicodename_db.h']))
         # _opcode module
         self.add(Extension('_opcode', ['_opcode.c']))
         # asyncio speedups
@@ -1081,8 +1089,7 @@ class PyBuildExt(build_ext):
 
     def detect_test_extensions(self):
         # Python C API test module
-        self.add(Extension('_testcapi', ['_testcapimodule.c'],
+        self.add(Extension('_testcapi', ['_testcapimodule.c']))
-                           depends=['testcapi_long.h']))
 
         # Python Internal C API test module
         self.add(Extension('_testinternalcapi', ['_testinternalcapi.c']))
@@ -1263,7 +1270,7 @@ class PyBuildExt(build_ext):
         self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs))
 
     def detect_socket(self):
-        self.add(Extension('_socket', ['socketmodule.c'], depends=['socketmodule.h']))
+        self.add(Extension('_socket', ['socketmodule.c']))
 
     def detect_dbm_gdbm(self):
         # Modules that provide persistent dictionary-like semantics.  You will
@@ -1527,11 +1534,6 @@ class PyBuildExt(build_ext):
         ldflags = parse_ldflags(sysconfig.get_config_var("EXPAT_LDFLAGS"))
         library_dirs, libraries, extra_link_args = ldflags
 
-        expat_depends = []
-        libexpat_a = sysconfig.get_config_var("LIBEXPAT_A")
-        if libexpat_a:
-            expat_depends.append(libexpat_a)
-
         self.add(Extension('pyexpat',
                            include_dirs=include_dirs,
                            define_macros=define_macros,
@@ -1540,8 +1542,7 @@ class PyBuildExt(build_ext):
                            library_dirs=library_dirs,
                            libraries=libraries,
                            extra_link_args=extra_link_args,
-                           sources=['pyexpat.c'],
+                           sources=['pyexpat.c']))
-                           depends=expat_depends))
 
         # Fredrik Lundh's cElementTree module.  Note that this also
         # uses expat (via the CAPI hook in pyexpat).
@@ -1551,8 +1552,7 @@ class PyBuildExt(build_ext):
                            undef_macros=undef_macros,
                            extra_compile_args=extra_compile_args,
                            # no EXPAT_LDFLAGS
-                           sources=['_elementtree.c'],
+                           sources=['_elementtree.c']))
-                           depends=['pyexpat.c', *expat_depends]))
 
     def detect_multibytecodecs(self):
         # Hye-Shik Chang's CJKCodecs modules.
@@ -1961,7 +1961,6 @@ class PyBuildExt(build_ext):
                    '_ctypes/callproc.c',
                    '_ctypes/stgdict.c',
                    '_ctypes/cfield.c']
-        depends = ['_ctypes/ctypes.h']
 
         if MACOS:
             sources.append('_ctypes/malloc_closure.c')
@@ -1988,8 +1987,7 @@ class PyBuildExt(build_ext):
                         extra_compile_args=extra_compile_args,
                         extra_link_args=extra_link_args,
                         libraries=[],
-                        sources=sources,
+                        sources=sources)
-                        depends=depends)
         self.add(ext)
         if TEST_EXTENSIONS:
             # function my_sqrt() needs libm for sqrt()
@@ -2049,7 +2047,6 @@ class PyBuildExt(build_ext):
     def detect_decimal(self):
         # Stefan Krah's _decimal module
         sources = ['_decimal/_decimal.c']
-        depends = ['_decimal/docstrings.h']
 
         cflags = parse_cflags(sysconfig.get_config_var("DECIMAL_CFLAGS"))
         include_dirs, define_macros, undef_macros, extra_compile_args = cflags
@@ -2058,10 +2055,6 @@ class PyBuildExt(build_ext):
         ldflags = parse_ldflags(sysconfig.get_config_var("DECIMAL_LDFLAGS"))
         library_dirs, libraries, extra_link_args = ldflags
 
-        libmpdec_a = sysconfig.get_config_var("LIBMPDEC_A")
-        if libmpdec_a:
-            depends.append(libmpdec_a)
-
         # Uncomment for extra functionality:
         #define_macros.append(('EXTRA_FUNCTIONALITY', 1))
         self.add(Extension('_decimal',
@@ -2072,8 +2065,7 @@ class PyBuildExt(build_ext):
                            library_dirs=library_dirs,
                            libraries=libraries,
                            extra_link_args=extra_link_args,
-                           sources=sources,
+                           sources=sources))
-                           depends=depends))
 
     def detect_openssl_hashlib(self):
         # Detect SSL support for the socket module (via _ssl)
@@ -2141,16 +2133,6 @@ class PyBuildExt(build_ext):
             Extension(
                 '_ssl',
                 ['_ssl.c'],
-                depends=[
-                    'socketmodule.h',
-                    '_ssl.h',
-                    '_ssl_data_111.h',
-                    '_ssl_data_300.h',
-                    '_ssl_data.h',
-                    '_ssl/debughelpers.c',
-                    '_ssl/misc.c',
-                    '_ssl/cert.c',
-                ],
                 **openssl_extension_kwargs
             )
         )
@@ -2158,7 +2140,6 @@ class PyBuildExt(build_ext):
             Extension(
                 '_hashlib',
                 ['_hashopenssl.c'],
-                depends=['hashlib.h'],
                 **openssl_extension_kwargs,
             )
         )
@@ -2182,52 +2163,38 @@ class PyBuildExt(build_ext):
 
         if "sha256" in configured:
             self.add(Extension(
-                '_sha256', ['sha256module.c'],
+                '_sha256', ['sha256module.c']
-                depends=['hashlib.h'],
             ))
 
         if "sha512" in configured:
             self.add(Extension(
                 '_sha512', ['sha512module.c'],
-                depends=['hashlib.h'],
             ))
 
         if "md5" in configured:
             self.add(Extension(
                 '_md5', ['md5module.c'],
-                depends=['hashlib.h'],
             ))
 
         if "sha1" in configured:
             self.add(Extension(
                 '_sha1', ['sha1module.c'],
-                depends=['hashlib.h'],
             ))
 
         if "blake2" in configured:
-            blake2_deps = glob(
-                os.path.join(escape(self.srcdir), 'Modules/_blake2/impl/*')
-            )
-            blake2_deps.append('hashlib.h')
             self.add(Extension(
                 '_blake2',
                 [
                     '_blake2/blake2module.c',
                     '_blake2/blake2b_impl.c',
                     '_blake2/blake2s_impl.c'
-                ],
+                ]
-                depends=blake2_deps,
             ))
 
         if "sha3" in configured:
-            sha3_deps = glob(
-                os.path.join(escape(self.srcdir), 'Modules/_sha3/kcp/*')
-            )
-            sha3_deps.append('hashlib.h')
             self.add(Extension(
                 '_sha3',
                 ['_sha3/sha3module.c'],
-                depends=sha3_deps,
             ))
 
     def detect_nis(self):