bpo-45020: Default to using frozen modules unless running from source tree. (gh-28940)

The default was "off".  Switching it to "on" means users get the benefit of frozen stdlib modules without having to do anything.  There's a special-case for running-in-source-tree, so contributors don't get surprised when their stdlib changes don't get used.

https://bugs.python.org/issue45020

Doc/using/cmdline.rst

@@ -483,7 +483,8 @@ Miscellaneous options
    * ``-X frozen_modules`` determines whether or not frozen modules are
      ignored by the import machinery.  A value of "on" means they get
      imported and "off" means they are ignored.  The default is "on"
-     for non-debug builds (the normal case) and "off" for debug builds.
+     if this is an installed Python (the normal case).  If it's under
+     development (running from the source tree) then the default is "off".
      Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
      frozen modules are always used, even if this flag is set to "off".
 

Include/internal/pycore_fileutils.h

@@ -79,6 +79,7 @@ extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
 extern int _Py_add_relfile(wchar_t *dirname,
                            const wchar_t *relfile,
                            size_t bufsize);
+extern size_t _Py_find_basename(const wchar_t *filename);
 
 // Macros to protect CRT calls against instant termination when passed an
 // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler.

Lib/site.py

@@ -418,8 +418,10 @@ def setcopyright():
     files, dirs = [], []
     # Not all modules are required to have a __file__ attribute.  See
     # PEP 420 for more details.
-    if hasattr(os, '__file__'):
+    here = getattr(sys, '_stdlib_dir', None)
+    if not here and hasattr(os, '__file__'):
         here = os.path.dirname(os.__file__)
+    if here:
         files.extend(["LICENSE.txt", "LICENSE"])
         dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
     builtins.license = _sitebuiltins._Printer(

Lib/test/test_embed.py

@@ -53,12 +53,13 @@ def remove_python_envvars():
 class EmbeddingTestsMixin:
     def setUp(self):
         exename = "_testembed"
+        builddir = os.path.dirname(sys.executable)
         if MS_WINDOWS:
             ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
             exename += ext
-            exepath = os.path.dirname(sys.executable)
+            exepath = builddir
         else:
-            exepath = os.path.join(support.REPO_ROOT, "Programs")
+            exepath = os.path.join(builddir, 'Programs')
         self.test_exe = exe = os.path.join(exepath, exename)
         if not os.path.exists(exe):
             self.skipTest("%r doesn't exist" % exe)
@@ -434,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
-        'use_frozen_modules': 0,
+        'use_frozen_modules': 1,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
@@ -1146,6 +1147,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
+            'use_frozen_modules': -1,
         }
         self.default_program_name(config)
         env = {'TESTPATH': os.path.pathsep.join(paths)}
@@ -1169,6 +1171,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
+            'use_frozen_modules': -1,
             # overridden by PyConfig
             'program_name': 'conf_program_name',
             'base_executable': 'conf_executable',
@@ -1265,6 +1268,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'stdlib_dir': stdlib,
         }
         self.default_program_name(config)
+        if not config['executable']:
+            config['use_frozen_modules'] = -1
         env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
@@ -1303,6 +1308,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 # The current getpath.c doesn't determine the stdlib dir
                 # in this case.
                 'stdlib_dir': None,
+                'use_frozen_modules': -1,
             }
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,
@@ -1361,6 +1367,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 config['base_prefix'] = pyvenv_home
                 config['prefix'] = pyvenv_home
                 config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
+                config['use_frozen_modules'] = 1
 
                 ver = sys.version_info
                 dll = f'python{ver.major}'
@@ -1373,6 +1380,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 # The current getpath.c doesn't determine the stdlib dir
                 # in this case.
                 config['stdlib_dir'] = None
+                config['use_frozen_modules'] = -1
 
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,

Misc/NEWS.d/3.11.0a1.rst

@@ -262,7 +262,7 @@ Compiler now removes trailing unused constants from co_consts.
 
 Add a new command line option, "-X frozen_modules=[on|off]" to opt out of
 (or into) using optional frozen modules.  This defaults to "on" (or "off" if
-it's a debug build).
+it's running out of the source tree).
 
 ..
 

Python/fileutils.c

@@ -2169,6 +2169,18 @@ _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize)
 }
 
 
+size_t
+_Py_find_basename(const wchar_t *filename)
+{
+    for (size_t i = wcslen(filename); i > 0; --i) {
+        if (filename[i] == SEP) {
+            return i + 1;
+        }
+    }
+    return 0;
+}
+
+
 /* Get the current directory. buflen is the buffer size in wide characters
    including the null character. Decode the path from the locale encoding.
 

Python/initconfig.c

@@ -739,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = -1;
 #endif
+    config->use_frozen_modules = -1;
 }
 
 
@@ -2090,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 }
 
 
+/* Determine if the current build is a "development" build (e.g. running
+   out of the source tree) or not.
+
+   A return value of -1 indicates that we do not know.
+  */
+static int
+is_dev_env(PyConfig *config)
+{
+    // This should only ever get called early in runtime initialization,
+    // before the global path config is written.  Otherwise we would
+    // use Py_GetProgramFullPath() and _Py_GetStdlibDir().
+    assert(config != NULL);
+
+    const wchar_t *executable = config->executable;
+    const wchar_t *stdlib = config->stdlib_dir;
+    if (executable == NULL || *executable == L'\0' ||
+            stdlib == NULL || *stdlib == L'\0') {
+        // _PyPathConfig_Calculate() hasn't run yet.
+        return -1;
+    }
+    size_t len = _Py_find_basename(executable);
+    if (wcscmp(executable + len, L"python") != 0 &&
+            wcscmp(executable + len, L"python.exe") != 0) {
+        return 0;
+    }
+    /* If dirname() is the same for both then it is a dev build. */
+    if (len != _Py_find_basename(stdlib)) {
+        return 0;
+    }
+    // We do not bother normalizing the two filenames first since
+    // for config_init_import() is does the right thing as-is.
+    if (wcsncmp(stdlib, executable, len) != 0) {
+        return 0;
+    }
+    return 1;
+}
+
+
 static PyStatus
 config_init_import(PyConfig *config, int compute_path_config)
 {
@@ -2101,25 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config)
     }
 
     /* -X frozen_modules=[on|off] */
-    const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
+    if (config->use_frozen_modules < 0) {
-    if (value == NULL) {
+        const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
-        // For now we always default to "off".
+        if (value == NULL) {
-        // In the near future we will be factoring in PGO and in-development.
+            int isdev = is_dev_env(config);
-        config->use_frozen_modules = 0;
+            if (isdev >= 0) {
-    }
+                config->use_frozen_modules = !isdev;
-    else if (wcscmp(value, L"on") == 0) {
+            }
-        config->use_frozen_modules = 1;
+        }
-    }
+        else if (wcscmp(value, L"on") == 0) {
-    else if (wcscmp(value, L"off") == 0) {
+            config->use_frozen_modules = 1;
-        config->use_frozen_modules = 0;
+        }
-    }
+        else if (wcscmp(value, L"off") == 0) {
-    else if (wcslen(value) == 0) {
+            config->use_frozen_modules = 0;
-        // "-X frozen_modules" and "-X frozen_modules=" both imply "on".
+        }
-        config->use_frozen_modules = 1;
+        else if (wcslen(value) == 0) {
-    }
+            // "-X frozen_modules" and "-X frozen_modules=" both imply "on".
-    else {
+            config->use_frozen_modules = 1;
-        return PyStatus_Error("bad value for option -X frozen_modules "
+        }
-                              "(expected \"on\" or \"off\")");
+        else {
+            return PyStatus_Error("bad value for option -X frozen_modules "
+                                  "(expected \"on\" or \"off\")");
+        }
     }
 
     return _PyStatus_OK();