gh-110152: regrtest handles cross compilation and HOSTRUNNER (#110156)

* _add_python_opts() now handles cross compilation and HOSTRUNNER.
* display_header() now tells if Python is cross-compiled, display
  HOSTRUNNER, and get the host platform.
* Remove Tools/scripts/run_tests.py script.
* Remove "make hostrunnertest": use "make buildbottest"
  or "make test" instead.

Lib/test/libregrtest/main.py

@@ -3,6 +3,7 @@ import random
 import re
 import shlex
 import sys
+import sysconfig
 import time
 
 from test import support
@@ -22,6 +23,7 @@ from .utils import (
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
     display_header, cleanup_temp_dir, print_warning,
+    is_cross_compiled, get_host_runner,
     MS_WINDOWS, EXIT_TIMEOUT)
 
 
@@ -71,10 +73,9 @@ class Regrtest:
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
 
-        ci_mode = (ns.fast_ci or ns.slow_ci)
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.want_add_python_opts: bool = (_add_python_opts
-                                           and ns._add_python_opts
-                                           and ci_mode)
+                                           and ns._add_python_opts)
 
         # Select tests
         if ns.match_tests:
@@ -431,7 +432,7 @@ class Regrtest:
         if (self.want_header
             or not(self.pgo or self.quiet or self.single_test_run
                    or tests or self.cmdline_args)):
-            display_header(self.use_resources)
+            display_header(self.use_resources, self.python_cmd)
 
         if self.randomize:
             print("Using random seed", self.random_seed)
@@ -489,8 +490,56 @@ class Regrtest:
                 # processes.
                 return self._run_tests(selected, tests)
 
-    def _add_python_opts(self):
-        python_opts = []
+    def _add_cross_compile_opts(self, regrtest_opts):
+        # WASM/WASI buildbot builders pass multiple PYTHON environment
+        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
+        keep_environ = bool(self.python_cmd)
+        environ = None
+
+        # Are we using cross-compilation?
+        cross_compile = is_cross_compiled()
+
+        # Get HOSTRUNNER
+        hostrunner = get_host_runner()
+
+        if cross_compile:
+            # emulate -E, but keep PYTHONPATH + cross compile env vars,
+            # so test executable can load correct sysconfigdata file.
+            keep = {
+                '_PYTHON_PROJECT_BASE',
+                '_PYTHON_HOST_PLATFORM',
+                '_PYTHON_SYSCONFIGDATA_NAME',
+                'PYTHONPATH'
+            }
+            old_environ = os.environ
+            new_environ = {
+                name: value for name, value in os.environ.items()
+                if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
+            }
+            # Only set environ if at least one variable was removed
+            if new_environ != old_environ:
+                environ = new_environ
+            keep_environ = True
+
+        if cross_compile and hostrunner:
+            if self.num_workers == 0:
+                # For now use only two cores for cross-compiled builds;
+                # hostrunner can be expensive.
+                regrtest_opts.extend(['-j', '2'])
+
+            # If HOSTRUNNER is set and -p/--python option is not given, then
+            # use hostrunner to execute python binary for tests.
+            if not self.python_cmd:
+                buildpython = sysconfig.get_config_var("BUILDPYTHON")
+                python_cmd = f"{hostrunner} {buildpython}"
+                regrtest_opts.extend(["--python", python_cmd])
+                keep_environ = True
+
+        return (environ, keep_environ)
+
+    def _add_ci_python_opts(self, python_opts, keep_environ):
+        # --fast-ci and --slow-ci add options to Python:
+        # "-u -W default -bb -E"
 
         # Unbuffered stdout and stderr
         if not sys.stdout.write_through:
@@ -504,32 +553,27 @@ class Regrtest:
         if sys.flags.bytes_warning < 2:
             python_opts.append('-bb')
 
-        # WASM/WASI buildbot builders pass multiple PYTHON environment
-        # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
-        if not self.python_cmd:
+        if not keep_environ:
             # Ignore PYTHON* environment variables
             if not sys.flags.ignore_environment:
                 python_opts.append('-E')
 
-        if not python_opts:
-            return
-
-        cmd = [*sys.orig_argv, "--dont-add-python-opts"]
-        cmd[1:1] = python_opts
-
+    def _execute_python(self, cmd, environ):
         # Make sure that messages before execv() are logged
         sys.stdout.flush()
         sys.stderr.flush()
 
         cmd_text = shlex.join(cmd)
         try:
+            print(f"+ {cmd_text}", flush=True)
+
             if hasattr(os, 'execv') and not MS_WINDOWS:
                 os.execv(cmd[0], cmd)
                 # On success, execv() do no return.
                 # On error, it raises an OSError.
             else:
                 import subprocess
-                with subprocess.Popen(cmd) as proc:
+                with subprocess.Popen(cmd, env=environ) as proc:
                     try:
                         proc.wait()
                     except KeyboardInterrupt:
@@ -548,6 +592,28 @@ class Regrtest:
                           f"Command: {cmd_text}")
             # continue executing main()
 
+    def _add_python_opts(self):
+        python_opts = []
+        regrtest_opts = []
+
+        environ, keep_environ = self._add_cross_compile_opts(regrtest_opts)
+        if self.ci_mode:
+            self._add_ci_python_opts(python_opts, keep_environ)
+
+        if (not python_opts) and (not regrtest_opts) and (environ is None):
+            # Nothing changed: nothing to do
+            return
+
+        # Create new command line
+        cmd = list(sys.orig_argv)
+        if python_opts:
+            cmd[1:1] = python_opts
+        if regrtest_opts:
+            cmd.extend(regrtest_opts)
+        cmd.append("--dont-add-python-opts")
+
+        self._execute_python(cmd, environ)
+
     def _init(self):
         # Set sys.stdout encoder error handler to backslashreplace,
         # similar to sys.stderr error handler, to avoid UnicodeEncodeError