gh-109566, regrtest: Add --fast-ci and --slow-ci options (#109570)

* Add --fast-ci and --slow-ci options to libregrtest:

  * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu"
    (skip slowest tests).
  * --slow-ci uses a default timeout of 20 minues and "-u all" (run
    all tests).

* regrtest header now lists test resources.
* Makefile changes:

  * "make test", "make hostrunnertest" and "make coverage-report" now
    use --fast-ci option and TESTTIMEOUT variable.
  * "make buildbottest" now uses "--slow-ci". Remove options which
    became redundant with "--slow-ci".
  * "make testall" and "make testuniversal" now use --slow-ci option
    and TESTTIMEOUT variable.
  * "make testall" now uses "find -exec rm ..." instead of
    "find ... -print|xargs rm ...", same as "make clean".

* GitHub Actions workflow:

  * Ubuntu and Address Sanitizer jobs now use "make test". Remove
    options which became redundant with "--fast-ci".
  * Windows jobs now use --fast-ci option.
  * Use -j0 to detect the number of CPUs.

* Set Makefile TESTTIMEOUT default to an empty string, since
  --slow-ci and --fast-ci use different default timeout. It's now
  accepted to pass "--timeout=" to regrtest: treated as not timeout.
* Tools/scripts/run_tests.py now uses --fast-ci option.
* Tools/buildbot/test.bat now uses --slow-ci option. Remove
  --timeout=1200 option, redundant with --slow-ci.
This commit is contained in:
Victor Stinner 2023-09-26 17:22:50 +02:00 committed by GitHub
parent 19bf398695
commit 859618c8cd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 161 additions and 51 deletions

View file

@ -23,8 +23,9 @@ import unittest
from test import support
from test.support import os_helper, TestStats, without_optimizer
from test.libregrtest import cmdline
from test.libregrtest import utils
from test.libregrtest import main
from test.libregrtest import setup
from test.libregrtest import utils
from test.libregrtest.utils import normalize_test_name
if not support.has_subprocess_support:
@ -75,8 +76,15 @@ class ParseArgsTestCase(unittest.TestCase):
def test_timeout(self):
ns = self.parse_args(['--timeout', '4.2'])
self.assertEqual(ns.timeout, 4.2)
# negative, zero and empty string are treated as "no timeout"
for value in ('-1', '0', ''):
with self.subTest(value=value):
ns = self.parse_args([f'--timeout={value}'])
self.assertEqual(ns.timeout, None)
self.checkError(['--timeout'], 'expected one argument')
self.checkError(['--timeout', 'foo'], 'invalid float value')
self.checkError(['--timeout', 'foo'], 'invalid timeout value:')
def test_wait(self):
ns = self.parse_args(['--wait'])
@ -366,6 +374,44 @@ class ParseArgsTestCase(unittest.TestCase):
self.checkError(['--unknown-option'],
'unrecognized arguments: --unknown-option')
def check_ci_mode(self, args, use_resources):
ns = cmdline._parse_args(args)
if utils.MS_WINDOWS:
self.assertTrue(ns.nowindows)
# Check Regrtest attributes which are more reliable than Namespace
# which has an unclear API
regrtest = main.Regrtest(ns)
self.assertNotEqual(regrtest.num_workers, 0)
self.assertTrue(regrtest.want_rerun)
self.assertTrue(regrtest.randomize)
self.assertIsNone(regrtest.random_seed)
self.assertTrue(regrtest.fail_env_changed)
self.assertTrue(regrtest.fail_rerun)
self.assertTrue(regrtest.print_slowest)
self.assertTrue(regrtest.output_on_failure)
self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
return regrtest
def test_fast_ci(self):
args = ['--fast-ci']
use_resources = sorted(cmdline.ALL_RESOURCES)
use_resources.remove('cpu')
regrtest = self.check_ci_mode(args, use_resources)
self.assertEqual(regrtest.timeout, 10 * 60)
def test_fast_ci_resource(self):
# it should be possible to override resources
args = ['--fast-ci', '-u', 'network']
use_resources = ['network']
self.check_ci_mode(args, use_resources)
def test_slow_ci(self):
args = ['--slow-ci']
use_resources = sorted(cmdline.ALL_RESOURCES)
regrtest = self.check_ci_mode(args, use_resources)
self.assertEqual(regrtest.timeout, 20 * 60)
@dataclasses.dataclass(slots=True)
class Rerun: