From a0e98d2620713d7079a83408c6c356a542446226 Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Mon, 26 May 2025 19:37:51 -0400 Subject: [PATCH] t/runner.py: New --flaky-attempts argument to retry failing tests Several of the tests are flaky on platforms such as CI runner VMs due to sleep imprecision mixed with tight timing requirements in the tests. This option allows those tests to be retried until they succeed, up to a maximum number of attempts. --- t/runner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/t/runner.py b/t/runner.py index a70a37d4..b3f5f9c4 100755 --- a/t/runner.py +++ b/t/runner.py @@ -332,6 +332,8 @@ def printerr(s): parser.add_argument('-v', '--verbose', default=False, action='store_true') parser.add_argument('-e', '--exit-on-fail', default=False, action='store_true') parser.add_argument('files', nargs=argparse.REMAINDER) +parser.add_argument('--flaky-attempts', default=1, type=int, metavar='INT', + help='number of times a failing test case will be tried before it is declared a failure (default: %(default)s)') args = parser.parse_args() @@ -353,7 +355,12 @@ def printerr(s): tests.append((name, input, output)) - if not run_test(name, input, output, args.verbose): + for attempt in range(1, args.flaky_attempts + 1): + if attempt > 1: + print(f'WARNING: Retrying failed test {file} (attempt {attempt} out of {args.flaky_attempts})...') + if run_test(name, input, output, args.verbose): + break + else: if args.exit_on_fail: exit(-1)