From 6a07424b363164ddef7f3bd3605be47b147ee48a Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 14 Dec 2022 12:21:53 -0800 Subject: [PATCH] [stable-2.14] Add more retries to ansible-test-container test. (cherry picked from commit f6c0e22f98e3ad1e0a98837053ed03a27d8a1fcf) Co-authored-by: Matt Clay --- .../targets/ansible-test-container/runme.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/integration/targets/ansible-test-container/runme.py b/test/integration/targets/ansible-test-container/runme.py index 1577b62286f..d91cf9333d1 100755 --- a/test/integration/targets/ansible-test-container/runme.py +++ b/test/integration/targets/ansible-test-container/runme.py @@ -255,7 +255,7 @@ def run_test(scenario: TestScenario) -> TestResult: try: if prime_storage_command: - retry_command(lambda: run_command(*prime_storage_command)) + retry_command(lambda: run_command(*prime_storage_command), retry_any_error=True) if scenario.disable_selinux: run_command('setenforce', 'permissive') @@ -277,7 +277,7 @@ def run_test(scenario: TestScenario) -> TestResult: cleanup_command = [scenario.engine, 'rmi', '-f', scenario.image] try: - retry_command(lambda: run_command(*client_become_cmd + [f'{format_env(common_env)}{shlex.join(cleanup_command)}'])) + retry_command(lambda: run_command(*client_become_cmd + [f'{format_env(common_env)}{shlex.join(cleanup_command)}']), retry_any_error=True) except SubprocessError as ex: display.error(str(ex)) @@ -667,7 +667,7 @@ def run_module( return run_command('ansible', '-m', module, '-v', '-a', json.dumps(args), 'localhost') -def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3) -> SubprocessResult: +def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3, retry_any_error: bool = False) -> SubprocessResult: """Run the given command function up to the specified number of attempts when the failure is due to an SSH error.""" for attempts_remaining in range(attempts - 1, -1, -1): try: @@ -681,6 +681,11 @@ def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3) -> time.sleep(3) continue + if retry_any_error: + display.warning('Command failed. Waiting a few seconds before retrying.') + time.sleep(3) + continue + raise