[stable-2.14] Add more retries to ansible-test-container test.

(cherry picked from commit f6c0e22f98)

Co-authored-by: Matt Clay <matt@mystile.com>

test/integration/targets/ansible-test-container/runme.py

@@ -255,7 +255,7 @@ def run_test(scenario: TestScenario) -> TestResult:
 
     try:
         if prime_storage_command:
-            retry_command(lambda: run_command(*prime_storage_command))
+            retry_command(lambda: run_command(*prime_storage_command), retry_any_error=True)
 
         if scenario.disable_selinux:
             run_command('setenforce', 'permissive')
@@ -277,7 +277,7 @@ def run_test(scenario: TestScenario) -> TestResult:
         cleanup_command = [scenario.engine, 'rmi', '-f', scenario.image]
 
         try:
-            retry_command(lambda: run_command(*client_become_cmd + [f'{format_env(common_env)}{shlex.join(cleanup_command)}']))
+            retry_command(lambda: run_command(*client_become_cmd + [f'{format_env(common_env)}{shlex.join(cleanup_command)}']), retry_any_error=True)
         except SubprocessError as ex:
             display.error(str(ex))
 
@@ -667,7 +667,7 @@ def run_module(
     return run_command('ansible', '-m', module, '-v', '-a', json.dumps(args), 'localhost')
 
 
-def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3) -> SubprocessResult:
+def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3, retry_any_error: bool = False) -> SubprocessResult:
     """Run the given command function up to the specified number of attempts when the failure is due to an SSH error."""
     for attempts_remaining in range(attempts - 1, -1, -1):
         try:
@@ -681,6 +681,11 @@ def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3) ->
                 time.sleep(3)
                 continue
 
+            if retry_any_error:
+                display.warning('Command failed. Waiting a few seconds before retrying.')
+                time.sleep(3)
+                continue
+
             raise