Fix unreachable host detection and have plays continue after unreachable failures

Fixes #12284

lib/ansible/executor/playbook_executor.py

@@ -50,6 +50,7 @@ class PlaybookExecutor:
         self._display          = display
         self._options          = options
         self.passwords         = passwords
+        self._unreachable_hosts = dict()
 
         if options.listhosts or options.listtasks or options.listtags or options.syntax:
             self._tqm = None
@@ -121,6 +122,7 @@ class PlaybookExecutor:
                     else:
                         # make sure the tqm has callbacks loaded
                         self._tqm.load_callbacks()
+                        self._tqm._unreachable_hosts.update(self._unreachable_hosts)
 
                         # we are actually running plays
                         for batch in self._get_serialized_batches(new_play):
@@ -148,10 +150,12 @@ class PlaybookExecutor:
                                 break
 
                             # clear the failed hosts dictionaires in the TQM for the next batch
+                            self._unreachable_hosts.update(self._tqm._unreachable_hosts)
                             self._tqm.clear_failed_hosts()
 
-                        # if the last result wasn't zero, break out of the serial batch loop
+                        # if the last result wasn't zero or 3 (some hosts were unreachable),
-                        if result != 0:
+                        # break out of the serial batch loop
+                        if result not in (0, 3):
                             break
 
                     i = i + 1 # per play
@@ -161,7 +165,7 @@ class PlaybookExecutor:
 
                 # if the last result wasn't zero, break out of the playbook file name loop
                 if result != 0:
-                    break
+                        break
 
             if entrylist:
                 return entrylist

lib/ansible/executor/task_executor.py

@@ -28,7 +28,7 @@ import time
 from six import iteritems
 
 from ansible import constants as C
-from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable
+from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
 from ansible.template import Templar
@@ -324,7 +324,10 @@ class TaskExecutor:
                 result['attempts'] = attempt + 1
 
             debug("running the handler")
-            result = self._handler.run(task_vars=variables)
+            try:
+                result = self._handler.run(task_vars=variables)
+            except AnsibleConnectionFailure as e:
+                return dict(unreachable=True, msg=str(e))
             debug("handler run complete")
 
             if self._task.async > 0:

lib/ansible/executor/task_queue_manager.py

@@ -222,8 +222,7 @@ class TaskQueueManager:
             worker_prc.terminate()
 
     def clear_failed_hosts(self):
-        self._failed_hosts      = dict()
+        self._failed_hosts = dict()
-        self._unreachable_hosts = dict()
 
     def get_inventory(self):
         return self._inventory

lib/ansible/plugins/action/__init__.py

@@ -30,7 +30,7 @@ import tempfile
 import time
 
 from ansible import constants as C
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleConnectionFailure
 from ansible.executor.module_common import modify_module
 from ansible.parsing.utils.jsonify import jsonify
 from ansible.utils.unicode import to_bytes
@@ -190,7 +190,7 @@ class ActionBase:
                 output = 'Authentication or permission failure.  In some cases, you may have been able to authenticate and did not have permissions on the remote directory. Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp". Failed command was: %s, exited with result %d' % (cmd, result['rc'])
             if 'stdout' in result and result['stdout'] != '':
                 output = output + ": %s" % result['stdout']
-            raise AnsibleError(output)
+            raise AnsibleConnectionFailure(output)
 
         # FIXME: do we still need to do this?
         #rc = self._connection._shell.join_path(utils.last_non_blank_line(result['stdout']).strip(), '')

lib/ansible/plugins/strategies/free.py

@@ -55,7 +55,7 @@ class StrategyModule(StrategyBase):
         work_to_do = True
         while work_to_do and not self._tqm._terminated:
 
-            hosts_left = self._inventory.get_hosts(iterator._play.hosts)
+            hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
             if len(hosts_left) == 0:
                 self._tqm.send_callback('v2_playbook_on_no_hosts_remaining')
                 result = False

lib/ansible/plugins/strategies/linear.py

@@ -155,7 +155,7 @@ class StrategyModule(StrategyBase):
 
             try:
                 self._display.debug("getting the remaining hosts for this loop")
-                hosts_left = self._inventory.get_hosts(iterator._play.hosts)
+                hosts_left = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
                 self._display.debug("done getting the remaining hosts for this loop")
 
                 # queue up this task for each host in the inventory