first_found lookup, let lookup handle templating errors (#81178)

* first_found lookup, let lookup handle templating errors

Avoids case in which TE was not sending valid and templatable entries to the lookup
The lookup already handles the case TE was attempting to itself, so no need for this code anymore.

Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com>

changelogs/fragments/first_found_template_fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - first_found lookup now gets 'untemplated' loop entries and handles templating itself as task_executor was removing even 'templatable' entries and breaking functionality. https://github.com/ansible/ansible/issues/70772

lib/ansible/executor/task_executor.py

@@ -224,14 +224,11 @@ class TaskExecutor:
         items = None
         if self._task.loop_with:
             if self._task.loop_with in self._shared_loader_obj.lookup_loader:
-                fail = True
-                if self._task.loop_with == 'first_found':
-                    # first_found loops are special. If the item is undefined then we want to fall through to the next value rather than failing.
-                    fail = False
 
+                # TODO: hardcoded so it fails for non first_found lookups, but thhis shoudl be generalized for those that don't do their own templating
+                # lookup prop/attribute?
+                fail = bool(self._task.loop_with != 'first_found')
                 loop_terms = listify_lookup_plugin_terms(terms=self._task.loop, templar=templar, fail_on_undefined=fail, convert_bare=False)
-                if not fail:
-                    loop_terms = [t for t in loop_terms if not templar.is_template(t)]
 
                 # get lookup
                 mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop_with, loader=self._loader, templar=templar)

lib/ansible/plugins/lookup/first_found.py

@@ -226,6 +226,8 @@ class LookupModule(LookupBase):
             try:
                 fn = self._templar.template(fn)
             except (AnsibleUndefinedVariable, UndefinedError):
+                # NOTE: backwards compat ff behaviour is to ignore errors when vars are undefined.
+                #       moved here from task_executor.
                 continue
 
             # get subdir if set by task executor, default to files otherwise

test/integration/targets/lookup_first_found/tasks/main.yml

@@ -102,3 +102,48 @@
 
 - assert:
     that: "no_terms|first == '/etc/hosts'"
+
+- name: handle templatable dictionary entries
+  block:
+
+  - name: Load variables specific for OS family
+    assert:
+      that:
+        - "{{item|quote}} is file"
+        - "{{item|basename == 'itworks.yml'}}"
+    with_first_found:
+      - files:
+          - "{{ansible_id}}-{{ansible_lsb.major_release}}.yml"  # invalid var, should be skipped
+          - "{{ansible_lsb.id}}-{{ansible_lsb.major_release}}.yml"  # does not exist, but should try
+          - "{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml"  # does not exist, but should try
+          - itworks.yml
+          - ishouldnotbefound.yml  # this exist, but should not be found
+        paths:
+          - "{{role_path}}/vars"
+
+  - name: Load variables specific for OS family, but now as list of dicts, same options as above
+    assert:
+      that:
+        - "{{item|quote}} is file"
+        - "{{item|basename == 'itworks.yml'}}"
+    with_first_found:
+      - files:
+          - "{{ansible_id}}-{{ansible_lsb.major_release}}.yml"
+        paths:
+          - "{{role_path}}/vars"
+      - files:
+          - "{{ansible_lsb.id}}-{{ansible_lsb.major_release}}.yml"
+        paths:
+          - "{{role_path}}/vars"
+      - files:
+          - "{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml"
+        paths:
+          - "{{role_path}}/vars"
+      - files:
+          - itworks.yml
+        paths:
+          - "{{role_path}}/vars"
+      - files:
+          - ishouldnotbefound.yml
+        paths:
+          - "{{role_path}}/vars"

test/integration/targets/lookup_first_found/vars/ishouldnotbefound.yml

@@ -0,0 +1 @@
+really: i hide

test/integration/targets/lookup_first_found/vars/itworks.yml

@@ -0,0 +1 @@
+doesit: yes it does