Add optional 'skip_missing' flag to subelements

docsite/rst/playbooks_loops.rst

@@ -147,9 +147,26 @@ How might that be accomplished?  Let's assume you had the following defined and
         authorized: 
           - /tmp/alice/onekey.pub
           - /tmp/alice/twokey.pub
+        mysql:
+            password: mysql-password
+            hosts:
+              - "%"
+              - "127.0.0.1"
+              - "::1"
+              - "localhost"
+            privs:
+              - "*.*:SELECT"
+              - "DB1.*:ALL"
       - name: bob
         authorized:
           - /tmp/bob/id_rsa.pub
+        mysql:
+            password: other-mysql-password
+            hosts:
+              - "db1"
+            privs:
+              - "*.*:SELECT"
+              - "DB2.*:ALL"
 
 It might happen like so::
 
@@ -161,9 +178,23 @@ It might happen like so::
          - users
          - authorized
 
-Subelements walks a list of hashes (aka dictionaries) and then traverses a list with a given key inside of those
+Given the mysql hosts and privs subkey lists, you can also iterate over a list in a nested subkey::
+
+    - name: Setup MySQL users
+      mysql_user: name={{ item.0.user }} password={{ item.0.mysql.password }} host={{ item.1 }} priv={{ item.0.mysql.privs | join('/') }}
+      with_subelements:
+        - users
+        - mysql.hosts
+
+Subelements walks a list of hashes (aka dictionaries) and then traverses a list with a given (nested sub-)key inside of those
 records.
 
+Optionally,  you can add a third element to the subelements list, that holds a
+dictionary of flags. Currently you can add the 'skip_missing' flag. If set to
+True, the lookup plugin will skip the lists items that do not contain the given
+subkey. Without this flag, or if that flag is set to False, the plugin will
+yield an error and complain about the missing subkey.
+
 The authorized_key pattern is exactly where it comes up most.
 
 .. _looping_over_integer_sequences:

lib/ansible/plugins/lookup/subelements.py

@@ -20,18 +20,33 @@ __metaclass__ = type
 from ansible.errors import *
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.listify import listify_lookup_plugin_terms
+from ansible.utils.boolean import boolean
+
+FLAGS = ('skip_missing',)
+
 
 class LookupModule(LookupBase):
 
     def run(self, terms, variables, **kwargs):
 
-        terms[0] = listify_lookup_plugin_terms(terms[0], variables, loader=self._loader)
+        def _raise_terms_error(msg=""):
+            raise errors.AnsibleError(
+                "subelements lookup expects a list of two or three items, "
+                + msg)
+        terms = listify_lookup_plugin_terms(terms, self.basedir, inject)
+        terms[0] = listify_lookup_plugin_terms(terms[0], self.basedir, inject)
 
-        if not isinstance(terms, list) or not len(terms) == 2:
-            raise AnsibleError("subelements lookup expects a list of two items, first a dict or a list, and second a string")
+        # check lookup terms - check number of terms
+        if not isinstance(terms, list) or not 2 <= len(terms) <= 3:
+            _raise_terms_error()
 
-        if isinstance(terms[0], dict): # convert to list:
-            if terms[0].get('skipped',False) != False:
+        # first term should be a list (or dict), second a string holding the subkey
+        if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], basestring):
+            _raise_terms_error("first a dict or a list, second a string pointing to the subkey")
+        subelements = terms[1].split(".")
+
+        if isinstance(terms[0], dict):  # convert to list:
+            if terms[0].get('skipped', False) is not False:
                 # the registered result was completely skipped
                 return []
             elementlist = []
@@ -40,20 +55,47 @@ class LookupModule(LookupBase):
         else:
             elementlist = terms[0]
 
-        subelement = terms[1]
+        # check for optional flags in third term
+        flags = {}
+        if len(terms) == 3:
+            flags = terms[2]
+        if not isinstance(flags, dict) and not all([isinstance(key, basestring) and key in FLAGS for key in flags]):
+            _raise_terms_error("the optional third item must be a dict with flags %s" % FLAGS)
 
+        # build_items
         ret = []
         for item0 in elementlist:
             if not isinstance(item0, dict):
-                raise AnsibleError("subelements lookup expects a dictionary, got '%s'" %item0)
-            if item0.get('skipped', False) != False:
+                raise errors.AnsibleError("subelements lookup expects a dictionary, got '%s'" % item0)
+            if item0.get('skipped', False) is not False:
                 # this particular item is to be skipped
                 continue
-            if not subelement in item0:
-                raise AnsibleError("could not find '%s' key in iterated item '%s'" % (subelement, item0))
-            if not isinstance(item0[subelement], list):
-                raise AnsibleError("the key %s should point to a list, got '%s'" % (subelement, item0[subelement]))
-            sublist = item0.pop(subelement, [])
+
+            skip_missing = boolean(flags.get('skip_missing', False))
+            subvalue = item0
+            lastsubkey = False
+            sublist = []
+            for subkey in subelements:
+                if subkey == subelements[-1]:
+                    lastsubkey = True
+                if not subkey in subvalue:
+                    if skip_missing:
+                        continue
+                    else:
+                        raise errors.AnsibleError("could not find '%s' key in iterated item '%s'" % (subkey, subvalue))
+                if not lastsubkey:
+                    if not isinstance(subvalue[subkey], dict):
+                        if skip_missing:
+                            continue
+                        else:
+                            raise errors.AnsibleError("the key %s should point to a dictionary, got '%s'" % (subkey, subvalue[subkey]))
+                    else:
+                        subvalue = subvalue[subkey]
+                else:  # lastsubkey
+                    if not isinstance(subvalue[subkey], list):
+                        raise errors.AnsibleError("the key %s should point to a list, got '%s'" % (subkey, subvalue[subkey]))
+                    else:
+                        sublist = subvalue.pop(subkey, [])
             for item1 in sublist:
                 ret.append((item0, item1))
 

test/integration/roles/test_iterators/tasks/main.yml

@@ -97,6 +97,39 @@
         - "_ye == 'e'"
         - "_yf == 'f'"
 
+- name: test with_subelements in subkeys
+  set_fact: "{{ '_'+ item.0.id + item.1 }}={{ item.1 }}"
+  with_subelements:
+    - element_data
+    - the.sub.key.list
+
+- name: verify with_subelements in subkeys results
+  assert:
+    that:
+        - "_xq == 'q'"
+        - "_xr == 'r'"
+        - "_yi == 'i'"
+        - "_yo == 'o'"
+
+- name: test with_subelements with missing key or subkey
+  set_fact: "{{ '_'+ item.0.id + item.1 }}={{ item.1 }}"
+  with_subelements:
+    - element_data_missing
+    - the.sub.key.list
+    - skip_missing: yes
+  register: _subelements_missing_subkeys
+
+- debug: var=_subelements_missing_subkeys.skipped
+- debug: var=_subelements_missing_subkeys.results|length
+- name: verify with_subelements in subkeys results
+  assert:
+    that:
+        - _subelements_missing_subkeys.skipped is not defined
+        - _subelements_missing_subkeys.results|length == 2
+        - "_xk == 'k'"
+        - "_xl == 'l'"
+
+
 # WITH_TOGETHER        
 
 - name: test with_together

test/integration/roles/test_iterators/vars/main.yml

@@ -3,7 +3,41 @@ element_data:
     the_list:
       - "f"
       - "d"
+    the:
+      sub:
+        key:
+          list:
+            - "q"
+            - "r"
   - id: y
     the_list:
       - "e"
       - "f"
+    the:
+      sub:
+        key:
+          list:
+            - "i"
+            - "o"
+element_data_missing:
+  - id: x
+    the_list:
+      - "f"
+      - "d"
+    the:
+      sub:
+        key:
+          list:
+            - "k"
+            - "l"
+  - id: y
+    the_list:
+      - "f"
+      - "d"
+  - id: z
+    the_list:
+      - "e"
+      - "f"
+    the:
+      sub:
+        key: