If we can't squash for any reason, then simply do not optimize the items loop.

Also add more squashing testcases

Fixes #15649

lib/ansible/executor/task_executor.py

@@ -256,6 +256,10 @@ class TaskExecutor:
         Squash items down to a comma-separated list for certain modules which support it
         (typically package management modules).
         '''
+        try:
+            # Hack just for the backport to 2.0
+            loop_var = 'item'
+
             # _task.action could contain templatable strings (via action: and
             # local_action:)  Template it before comparing.  If we don't end up
             # optimizing it here, the templatable string might use template vars
@@ -281,18 +285,18 @@ class TaskExecutor:
                     template_no_item = template_with_item = None
                     if name:
                         if templar._contains_vars(name):
-                        variables['item'] = '\0$'
+                            variables[loop_var] = '\0$'
                             template_no_item = templar.template(name, variables, cache=False)
-                        variables['item'] = '\0@'
+                            variables[loop_var] = '\0@'
                             template_with_item = templar.template(name, variables, cache=False)
-                        del variables['item']
+                            del variables[loop_var]
 
                         # Check if the user is doing some operation that doesn't take
                         # name/pkg or the name/pkg field doesn't have any variables
                         # and thus the items can't be squashed
                         if template_no_item != template_with_item:
                             for item in items:
-                            variables['item'] = item
+                                variables[loop_var] = item
                                 if self._task.evaluate_conditional(templar, variables):
                                     new_item = templar.template(name, cache=False)
                                     final_items.append(new_item)
@@ -309,6 +313,10 @@ class TaskExecutor:
                     # * lists can be squashed together
                     # * dicts could squash entries that match in all cases except the
                     #   name or pkg field.
+        except:
+            # Squashing is an optimization.  If it fails for any reason,
+            # simply use the unoptimized list of items.
+            pass
         return items
 
     def _execute(self, variables=None):

test/units/executor/test_task_executor.py

@@ -230,9 +230,31 @@ class TestTaskExecutor(unittest.TestCase):
         new_items = te._squash_items(items=items, variables=job_vars)
         self.assertEqual(new_items, ['a', 'b', 'c'])
 
+        mock_task.action = '{{unknown}}'
+        mock_task.args={'name': '{{item}}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        self.assertEqual(new_items, ['a', 'b', 'c'])
+
+        # Maybe should raise an error in this case.  The user would have to specify:
+        # - yum: name="{{ packages[item] }}"
+        #   with_items:
+        #     - ['a', 'b']
+        #     - ['foo', 'bar']
+        # you can't use a list as a dict key so that would probably throw
+        # an error later.  If so, we can throw it now instead.
+        # Squashing in this case would not be intuitive as the user is being
+        # explicit in using each list entry as a key.  
+        job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "foo": "baz", "bar": "quux" })
+        items = [['a', 'b'], ['foo', 'bar']]
+        mock_task.action = 'yum'
+        mock_task.args = {'name': '{{ packages[item] }}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        self.assertEqual(new_items, items)
+
         #
         # Replaces
         #
+        items = ['a', 'b', 'c']
         mock_task.action = 'yum'
         mock_task.args={'name': '{{item}}'}
         new_items = te._squash_items(items=items, variables=job_vars)
@@ -243,22 +265,75 @@ class TestTaskExecutor(unittest.TestCase):
         new_items = te._squash_items(items=items, variables=job_vars)
         self.assertEqual(new_items, [['a', 'c']])
 
+        # Loop var isn't present until 2.1.x
+        # New loop_var
+        #mock_task.action = 'yum'
+        #mock_task.args = {'name': '{{a_loop_var_item}}'}
+        #mock_task.loop_control = {'loop_var': 'a_loop_var_item'}
+        #loop_var = 'a_loop_var_item'
+        #new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars)
+        #self.assertEqual(new_items, [['a', 'c']])
+        #loop_var = 'item'
+
         #
-        # Smoketests -- these won't optimize but make sure that they don't
+        # These are presently not optimized but could be in the future.
-        # traceback either
+        # Expected output if they were optimized is given as a comment
+        # Please move these to a different section if they are optimized
         #
-        mock_task.action = '{{unknown}}'
+
-        mock_task.args={'name': '{{item}}'}
+        # Squashing lists
+        job_vars = dict(pkg_mgr='yum')
+        items = [['a', 'b'], ['foo', 'bar']]
+        mock_task.action = 'yum'
+        mock_task.args = {'name': '{{ item }}'}
         new_items = te._squash_items(items=items, variables=job_vars)
-        self.assertEqual(new_items, ['a', 'b', 'c'])
+        #self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']])
+        self.assertEqual(new_items, items)
+
+        # Retrieving from a dict
+        items = ['a', 'b', 'foo']
+        mock_task.action = 'yum'
+        mock_task.args = {'name': '{{ packages[item] }}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        #self.assertEqual(new_items, [['foo', 'baz']])
+        self.assertEqual(new_items, items)
+
+        # Another way to retrieve from a dict
+        job_vars = dict(pkg_mgr='yum')
+        items = [{'package': 'foo'}, {'package': 'bar'}]
+        mock_task.action = 'yum'
+        mock_task.args = {'name': '{{ item["package"] }}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        #self.assertEqual(new_items, [['foo', 'bar']])
+        self.assertEqual(new_items, items)
 
         items = [dict(name='a', state='present'),
                 dict(name='b', state='present'),
                 dict(name='c', state='present')]
         mock_task.action = 'yum'
-        mock_task.args={'name': '{{item}}'}
+        mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
         new_items = te._squash_items(items=items, variables=job_vars)
         self.assertEqual(new_items, items)
+        #self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')])
+
+        items = [dict(name='a', state='present'),
+                dict(name='b', state='present'),
+                dict(name='c', state='absent')]
+        mock_task.action = 'yum'
+        mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        self.assertEqual(new_items, items)
+        #self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'),
+        #        dict(name='c', state='absent')])
+
+        # Could do something like this to recover from bad deps in a package
+        job_vars = dict(pkg_mgr='yum', packages=['a', 'b'])
+        items = [ 'absent', 'latest' ]
+        mock_task.action = 'yum'
+        mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'}
+        new_items = te._squash_items(items=items, variables=job_vars)
+        self.assertEqual(new_items, items)
+
 
     def test_task_executor_execute(self):
         fake_loader = DictDataLoader({})