FEATURE: adding variable serial batches

This feature changes the scalar value of `serial:` to a list, which
allows users to specify a list of values, so batches can be ramped
up (commonly called "canary" setups):

- hosts: all
  serial: [1, 5, 10, "100%"]
  tasks:
  ...

CHANGELOG.md

@@ -13,6 +13,7 @@ Ansible Changes By Release
   - ansible_date_time.iso8601 (changed to UTC instead of local time)
   - ansible_distribution (now uses OS caption string, e.g.: "Microsoft Windows Server 2012 R2 Standard", version is still available on ansible_distribution_version)
   - ansible_totalmem (renamed to ansible_memtotal_mb, units changed to MB instead of bytes)
+- Added the ability to specify serial batches as a list (`serial: [1, 5, 10]`), which allows for so-called "canary" actions in one play.
 
 ####New Modules:
 - asa

docsite/rst/playbooks_delegation.rst

@@ -42,6 +42,36 @@ play, in order to determine the number of hosts per pass::
 
 If the number of hosts does not divide equally into the number of passes, the final pass will contain the remainder.
 
+As of Ansible 2.2, the batch sizes can be specified as a list, as follows::
+
+    - name: test play
+      hosts: webservers
+      serial:
+      - 1
+      - 5
+      - 10
+
+In the above example, the first batch would contain a single host, the next would contain 5 hosts, and (if there are any hosts left),
+every following batch would contain 10 hosts until all available hosts are used.
+
+It is also possible to list multiple batche sizes as percentages::
+
+    - name: test play
+      hosts: webservers
+      serial:
+      - "10%"
+      - "20%"
+      - "100%"
+
+You can also mix and match the values::
+
+    - name: test play
+      hosts: webservers
+      serial:
+      - 1
+      - 5
+      - "20%"
+
 .. note::
      No matter how small the percentage, the number of hosts per pass will always be 1 or greater.
 

lib/ansible/executor/playbook_executor.py

@@ -27,6 +27,7 @@ from ansible import constants as C
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.playbook import Playbook
 from ansible.template import Templar
+from ansible.utils.helpers import pct_to_int
 from ansible.utils.path import makedirs_safe
 from ansible.utils.unicode import to_unicode, to_str
 
@@ -228,27 +229,28 @@ class PlaybookExecutor:
 
         # make sure we have a unique list of hosts
         all_hosts = self._inventory.get_hosts(play.hosts)
-
-        # check to see if the serial number was specified as a percentage,
-        # and convert it to an integer value based on the number of hosts
-        if isinstance(play.serial, string_types) and play.serial.endswith('%'):
-            serial_pct = int(play.serial.replace("%",""))
-            serial = int((serial_pct/100.0) * len(all_hosts)) or 1
-        else:
-            if play.serial is None:
-                serial = -1
+        all_hosts_len = len(all_hosts)
+
+        # the serial value can be listed as a scalar or a list of
+        # scalars, so we make sure it's a list here
+        serial_batch_list = play.serial
+        if len(serial_batch_list) == 0:
+            serial_batch_list = [-1]
+
+        cur_item = 0
+        serialized_batches = []
+
+        while len(all_hosts) > 0:
+            # get the serial value from current item in the list
+            serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len)
+
+            # if the serial count was not specified or is invalid, default to
+            # a list of all hosts, otherwise grab a chunk of the hosts equal
+            # to the current serial item size
+            if serial <= 0:
+                serialized_batches.append(all_hosts)
+                break
             else:
-                serial = int(play.serial)
-
-        # if the serial count was not specified or is invalid, default to
-        # a list of all hosts, otherwise split the list of hosts into chunks
-        # which are based on the serial size
-        if serial <= 0:
-            return [all_hosts]
-        else:
-            serialized_batches = []
-
-            while len(all_hosts) > 0:
                 play_hosts = []
                 for x in range(serial):
                     if len(all_hosts) > 0:
@@ -256,7 +258,14 @@ class PlaybookExecutor:
 
                 serialized_batches.append(play_hosts)
 
-            return serialized_batches
+            # increment the current batch list item number, and if we've hit
+            # the end keep using the last element until we've consumed all of
+            # the hosts in the inventory
+            cur_item += 1
+            if cur_item > len(serial_batch_list) - 1:
+                cur_item = len(serial_batch_list) - 1
+
+        return serialized_batches
 
     def _generate_retry_inventory(self, retry_path, replay_hosts):
         '''

lib/ansible/executor/task_queue_manager.py

@@ -34,6 +34,7 @@ from ansible.plugins import callback_loader, strategy_loader, module_loader
 from ansible.template import Templar
 from ansible.vars.hostvars import HostVars
 from ansible.plugins.callback import CallbackBase
+from ansible.utils.helpers import pct_to_int
 from ansible.utils.unicode import to_unicode
 from ansible.compat.six import string_types
 
@@ -225,8 +226,19 @@ class TaskQueueManager:
         )
 
         # Fork # of forks, # of hosts or serial, whichever is lowest
-        contenders =  [self._options.forks, play.serial, len(self._inventory.get_hosts(new_play.hosts))]
-        contenders =  [ v for v in contenders if v is not None and v > 0 ]
+        num_hosts = len(self._inventory.get_hosts(new_play.hosts))
+
+        max_serial = 0
+        if play.serial:
+            # the play has not been post_validated here, so we may need
+            # to convert the scalar value to a list at this point
+            serial_items = play.serial
+            if not isinstance(serial_items, list):
+                serial_items = [serial_items]
+            max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])
+
+        contenders =  [self._options.forks, max_serial, num_hosts]
+        contenders =  [v for v in contenders if v is not None and v > 0]
         self._initialize_processes(min(contenders))
 
         play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno())

lib/ansible/playbook/play.py

@@ -87,7 +87,7 @@ class Play(Base, Taggable, Become):
     _any_errors_fatal    = FieldAttribute(isa='bool', default=False, always_post_validate=True)
     _force_handlers      = FieldAttribute(isa='bool', always_post_validate=True)
     _max_fail_percentage = FieldAttribute(isa='percent', always_post_validate=True)
-    _serial              = FieldAttribute(isa='string',  always_post_validate=True)
+    _serial              = FieldAttribute(isa='list', default=[], always_post_validate=True)
     _strategy            = FieldAttribute(isa='string', default='linear', always_post_validate=True)
 
     # =================================================================================

lib/ansible/utils/helpers.py

@@ -0,0 +1,34 @@
+# (c) 2016, Ansible by Red Hat <support@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.compat.six import string_types
+
+def pct_to_int(value, num_items, min_value=1):
+    '''
+    Converts a given value to a percentage if specified as "x%",
+    otherwise converts the given value to an integer. 
+    '''
+    if isinstance(value, string_types) and value.endswith('%'):
+        value_pct = int(value.replace("%",""))
+        return int((value_pct/100.0) * num_items) or min_value
+    else:
+        return int(value)
+

test/units/executor/test_playbook_executor.py

@@ -25,10 +25,11 @@ from ansible.compat.tests.mock import patch, MagicMock
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.playbook import Playbook
+from ansible.template import Templar
 
 from units.mock.loader import DictDataLoader
 
-class TestPlayIterator(unittest.TestCase):
+class TestPlaybookExecutor(unittest.TestCase):
 
     def setUp(self):
         pass
@@ -58,6 +59,20 @@ class TestPlayIterator(unittest.TestCase):
               tasks:
               - debug: var=inventory_hostname
             ''',
+            'serial_list.yml': '''
+            - hosts: all
+              gather_facts: no
+              serial: [1, 2, 3]
+              tasks:
+              - debug: var=inventory_hostname
+            ''',
+            'serial_list_mixed.yml': '''
+            - hosts: all
+              gather_facts: no
+              serial: [1, "20%", -1]
+              tasks:
+              - debug: var=inventory_hostname
+            ''',
         })
 
         mock_inventory = MagicMock()
@@ -68,8 +83,10 @@ class TestPlayIterator(unittest.TestCase):
         mock_options = MagicMock()
         mock_options.syntax.value = True
 
+        templar = Templar(loader=fake_loader)
+
         pbe = PlaybookExecutor(
-            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
+            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml', 'serial_list.yml', 'serial_list_mixed.yml'],
             inventory=mock_inventory,
             variable_manager=mock_var_manager,
             loader=fake_loader,
@@ -79,27 +96,44 @@ class TestPlayIterator(unittest.TestCase):
 
         playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
         play = playbook.get_plays()[0]
+        play.post_validate(templar)
         mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
         self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']])
 
         playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
         play = playbook.get_plays()[0]
+        play.post_validate(templar)
         mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
         self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
 
         playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
         play = playbook.get_plays()[0]
+        play.post_validate(templar)
         mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
         self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
 
+        playbook = Playbook.load(pbe._playbooks[3], variable_manager=mock_var_manager, loader=fake_loader)
+        play = playbook.get_plays()[0]
+        play.post_validate(templar)
+        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
+        self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5'],['host6','host7','host8'],['host9']])
+
+        playbook = Playbook.load(pbe._playbooks[4], variable_manager=mock_var_manager, loader=fake_loader)
+        play = playbook.get_plays()[0]
+        play.post_validate(templar)
+        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
+        self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5','host6','host7','host8','host9']])
+
         # Test when serial percent is under 1.0
         playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
         play = playbook.get_plays()[0]
+        play.post_validate(templar)
         mock_inventory.get_hosts.return_value = ['host0','host1','host2']
         self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1'],['host2']])
 
         # Test when there is a remainder for serial as a percent
         playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
         play = playbook.get_plays()[0]
+        play.post_validate(templar)
         mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9','host10']
         self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9'],['host10']])

test/units/utils/test_helpers.py

@@ -0,0 +1,31 @@
+# (c) 2015, Marius Gedminas <marius@gedmin.as>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+from ansible.utils.helpers import pct_to_int
+
+class TestHelpers(unittest.TestCase):
+
+    def test_pct_to_int(self):
+        self.assertEqual(pct_to_int(1, 100), 1)
+        self.assertEqual(pct_to_int(-1, 100), -1)
+        self.assertEqual(pct_to_int("1%", 10), 1)
+        self.assertEqual(pct_to_int("1%", 10, 0), 0)
+        self.assertEqual(pct_to_int("1", 100), 1)
+        self.assertEqual(pct_to_int("10%", 100), 10)
+