vultr: use exponential backoff for api query retries (#60529)

changelogs/fragments/60529-vultry_retry_backoff.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - vultr - the retry on failure functionality was changed to use an exponential backoff behaviour.

lib/ansible/module_utils/vultr.py

@@ -7,6 +7,7 @@ __metaclass__ = type
 
 import os
 import time
+import random
 import urllib
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils._text import to_text, to_native
@@ -22,6 +23,7 @@ def vultr_argument_spec():
         api_key=dict(type='str', default=os.environ.get('VULTR_API_KEY'), no_log=True),
         api_timeout=dict(type='int', default=os.environ.get('VULTR_API_TIMEOUT')),
         api_retries=dict(type='int', default=os.environ.get('VULTR_API_RETRIES')),
+        api_retry_max_delay=dict(type='int', default=os.environ.get('VULTR_API_RETRY_MAX_DELAY')),
         api_account=dict(type='str', default=os.environ.get('VULTR_API_ACCOUNT') or 'default'),
         api_endpoint=dict(type='str', default=os.environ.get('VULTR_API_ENDPOINT')),
         validate_certs=dict(type='bool', default=True),
@@ -59,6 +61,7 @@ class Vultr:
                 'api_key': self.module.params.get('api_key') or config.get('key'),
                 'api_timeout': self.module.params.get('api_timeout') or int(config.get('timeout') or 60),
                 'api_retries': self.module.params.get('api_retries') or int(config.get('retries') or 5),
+                'api_retry_max_delay': self.module.params.get('api_retries') or int(config.get('retry_max_delay') or 12),
                 'api_endpoint': self.module.params.get('api_endpoint') or config.get('endpoint') or VULTR_API_ENDPOINT,
             }
         except ValueError as e:
@@ -74,6 +77,7 @@ class Vultr:
             'api_account': self.module.params.get('api_account'),
             'api_timeout': self.api_config['api_timeout'],
             'api_retries': self.api_config['api_retries'],
+            'api_retry_max_delay': self.api_config['api_retry_max_delay'],
             'api_endpoint': self.api_config['api_endpoint'],
         }
 
@@ -85,7 +89,7 @@ class Vultr:
         }
 
     def read_env_variables(self):
-        keys = ['key', 'timeout', 'retries', 'endpoint']
+        keys = ['key', 'timeout', 'retries', 'retry_max_delay', 'endpoint']
         env_conf = {}
         for key in keys:
             if 'VULTR_API_%s' % key.upper() not in os.environ:
@@ -159,7 +163,10 @@ class Vultr:
             except AttributeError:
                 data = urllib.parse.urlencode(data_encoded) + data_list
 
-        for s in range(0, self.api_config['api_retries']):
+        retry_max_delay = self.api_config['api_retry_max_delay']
+        randomness = random.randint(0, 1000) / 1000.0
+
+        for retry in range(0, self.api_config['api_retries']):
             response, info = fetch_url(
                 module=self.module,
                 url=url,
@@ -173,7 +180,11 @@ class Vultr:
                 break
 
             # Vultr has a rate limiting requests per second, try to be polite
-            time.sleep(1)
+            # Use exponential backoff plus a little bit of randomness
+            delay = 2 ** retry + randomness
+            if delay > retry_max_delay:
+                delay = retry_max_delay + randomness
+            time.sleep(delay)
 
         else:
             self.fail_json(msg="Reached API retries limit %s for URL %s, method %s with data %s. Returned %s, with body: %s %s" % (

lib/ansible/modules/cloud/vultr/_vultr_account_facts.py

@@ -58,6 +58,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/_vultr_block_storage_facts.py

@@ -57,6 +57,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/_vultr_os_facts.py

@@ -58,6 +58,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/_vultr_ssh_key_facts.py

@@ -58,6 +58,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/_vultr_user_facts.py

@@ -57,6 +57,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_account_info.py

@@ -54,6 +54,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_block_storage.py

@@ -78,6 +78,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_block_storage_info.py

@@ -57,6 +57,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_dns_domain.py

@@ -74,6 +74,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_dns_domain_facts.py

@@ -53,6 +53,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_dns_record.py

@@ -139,6 +139,22 @@ vultr_api:
       returned: success
       type: int
       sample: 60
+    api_retries:
+      description: Amount of max retries for the API requests
+      returned: success
+      type: int
+      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
+    api_endpoint:
+      description: Endpoint used for the API requests
+      returned: success
+      type: str
+      sample: "https://api.vultr.com"
 vultr_dns_record:
   description: Response from Vultr API
   returned: success

lib/ansible/modules/cloud/vultr/vultr_firewall_group.py

@@ -68,6 +68,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_firewall_group_facts.py

@@ -53,6 +53,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_firewall_rule.py

@@ -116,6 +116,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_network.py

@@ -76,6 +76,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_network_facts.py

@@ -53,6 +53,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_os_info.py

@@ -57,6 +57,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_plan_facts.py

@@ -54,6 +54,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_region_facts.py

@@ -54,6 +54,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_server.py

@@ -184,6 +184,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_server_facts.py

@@ -54,6 +54,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_ssh_key.py

@@ -73,6 +73,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_ssh_key_info.py

@@ -58,6 +58,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_startup_script.py

@@ -86,6 +86,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_startup_script_facts.py

@@ -54,6 +54,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_user.py

@@ -113,6 +113,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/modules/cloud/vultr/vultr_user_info.py

@@ -57,6 +57,12 @@ vultr_api:
       returned: success
       type: int
       sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
     api_endpoint:
       description: Endpoint used for the API requests
       returned: success

lib/ansible/plugins/doc_fragments/vultr.py

@@ -26,6 +26,13 @@ options:
       - The ENV variable C(VULTR_API_RETRIES) is used as default, when defined.
       - Fallback value is 5 retries if not specified.
     type: int
+  api_retry_max_delay:
+    description:
+      - Retry backoff delay in seconds is exponential up to this max. value, in seconds.
+      - The ENV variable C(VULTR_API_RETRY_MAX_DELAY) is used as default, when defined.
+      - Fallback value is 12 seconds.
+    type: int
+    version_added: '2.9'
   api_account:
     description:
       - Name of the ini section in the C(vultr.ini) file.