|
|
|
@ -31,10 +31,15 @@ The 'cloud' module provides the following common classes:
|
|
|
|
|
provide a backoff/retry decorator based on status codes.
|
|
|
|
|
|
|
|
|
|
- Example using the AWSRetry class which inherits from CloudRetry.
|
|
|
|
|
@AWSRetry.retry(tries=20, delay=2, backoff=2)
|
|
|
|
|
|
|
|
|
|
@AWSRetry.exponential_backoff(retries=10, delay=3)
|
|
|
|
|
get_ec2_security_group_ids_from_names()
|
|
|
|
|
|
|
|
|
|
@AWSRetry.jittered_backoff()
|
|
|
|
|
get_ec2_security_group_ids_from_names()
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
import random
|
|
|
|
|
from functools import wraps
|
|
|
|
|
import syslog
|
|
|
|
|
import time
|
|
|
|
@ -42,6 +47,60 @@ import time
|
|
|
|
|
from ansible.module_utils.pycompat24 import get_exception
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60):
|
|
|
|
|
""" Customizable exponential backoff strategy.
|
|
|
|
|
Args:
|
|
|
|
|
retries (int): Maximum number of times to retry a request.
|
|
|
|
|
delay (float): Initial (base) delay.
|
|
|
|
|
backoff (float): base of the exponent to use for exponential
|
|
|
|
|
backoff.
|
|
|
|
|
max_delay (int): Optional. If provided each delay generated is capped
|
|
|
|
|
at this amount. Defaults to 60 seconds.
|
|
|
|
|
Returns:
|
|
|
|
|
Callable that returns a generator. This generator yields durations in
|
|
|
|
|
seconds to be used as delays for an exponential backoff strategy.
|
|
|
|
|
Usage:
|
|
|
|
|
>>> backoff = _exponential_backoff()
|
|
|
|
|
>>> backoff
|
|
|
|
|
<function backoff_backoff at 0x7f0d939facf8>
|
|
|
|
|
>>> list(backoff())
|
|
|
|
|
[2, 4, 8, 16, 32, 60, 60, 60, 60, 60]
|
|
|
|
|
"""
|
|
|
|
|
def backoff_gen():
|
|
|
|
|
for retry in range(0, retries):
|
|
|
|
|
sleep = delay * backoff ** retry
|
|
|
|
|
yield sleep if max_delay is None else min(sleep, max_delay)
|
|
|
|
|
return backoff_gen
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random):
|
|
|
|
|
""" Implements the "Full Jitter" backoff strategy described here
|
|
|
|
|
https://www.awsarchitectureblog.com/2015/03/backoff.html
|
|
|
|
|
Args:
|
|
|
|
|
retries (int): Maximum number of times to retry a request.
|
|
|
|
|
delay (float): Approximate number of seconds to sleep for the first
|
|
|
|
|
retry.
|
|
|
|
|
max_delay (int): The maximum number of seconds to sleep for any retry.
|
|
|
|
|
_random (random.Random or None): Makes this generator testable by
|
|
|
|
|
allowing developers to explicitly pass in the a seeded Random.
|
|
|
|
|
Returns:
|
|
|
|
|
Callable that returns a generator. This generator yields durations in
|
|
|
|
|
seconds to be used as delays for a full jitter backoff strategy.
|
|
|
|
|
Usage:
|
|
|
|
|
>>> backoff = _full_jitter_backoff(retries=5)
|
|
|
|
|
>>> backoff
|
|
|
|
|
<function backoff_backoff at 0x7f0d939facf8>
|
|
|
|
|
>>> list(backoff())
|
|
|
|
|
[3, 6, 5, 23, 38]
|
|
|
|
|
>>> list(backoff())
|
|
|
|
|
[2, 1, 6, 6, 31]
|
|
|
|
|
"""
|
|
|
|
|
def backoff_gen():
|
|
|
|
|
for retry in range(0, retries):
|
|
|
|
|
yield _random.randint(0, min(max_delay, delay * 2 ** retry))
|
|
|
|
|
return backoff_gen
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CloudRetry(object):
|
|
|
|
|
""" CloudRetry can be used by any cloud provider, in order to implement a
|
|
|
|
|
backoff algorithm/retry effect based on Status Code from Exceptions.
|
|
|
|
@ -67,22 +126,18 @@ class CloudRetry(object):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def backoff(cls, tries=10, delay=3, backoff=1.1):
|
|
|
|
|
""" Retry calling the Cloud decorated function using an exponential backoff.
|
|
|
|
|
Kwargs:
|
|
|
|
|
tries (int): Number of times to try (not retry) before giving up
|
|
|
|
|
default=10
|
|
|
|
|
delay (int): Initial delay between retries in seconds
|
|
|
|
|
default=3
|
|
|
|
|
backoff (int): backoff multiplier e.g. value of 2 will double the delay each retry
|
|
|
|
|
default=2
|
|
|
|
|
|
|
|
|
|
def _backoff(cls, backoff_strategy):
|
|
|
|
|
""" Retry calling the Cloud decorated function using the provided
|
|
|
|
|
backoff strategy.
|
|
|
|
|
Args:
|
|
|
|
|
backoff_strategy (callable): Callable that returns a generator. The
|
|
|
|
|
generator should yield sleep times for each retry of the decorated
|
|
|
|
|
function.
|
|
|
|
|
"""
|
|
|
|
|
def deco(f):
|
|
|
|
|
@wraps(f)
|
|
|
|
|
def retry_func(*args, **kwargs):
|
|
|
|
|
max_tries, max_delay = tries, delay
|
|
|
|
|
while max_tries > 1:
|
|
|
|
|
for delay in backoff_strategy():
|
|
|
|
|
try:
|
|
|
|
|
return f(*args, **kwargs)
|
|
|
|
|
except Exception:
|
|
|
|
@ -90,11 +145,9 @@ class CloudRetry(object):
|
|
|
|
|
if isinstance(e, cls.base_class):
|
|
|
|
|
response_code = cls.status_code_from_exception(e)
|
|
|
|
|
if cls.found(response_code):
|
|
|
|
|
msg = "{0}: Retrying in {1} seconds...".format(str(e), max_delay)
|
|
|
|
|
msg = "{0}: Retrying in {1} seconds...".format(str(e), delay)
|
|
|
|
|
syslog.syslog(syslog.LOG_INFO, msg)
|
|
|
|
|
time.sleep(max_delay)
|
|
|
|
|
max_tries -= 1
|
|
|
|
|
max_delay *= backoff
|
|
|
|
|
time.sleep(delay)
|
|
|
|
|
else:
|
|
|
|
|
# Return original exception if exception is not a ClientError
|
|
|
|
|
raise e
|
|
|
|
@ -106,3 +159,62 @@ class CloudRetry(object):
|
|
|
|
|
return retry_func # true decorator
|
|
|
|
|
|
|
|
|
|
return deco
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60):
|
|
|
|
|
"""
|
|
|
|
|
Retry calling the Cloud decorated function using an exponential backoff.
|
|
|
|
|
|
|
|
|
|
Kwargs:
|
|
|
|
|
retries (int): Number of times to retry a failed request before giving up
|
|
|
|
|
default=10
|
|
|
|
|
delay (int or float): Initial delay between retries in seconds
|
|
|
|
|
default=3
|
|
|
|
|
backoff (int or float): backoff multiplier e.g. value of 2 will
|
|
|
|
|
double the delay each retry
|
|
|
|
|
default=1.1
|
|
|
|
|
max_delay (int or None): maximum amount of time to wait between retries.
|
|
|
|
|
default=60
|
|
|
|
|
"""
|
|
|
|
|
return cls._backoff(_exponential_backoff(
|
|
|
|
|
retries=retries, delay=delay, backoff=backoff, max_delay=max_delay))
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def jittered_backoff(cls, retries=10, delay=3, max_delay=60):
|
|
|
|
|
"""
|
|
|
|
|
Retry calling the Cloud decorated function using a jittered backoff
|
|
|
|
|
strategy. More on this strategy here:
|
|
|
|
|
|
|
|
|
|
https://www.awsarchitectureblog.com/2015/03/backoff.html
|
|
|
|
|
|
|
|
|
|
Kwargs:
|
|
|
|
|
retries (int): Number of times to retry a failed request before giving up
|
|
|
|
|
default=10
|
|
|
|
|
delay (int): Initial delay between retries in seconds
|
|
|
|
|
default=3
|
|
|
|
|
max_delay (int): maximum amount of time to wait between retries.
|
|
|
|
|
default=60
|
|
|
|
|
"""
|
|
|
|
|
return cls._backoff(_full_jitter_backoff(
|
|
|
|
|
retries=retries, delay=delay, max_delay=max_delay))
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def backoff(cls, tries=10, delay=3, backoff=1.1):
|
|
|
|
|
"""
|
|
|
|
|
Retry calling the Cloud decorated function using an exponential backoff.
|
|
|
|
|
|
|
|
|
|
Compatibility for the original implementation of CloudRetry.backoff that
|
|
|
|
|
did not provide configurable backoff strategies. Developers should use
|
|
|
|
|
CloudRetry.exponential_backoff instead.
|
|
|
|
|
|
|
|
|
|
Kwargs:
|
|
|
|
|
tries (int): Number of times to try (not retry) before giving up
|
|
|
|
|
default=10
|
|
|
|
|
delay (int or float): Initial delay between retries in seconds
|
|
|
|
|
default=3
|
|
|
|
|
backoff (int or float): backoff multiplier e.g. value of 2 will
|
|
|
|
|
double the delay each retry
|
|
|
|
|
default=1.1
|
|
|
|
|
"""
|
|
|
|
|
return cls.exponential_backoff(
|
|
|
|
|
retries=tries - 1, delay=delay, backoff=backoff, max_delay=None)
|
|
|
|
|