# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import time
from functools
import wraps
from contextlib
import contextmanager
import logging
import random
log = logging.getLogger(__name__)
def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1):
"""
A generator function that sleeps between retries, handles exponential
backoff
and jitter. The action you are retrying
is meant to run after
retrier yields.
At each iteration, we sleep
for sleeptime + random.uniform(-jitter, jitter).
Afterwards sleeptime
is multiplied by sleepscale
for the next iteration.
Args:
attempts (int): maximum number of times to
try; defaults to 5
sleeptime (float): how many seconds to sleep between tries; defaults to
10 seconds
max_sleeptime (float): the longest we
'll sleep, in seconds; defaults to
300s (five minutes)
sleepscale (float): how much to multiply the sleep time by each
iteration; defaults to 1.5
jitter (float): random jitter to introduce to sleep time each iteration.
the amount
is chosen at random between [-jitter, +jitter]
defaults to 1
Yields:
None, a maximum of `attempts` number of times
Example:
>>> n = 0
>>>
for _
in retrier(sleeptime=0, jitter=0):
...
if n == 3:
...
# We did the thing!
...
break
... n += 1
>>> n
3
>>> n = 0
>>>
for _
in retrier(sleeptime=0, jitter=0):
...
if n == 6:
...
# We did the thing!
...
break
... n += 1
...
else:
... print(
"max tries hit")
max tries hit
"""
jitter = jitter
or 0
# py35 barfs on the next line if jitter is None
if jitter > sleeptime:
# To prevent negative sleep times
raise Exception(
"jitter ({}) must be less than sleep time ({})".format(jitter, sleeptime)
)
sleeptime_real = sleeptime
for _
in range(attempts):
log.debug(
"attempt %i/%i", _ + 1, attempts)
yield sleeptime_real
if jitter:
sleeptime_real = sleeptime + random.uniform(-jitter, jitter)
# our jitter should scale along with the sleeptime
jitter = jitter * sleepscale
else:
sleeptime_real = sleeptime
sleeptime *= sleepscale
if sleeptime_real > max_sleeptime:
sleeptime_real = max_sleeptime
# Don't need to sleep the last time
if _ < attempts - 1:
log.debug(
"sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts
)
time.sleep(sleeptime_real)
def retry(
action,
attempts=5,
sleeptime=60,
max_sleeptime=5 * 60,
sleepscale=1.5,
jitter=1,
retry_exceptions=(Exception,),
cleanup=
None,
args=(),
kwargs={},
log_args=
True,
):
"""
Calls an action function until it succeeds,
or we give up.
Args:
action (callable): the function to retry
attempts (int): maximum number of times to
try; defaults to 5
sleeptime (float): how many seconds to sleep between tries; defaults to
60s (one minute)
max_sleeptime (float): the longest we
'll sleep, in seconds; defaults to
300s (five minutes)
sleepscale (float): how much to multiply the sleep time by each
iteration; defaults to 1.5
jitter (float): random jitter to introduce to sleep time each iteration.
the amount
is chosen at random between [-jitter, +jitter]
defaults to 1
retry_exceptions (tuple): tuple of exceptions to be caught.
If other
exceptions are raised by action(), then these
are immediately re-raised to the caller.
cleanup (callable): optional; called
if one of `retry_exceptions`
is
caught. No arguments are passed to the cleanup
function;
if your cleanup requires arguments,
consider using functools.partial
or a
lambda
function.
args (tuple): positional arguments to call `action`
with
kwargs (dict): keyword arguments to call `action`
with
log_args (bool): whether
or not to include args
and kwargs
in log
messages. Defaults to
True.
Returns:
Whatever action(*args, **kwargs) returns
Raises:
Whatever action(*args, **kwargs) raises. `retry_exceptions` are caught
up until the last attempt,
in which case they are re-raised.
Example:
>>> count = 0
>>>
def foo():
...
global count
... count += 1
... print(count)
...
if count < 3:
...
raise ValueError(
"count is too small!")
...
return "success!"
>>> retry(foo, sleeptime=0, jitter=0)
1
2
3
'success!'
"""
assert callable(action)
assert not cleanup
or callable(cleanup)
action_name = getattr(action,
"__name__", action)
if log_args
and (args
or kwargs):
log_attempt_args = (
"retry: calling %s with args: %s," " kwargs: %s, attempt #%d",
action_name,
args,
kwargs,
)
else:
log_attempt_args = (
"retry: calling %s, attempt #%d", action_name)
if max_sleeptime < sleeptime:
log.debug(
"max_sleeptime %d less than sleeptime %d", max_sleeptime, sleeptime)
n = 1
for _
in retrier(
attempts=attempts,
sleeptime=sleeptime,
max_sleeptime=max_sleeptime,
sleepscale=sleepscale,
jitter=jitter,
):
try:
logfn = log.info
if n != 1
else log.debug
logfn_args = log_attempt_args + (n,)
logfn(*logfn_args)
return action(*args, **kwargs)
except retry_exceptions:
log.debug(
"retry: Caught exception: ", exc_info=
True)
if cleanup:
cleanup()
if n == attempts:
log.info(
"retry: Giving up on %s", action_name)
raise
continue
finally:
n += 1
def retriable(*retry_args, **retry_kwargs):
"""
A decorator factory
for retry(). Wrap your function
in @retriable(...) to
give it retry powers!
Arguments:
Same
as for `retry`,
with the exception of `action`, `args`,
and `kwargs`,
which are left to the normal function definition.
Returns:
A function decorator
Example:
>>> count = 0
>>> @retriable(sleeptime=0, jitter=0)
...
def foo():
...
global count
... count += 1
... print(count)
...
if count < 3:
...
raise ValueError(
"count too small")
...
return "success!"
>>> foo()
1
2
3
'success!'
"""
def _retriable_factory(func):
@wraps(func)
def _retriable_wrapper(*args, **kwargs):
return retry(func, args=args, kwargs=kwargs, *retry_args, **retry_kwargs)
return _retriable_wrapper
return _retriable_factory
@contextmanager
def retrying(func, *retry_args, **retry_kwargs):
"""
A context manager
for wrapping functions
with retry functionality.
Arguments:
func (callable): the function to wrap
other arguments
as per `retry`
Returns:
A context manager that returns retriable(func) on __enter__
Example:
>>> count = 0
>>>
def foo():
...
global count
... count += 1
... print(count)
...
if count < 3:
...
raise ValueError(
"count too small")
...
return "success!"
>>>
with retrying(foo, sleeptime=0, jitter=0)
as f:
... f()
1
2
3
'success!'
"""
yield retriable(*retry_args, **retry_kwargs)(func)
