261 lines
10 KiB
Python
261 lines
10 KiB
Python
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import logging
|
|
import random
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
from autotest_lib.client.common_lib import env
|
|
from autotest_lib.client.common_lib import error
|
|
|
|
|
|
def install_sigalarm_handler(new_handler):
|
|
"""
|
|
Try installing a sigalarm handler.
|
|
|
|
In order to protect apache, wsgi intercepts any attempt to install a
|
|
sigalarm handler, so our function will feel the full force of a sigalarm
|
|
even if we try to install a pacifying signal handler. To avoid this we
|
|
need to confirm that the handler we tried to install really was installed.
|
|
|
|
@param new_handler: The new handler to install. This must be a callable
|
|
object, or signal.SIG_IGN/SIG_DFL which correspond to
|
|
the numbers 1,0 respectively.
|
|
@return: True if the installation of new_handler succeeded, False otherwise.
|
|
"""
|
|
# Installing signal handlers does not and is never expected to work if we're
|
|
# running in a mod_wsgi process.
|
|
if env.IN_MOD_WSGI:
|
|
return False
|
|
|
|
if (new_handler is None or
|
|
(not callable(new_handler) and
|
|
new_handler != signal.SIG_IGN and
|
|
new_handler != signal.SIG_DFL)):
|
|
logging.warning('Trying to install an invalid sigalarm handler.')
|
|
return False
|
|
|
|
signal.signal(signal.SIGALRM, new_handler)
|
|
installed_handler = signal.getsignal(signal.SIGALRM)
|
|
return installed_handler == new_handler
|
|
|
|
|
|
def set_sigalarm_timeout(timeout_secs, default_timeout=60):
|
|
"""
|
|
Set the sigalarm timeout.
|
|
|
|
This methods treats any timeout <= 0 as a possible error and falls back to
|
|
using it's default timeout, since negative timeouts can have 'alarming'
|
|
effects. Though 0 is a valid timeout, it is often used to cancel signals; in
|
|
order to set a sigalarm of 0 please call signal.alarm directly as there are
|
|
many situations where a 0 timeout is considered invalid.
|
|
|
|
@param timeout_secs: The new timeout, in seconds.
|
|
@param default_timeout: The default timeout to use, if timeout <= 0.
|
|
@return: The old sigalarm timeout
|
|
"""
|
|
timeout_sec_n = int(timeout_secs)
|
|
if timeout_sec_n <= 0:
|
|
timeout_sec_n = int(default_timeout)
|
|
return signal.alarm(timeout_sec_n)
|
|
|
|
|
|
def sigalarm_wrapper(message):
|
|
"""
|
|
Raise a TimeoutException with the given message. Needed because the body
|
|
of a closure (lambda) can only be an expression, not a statement (such
|
|
as "raise") :P :P :P
|
|
|
|
@param message: the exception message.
|
|
"""
|
|
raise error.TimeoutException(message)
|
|
|
|
|
|
def custom_sigalarm_handler(func, timeout_sec):
|
|
"""
|
|
Returns a sigalarm handler which produces an exception with a custom
|
|
error message (function name and timeout length) instead of a generic
|
|
one.
|
|
|
|
@param func: the function that may time out
|
|
@param timeout_sec: timeout length in seconds
|
|
"""
|
|
try:
|
|
name = str(func.__name__)
|
|
except Exception as e:
|
|
name = '(unavailable function name: exception: %s)' % e
|
|
message = "sigalarm timeout (%d seconds) in %s" % (timeout_sec, name)
|
|
return lambda signum, frame: sigalarm_wrapper(message)
|
|
|
|
|
|
def timeout(func, args=(), kwargs={}, timeout_sec=60.0, default_result=None):
|
|
"""
|
|
This function run the given function using the args, kwargs and
|
|
return the given default value if the timeout_sec is exceeded.
|
|
|
|
@param func: function to be called.
|
|
@param args: arguments for function to be called.
|
|
@param kwargs: keyword arguments for function to be called.
|
|
@param timeout_sec: timeout setting for call to exit, in seconds.
|
|
@param default_result: default return value for the function call.
|
|
|
|
@return 1: is_timeout 2: result of the function call. If
|
|
is_timeout is True, the call is timed out. If the
|
|
value is False, the call is finished on time.
|
|
"""
|
|
old_alarm_sec = 0
|
|
old_handler = signal.getsignal(signal.SIGALRM)
|
|
handler = custom_sigalarm_handler(func, timeout_sec)
|
|
installed_handler = install_sigalarm_handler(handler)
|
|
if installed_handler:
|
|
old_alarm_sec = set_sigalarm_timeout(timeout_sec, default_timeout=60)
|
|
|
|
# If old_timeout_time = 0 we either didn't install a handler, or sigalrm
|
|
# had a signal.SIG_DFL handler with 0 timeout. In the latter case we still
|
|
# need to restore the handler/timeout.
|
|
old_timeout_time = (time.time() + old_alarm_sec) if old_alarm_sec > 0 else 0
|
|
|
|
try:
|
|
default_result = func(*args, **kwargs)
|
|
return False, default_result
|
|
except error.TimeoutException:
|
|
return True, default_result
|
|
finally:
|
|
# If we installed a sigalarm handler, cancel it since our function
|
|
# returned on time. If we can successfully restore the old handler,
|
|
# reset the old timeout, or, if the old timeout's deadline has passed,
|
|
# set the sigalarm to fire in one second. If the old_timeout_time is 0
|
|
# we don't need to set the sigalarm timeout since we have already set it
|
|
# as a byproduct of cancelling the current signal.
|
|
if installed_handler:
|
|
signal.alarm(0)
|
|
if install_sigalarm_handler(old_handler) and old_timeout_time:
|
|
set_sigalarm_timeout(int(old_timeout_time - time.time()),
|
|
default_timeout=1)
|
|
|
|
|
|
|
|
def retry(ExceptionToCheck, timeout_min=1.0, delay_sec=3, raiselist=None,
|
|
exception_to_raise=None, label=None, callback=None, backoff=1):
|
|
"""Retry calling the decorated function using a delay with jitter.
|
|
|
|
Will raise RPC ValidationError exceptions from the decorated
|
|
function without retrying; a malformed RPC isn't going to
|
|
magically become good. Will raise exceptions in raiselist as well.
|
|
|
|
If the retry is done in a child thread, timeout may not be enforced as
|
|
signal only works in main thread. Therefore, the retry inside a child
|
|
thread may run longer than timeout or even hang.
|
|
|
|
original from:
|
|
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
|
|
|
|
@param ExceptionToCheck: the exception to check. May be a tuple of
|
|
exceptions to check.
|
|
@param timeout_min: timeout in minutes until giving up.
|
|
@param delay_sec: pre-jittered base delay between retries in seconds. Actual
|
|
delays will be first calculated with exponential backoff,
|
|
then randomized around this new value, ranging up to 50%
|
|
off this midpoint.
|
|
@param raiselist: a list of exceptions that will be raised without retrying.
|
|
@param exception_to_raise: the exception to raise. Callers can specify the
|
|
exception they want to raise.
|
|
@param label: a label added to the exception message to help debug.
|
|
@param callback: a function to call before each retry.
|
|
@param backoff: exponent to calculate exponential backoff for the actual
|
|
delay. Set to 1 to disable exponential backoff.
|
|
"""
|
|
def deco_retry(func):
|
|
"""
|
|
Decorator wrapper.
|
|
|
|
@param func: the function to be retried and timed-out.
|
|
"""
|
|
random.seed()
|
|
|
|
|
|
def delay(delay_with_backoff_sec):
|
|
"""
|
|
'Jitter' the delay with backoff, up to 50% in either direction.
|
|
"""
|
|
random_delay = random.uniform(0.5 * delay_with_backoff_sec,
|
|
1.5 * delay_with_backoff_sec)
|
|
logging.warning('Retrying in %f seconds...', random_delay)
|
|
time.sleep(random_delay)
|
|
|
|
|
|
def func_retry(*args, **kwargs):
|
|
"""
|
|
Used to cache exception to be raised later.
|
|
"""
|
|
exc_info = None
|
|
delayed_enabled = False
|
|
exception_tuple = () if raiselist is None else tuple(raiselist)
|
|
start_time = time.time()
|
|
remaining_time = timeout_min * 60
|
|
delay_with_backoff_sec = delay_sec
|
|
is_main_thread = isinstance(threading.current_thread(),
|
|
threading._MainThread)
|
|
if label:
|
|
details = 'label="%s"' % label
|
|
elif hasattr(func, '__name__'):
|
|
details = 'function="%s()"' % func.__name__
|
|
else:
|
|
details = 'unknown function'
|
|
|
|
exception_message = ('retry exception (%s), timeout = %ds' %
|
|
(details, timeout_min * 60))
|
|
|
|
while remaining_time > 0:
|
|
if delayed_enabled:
|
|
delay(delay_with_backoff_sec)
|
|
delay_with_backoff_sec *= backoff
|
|
else:
|
|
delayed_enabled = True
|
|
try:
|
|
# Clear the cache
|
|
exc_info = None
|
|
if is_main_thread:
|
|
is_timeout, result = timeout(func, args, kwargs,
|
|
remaining_time)
|
|
if not is_timeout:
|
|
return result
|
|
else:
|
|
return func(*args, **kwargs)
|
|
except exception_tuple:
|
|
raise
|
|
except error.CrosDynamicSuiteException:
|
|
raise
|
|
except ExceptionToCheck as e:
|
|
logging.warning('%s(%s)', e.__class__, e)
|
|
# Cache the exception to be raised later.
|
|
exc_info = sys.exc_info()
|
|
|
|
remaining_time = int(timeout_min * 60 -
|
|
(time.time() - start_time))
|
|
|
|
if remaining_time > 0 and callback:
|
|
callback()
|
|
remaining_time = int(timeout_min * 60 -
|
|
(time.time() - start_time))
|
|
|
|
|
|
# The call must have timed out or raised ExceptionToCheck.
|
|
if not exc_info:
|
|
if exception_to_raise:
|
|
raise exception_to_raise(exception_message)
|
|
else:
|
|
raise error.TimeoutException(exception_message)
|
|
# Raise the cached exception with original backtrace.
|
|
if exception_to_raise:
|
|
raise exception_to_raise('%s: %s' % (exc_info[0], exc_info[1]))
|
|
raise exc_info[0], exc_info[1], exc_info[2]
|
|
|
|
|
|
return func_retry # true decorator
|
|
return deco_retry
|