build/android/devil/utils/timeout_retry.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """A utility to run functions with timeouts and retries."""
   6 # pylint: disable=W0702
   7
   8 import logging
   9 import threading
  10 import time
  11 import traceback
  12
  13 from devil.utils import reraiser_thread
  14 from devil.utils import watchdog_timer
  15
  16
  17 class TimeoutRetryThread(reraiser_thread.ReraiserThread):
  18   def __init__(self, func, timeout, name):
  19     super(TimeoutRetryThread, self).__init__(func, name=name)
  20     self._watcher = watchdog_timer.WatchdogTimer(timeout)
  21     self._expired = False
  22
  23   def GetWatcher(self):
  24     """Returns the watchdog keeping track of this thread's time."""
  25     return self._watcher
  26
  27   def GetElapsedTime(self):
  28     return self._watcher.GetElapsed()
  29
  30   def GetRemainingTime(self, required=0, msg=None):
  31     """Get the remaining time before the thread times out.
  32
  33     Useful to send as the |timeout| parameter of async IO operations.
  34
  35     Args:
  36       required: minimum amount of time that will be required to complete, e.g.,
  37         some sleep or IO operation.
  38       msg: error message to show if timing out.
  39
  40     Returns:
  41       The number of seconds remaining before the thread times out, or None
  42       if the thread never times out.
  43
  44     Raises:
  45       reraiser_thread.TimeoutError if the remaining time is less than the
  46         required time.
  47     """
  48     remaining = self._watcher.GetRemaining()
  49     if remaining is not None and remaining < required:
  50       if msg is None:
  51         msg = 'Timeout expired'
  52       if remaining > 0:
  53         msg += (', wait of %.1f secs required but only %.1f secs left'
  54                 % (required, remaining))
  55       self._expired = True
  56       raise reraiser_thread.TimeoutError(msg)
  57     return remaining
  58
  59   def LogTimeoutException(self):
  60     """Log the exception that terminated this thread."""
  61     if not self._expired:
  62       return
  63     logging.critical('*' * 80)
  64     logging.critical('%s on thread %r', self._exc_info[0].__name__, self.name)
  65     logging.critical('*' * 80)
  66     fmt_exc = ''.join(traceback.format_exception(*self._exc_info))
  67     for line in fmt_exc.splitlines():
  68       logging.critical(line.rstrip())
  69     logging.critical('*' * 80)
  70
  71
  72 def CurrentTimeoutThread():
  73   """Get the current thread if it is a TimeoutRetryThread.
  74
  75   Returns:
  76     The current thread if it is a TimeoutRetryThread, otherwise None.
  77   """
  78   current_thread = threading.current_thread()
  79   if isinstance(current_thread, TimeoutRetryThread):
  80     return current_thread
  81   else:
  82     return None
  83
  84
  85 def WaitFor(condition, wait_period=5, max_tries=None):
  86   """Wait for a condition to become true.
  87
  88   Repeadly call the function condition(), with no arguments, until it returns
  89   a true value.
  90
  91   If called within a TimeoutRetryThread, it cooperates nicely with it.
  92
  93   Args:
  94     condition: function with the condition to check
  95     wait_period: number of seconds to wait before retrying to check the
  96       condition
  97     max_tries: maximum number of checks to make, the default tries forever
  98       or until the TimeoutRetryThread expires.
  99
 100   Returns:
 101     The true value returned by the condition, or None if the condition was
 102     not met after max_tries.
 103
 104   Raises:
 105     reraiser_thread.TimeoutError if the current thread is a TimeoutRetryThread
 106       and the timeout expires.
 107   """
 108   condition_name = condition.__name__
 109   timeout_thread = CurrentTimeoutThread()
 110   while max_tries is None or max_tries > 0:
 111     result = condition()
 112     if max_tries is not None:
 113       max_tries -= 1
 114     msg = ['condition', repr(condition_name), 'met' if result else 'not met']
 115     if timeout_thread:
 116       # pylint: disable=no-member
 117       msg.append('(%.1fs)' % timeout_thread.GetElapsedTime())
 118     logging.info(' '.join(msg))
 119     if result:
 120       return result
 121     if timeout_thread:
 122       # pylint: disable=no-member
 123       timeout_thread.GetRemainingTime(wait_period,
 124           msg='Timed out waiting for %r' % condition_name)
 125     time.sleep(wait_period)
 126   return None
 127
 128
 129 def Run(func, timeout, retries, args=None, kwargs=None):
 130   """Runs the passed function in a separate thread with timeouts and retries.
 131
 132   Args:
 133     func: the function to be wrapped.
 134     timeout: the timeout in seconds for each try.
 135     retries: the number of retries.
 136     args: list of positional args to pass to |func|.
 137     kwargs: dictionary of keyword args to pass to |func|.
 138
 139   Returns:
 140     The return value of func(*args, **kwargs).
 141   """
 142   if not args:
 143     args = []
 144   if not kwargs:
 145     kwargs = {}
 146
 147   # The return value uses a list because Python variables are references, not
 148   # values. Closures make a copy of the reference, so updating the closure's
 149   # reference wouldn't update where the original reference pointed.
 150   ret = [None]
 151   def RunOnTimeoutThread():
 152     ret[0] = func(*args, **kwargs)
 153
 154   num_try = 1
 155   while True:
 156     child_thread = TimeoutRetryThread(
 157       RunOnTimeoutThread, timeout,
 158       name='TimeoutThread-%d-for-%s' % (num_try,
 159                                         threading.current_thread().name))
 160     try:
 161       thread_group = reraiser_thread.ReraiserThreadGroup([child_thread])
 162       thread_group.StartAll()
 163       thread_group.JoinAll(child_thread.GetWatcher())
 164       return ret[0]
 165     except:
 166       child_thread.LogTimeoutException()
 167       if num_try > retries:
 168         raise
 169       num_try += 1