lldb/packages/Python/lldbsuite/test/test_runner/process_control.py

   1 """
   2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   3 See https://llvm.org/LICENSE.txt for license information.
   4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   5
   6 Provides classes used by the test results reporting infrastructure
   7 within the LLDB test suite.
   8
   9
  10 This module provides process-management support for the LLDB test
  11 running infrastructure.
  12 """
  13
  14 # System imports
  15 import os
  16 import re
  17 import signal
  18 import subprocess
  19 import sys
  20 import threading
  21
  22
  23 class CommunicatorThread(threading.Thread):
  24     """Provides a thread class that communicates with a subprocess."""
  25
  26     def __init__(self, process, event, output_file):
  27         super(CommunicatorThread, self).__init__()
  28         # Don't let this thread prevent shutdown.
  29         self.daemon = True
  30         self.process = process
  31         self.pid = process.pid
  32         self.event = event
  33         self.output_file = output_file
  34         self.output = None
  35
  36     def run(self):
  37         try:
  38             # Communicate with the child process.
  39             # This will not complete until the child process terminates.
  40             self.output = self.process.communicate()
  41         except Exception as exception:  # pylint: disable=broad-except
  42             if self.output_file:
  43                 self.output_file.write(
  44                     "exception while using communicate() for pid: {}\n".format(
  45                         exception
  46                     )
  47                 )
  48         finally:
  49             # Signal that the thread's run is complete.
  50             self.event.set()
  51
  52
  53 # Provides a regular expression for matching gtimeout-based durations.
  54 TIMEOUT_REGEX = re.compile(r"(^\d+)([smhd])?$")
  55
  56
  57 def timeout_to_seconds(timeout):
  58     """Converts timeout/gtimeout timeout values into seconds.
  59
  60     @param timeout a timeout in the form of xm representing x minutes.
  61
  62     @return None if timeout is None, or the number of seconds as a float
  63     if a valid timeout format was specified.
  64     """
  65     if timeout is None:
  66         return None
  67     else:
  68         match = TIMEOUT_REGEX.match(timeout)
  69         if match:
  70             value = float(match.group(1))
  71             units = match.group(2)
  72             if units is None:
  73                 # default is seconds.  No conversion necessary.
  74                 return value
  75             elif units == "s":
  76                 # Seconds.  No conversion necessary.
  77                 return value
  78             elif units == "m":
  79                 # Value is in minutes.
  80                 return 60.0 * value
  81             elif units == "h":
  82                 # Value is in hours.
  83                 return (60.0 * 60.0) * value
  84             elif units == "d":
  85                 # Value is in days.
  86                 return 24 * (60.0 * 60.0) * value
  87             else:
  88                 raise Exception("unexpected units value '{}'".format(units))
  89         else:
  90             raise Exception("could not parse TIMEOUT spec '{}'".format(timeout))
  91
  92
  93 class ProcessHelper(object):
  94     """Provides an interface for accessing process-related functionality.
  95
  96     This class provides a factory method that gives the caller a
  97     platform-specific implementation instance of the class.
  98
  99     Clients of the class should stick to the methods provided in this
 100     base class.
 101
 102     \see ProcessHelper.process_helper()
 103     """
 104
 105     def __init__(self):
 106         super(ProcessHelper, self).__init__()
 107
 108     @classmethod
 109     def process_helper(cls):
 110         """Returns a platform-specific ProcessHelper instance.
 111         @return a ProcessHelper instance that does the right thing for
 112         the current platform.
 113         """
 114
 115         # If you add a new platform, create an instance here and
 116         # return it.
 117         if os.name == "nt":
 118             return WindowsProcessHelper()
 119         else:
 120             # For all POSIX-like systems.
 121             return UnixProcessHelper()
 122
 123     def create_piped_process(self, command, new_process_group=True):
 124         # pylint: disable=no-self-use,unused-argument
 125         # As expected.  We want derived classes to implement this.
 126         """Creates a subprocess.Popen-based class with I/O piped to the parent.
 127
 128         @param command the command line list as would be passed to
 129         subprocess.Popen().  Use the list form rather than the string form.
 130
 131         @param new_process_group indicates if the caller wants the
 132         process to be created in its own process group.  Each OS handles
 133         this concept differently.  It provides a level of isolation and
 134         can simplify or enable terminating the process tree properly.
 135
 136         @return a subprocess.Popen-like object.
 137         """
 138         raise Exception("derived class must implement")
 139
 140     def supports_soft_terminate(self):
 141         # pylint: disable=no-self-use
 142         # As expected.  We want derived classes to implement this.
 143         """Indicates if the platform supports soft termination.
 144
 145         Soft termination is the concept of a terminate mechanism that
 146         allows the target process to shut down nicely, but with the
 147         catch that the process might choose to ignore it.
 148
 149         Platform supporter note: only mark soft terminate as supported
 150         if the target process has some way to evade the soft terminate
 151         request; otherwise, just support the hard terminate method.
 152
 153         @return True if the platform supports a soft terminate mechanism.
 154         """
 155         # By default, we do not support a soft terminate mechanism.
 156         return False
 157
 158     def soft_terminate(self, popen_process, log_file=None, want_core=True):
 159         # pylint: disable=no-self-use,unused-argument
 160         # As expected.  We want derived classes to implement this.
 161         """Attempts to terminate the process in a polite way.
 162
 163         This terminate method is intended to give the child process a
 164         chance to clean up and exit on its own, possibly with a request
 165         to drop a core file or equivalent (i.e. [mini-]crashdump, crashlog,
 166         etc.)  If new_process_group was set in the process creation method
 167         and the platform supports it, this terminate call will attempt to
 168         kill the whole process tree rooted in this child process.
 169
 170         @param popen_process the subprocess.Popen-like object returned
 171         by one of the process-creation methods of this class.
 172
 173         @param log_file file-like object used to emit error-related
 174         logging info.  May be None if no error-related info is desired.
 175
 176         @param want_core True if the caller would like to get a core
 177         dump (or the analogous crash report) from the terminated process.
 178         """
 179         popen_process.terminate()
 180
 181     def hard_terminate(self, popen_process, log_file=None):
 182         # pylint: disable=no-self-use,unused-argument
 183         # As expected.  We want derived classes to implement this.
 184         """Attempts to terminate the process immediately.
 185
 186         This terminate method is intended to kill child process in
 187         a manner in which the child process has no ability to block,
 188         and also has no ability to clean up properly.  If new_process_group
 189         was specified when creating the process, and if the platform
 190         implementation supports it, this will attempt to kill the
 191         whole process tree rooted in the child process.
 192
 193         @param popen_process the subprocess.Popen-like object returned
 194         by one of the process-creation methods of this class.
 195
 196         @param log_file file-like object used to emit error-related
 197         logging info.  May be None if no error-related info is desired.
 198         """
 199         popen_process.kill()
 200
 201     def was_soft_terminate(self, returncode, with_core):
 202         # pylint: disable=no-self-use,unused-argument
 203         # As expected.  We want derived classes to implement this.
 204         """Returns if Popen-like object returncode matches soft terminate.
 205
 206         @param returncode the returncode from the Popen-like object that
 207         terminated with a given return code.
 208
 209         @param with_core indicates whether the returncode should match
 210         a core-generating return signal.
 211
 212         @return True when the returncode represents what the system would
 213         issue when a soft_terminate() with the given with_core arg occurred;
 214         False otherwise.
 215         """
 216         if not self.supports_soft_terminate():
 217             # If we don't support soft termination on this platform,
 218             # then this should always be False.
 219             return False
 220         else:
 221             # Once a platform claims to support soft terminate, it
 222             # needs to be able to identify it by overriding this method.
 223             raise Exception("platform needs to implement")
 224
 225     def was_hard_terminate(self, returncode):
 226         # pylint: disable=no-self-use,unused-argument
 227         # As expected.  We want derived classes to implement this.
 228         """Returns if Popen-like object returncode matches that of a hard
 229         terminate attempt.
 230
 231         @param returncode the returncode from the Popen-like object that
 232         terminated with a given return code.
 233
 234         @return True when the returncode represents what the system would
 235         issue when a hard_terminate() occurred; False
 236         otherwise.
 237         """
 238         raise Exception("platform needs to implement")
 239
 240     def soft_terminate_signals(self):
 241         # pylint: disable=no-self-use
 242         """Retrieve signal numbers that can be sent to soft terminate.
 243         @return a list of signal numbers that can be sent to soft terminate
 244         a process, or None if not applicable.
 245         """
 246         return None
 247
 248     def is_exceptional_exit(self, popen_status):
 249         """Returns whether the program exit status is exceptional.
 250
 251         Returns whether the return code from a Popen process is exceptional
 252         (e.g. signals on POSIX systems).
 253
 254         Derived classes should override this if they can detect exceptional
 255         program exit.
 256
 257         @return True if the given popen_status represents an exceptional
 258         program exit; False otherwise.
 259         """
 260         return False
 261
 262     def exceptional_exit_details(self, popen_status):
 263         """Returns the normalized exceptional exit code and a description.
 264
 265         Given an exceptional exit code, returns the integral value of the
 266         exception (e.g. signal number for POSIX) and a description (e.g.
 267         signal name on POSIX) for the result.
 268
 269         Derived classes should override this if they can detect exceptional
 270         program exit.
 271
 272         It is fine to not implement this so long as is_exceptional_exit()
 273         always returns False.
 274
 275         @return (normalized exception code, symbolic exception description)
 276         """
 277         raise Exception("exception_exit_details() called on unsupported class")
 278
 279
 280 class UnixProcessHelper(ProcessHelper):
 281     """Provides a ProcessHelper for Unix-like operating systems.
 282
 283     This implementation supports anything that looks Posix-y
 284     (e.g. Darwin, Linux, *BSD, etc.)
 285     """
 286
 287     def __init__(self):
 288         super(UnixProcessHelper, self).__init__()
 289
 290     @classmethod
 291     def _create_new_process_group(cls):
 292         """Creates a new process group for the calling process."""
 293         os.setpgid(os.getpid(), os.getpid())
 294
 295     def create_piped_process(self, command, new_process_group=True):
 296         # Determine what to run after the fork but before the exec.
 297         if new_process_group:
 298             preexec_func = self._create_new_process_group
 299         else:
 300             preexec_func = None
 301
 302         # Create the process.
 303         process = subprocess.Popen(
 304             command,
 305             stdin=subprocess.PIPE,
 306             stdout=subprocess.PIPE,
 307             stderr=subprocess.PIPE,
 308             universal_newlines=True,  # Elicits automatic byte -> string decoding in Py3
 309             close_fds=True,
 310             preexec_fn=preexec_func,
 311         )
 312
 313         # Remember whether we're using process groups for this
 314         # process.
 315         process.using_process_groups = new_process_group
 316         return process
 317
 318     def supports_soft_terminate(self):
 319         # POSIX does support a soft terminate via:
 320         # * SIGTERM (no core requested)
 321         # * SIGQUIT (core requested if enabled, see ulimit -c)
 322         return True
 323
 324     @classmethod
 325     def _validate_pre_terminate(cls, popen_process, log_file):
 326         # Validate args.
 327         if popen_process is None:
 328             raise ValueError("popen_process is None")
 329
 330         # Ensure we have something that looks like a valid process.
 331         if popen_process.pid < 1:
 332             if log_file:
 333                 log_file.write("skipping soft_terminate(): no process id")
 334             return False
 335
 336         # We only do the process liveness check if we're not using
 337         # process groups.  With process groups, checking if the main
 338         # inferior process is dead and short circuiting here is no
 339         # good - children of it in the process group could still be
 340         # alive, and they should be killed during a timeout.
 341         if not popen_process.using_process_groups:
 342             # Don't kill if it's already dead.
 343             popen_process.poll()
 344             if popen_process.returncode is not None:
 345                 # It has a returncode.  It has already stopped.
 346                 if log_file:
 347                     log_file.write(
 348                         "requested to terminate pid {} but it has already "
 349                         "terminated, returncode {}".format(
 350                             popen_process.pid, popen_process.returncode
 351                         )
 352                     )
 353                 # Move along...
 354                 return False
 355
 356         # Good to go.
 357         return True
 358
 359     def _kill_with_signal(self, popen_process, log_file, signum):
 360         # Validate we're ready to terminate this.
 361         if not self._validate_pre_terminate(popen_process, log_file):
 362             return
 363
 364         # Choose kill mechanism based on whether we're targeting
 365         # a process group or just a process.
 366         try:
 367             if popen_process.using_process_groups:
 368                 # if log_file:
 369                 #    log_file.write(
 370                 #        "sending signum {} to process group {} now\n".format(
 371                 #            signum, popen_process.pid))
 372                 os.killpg(popen_process.pid, signum)
 373             else:
 374                 # if log_file:
 375                 #    log_file.write(
 376                 #        "sending signum {} to process {} now\n".format(
 377                 #            signum, popen_process.pid))
 378                 os.kill(popen_process.pid, signum)
 379         except OSError as error:
 380             import errno
 381
 382             if error.errno == errno.ESRCH:
 383                 # This is okay - failed to find the process.  It may be that
 384                 # that the timeout pre-kill hook eliminated the process.  We'll
 385                 # ignore.
 386                 pass
 387             else:
 388                 raise
 389
 390     def soft_terminate(self, popen_process, log_file=None, want_core=True):
 391         # Choose signal based on desire for core file.
 392         if want_core:
 393             # SIGQUIT will generate core by default.  Can be caught.
 394             signum = signal.SIGQUIT
 395         else:
 396             # SIGTERM is the traditional nice way to kill a process.
 397             # Can be caught, doesn't generate a core.
 398             signum = signal.SIGTERM
 399
 400         self._kill_with_signal(popen_process, log_file, signum)
 401
 402     def hard_terminate(self, popen_process, log_file=None):
 403         self._kill_with_signal(popen_process, log_file, signal.SIGKILL)
 404
 405     def was_soft_terminate(self, returncode, with_core):
 406         if with_core:
 407             return returncode == -signal.SIGQUIT
 408         else:
 409             return returncode == -signal.SIGTERM
 410
 411     def was_hard_terminate(self, returncode):
 412         return returncode == -signal.SIGKILL
 413
 414     def soft_terminate_signals(self):
 415         return [signal.SIGQUIT, signal.SIGTERM]
 416
 417     def is_exceptional_exit(self, popen_status):
 418         return popen_status < 0
 419
 420     @classmethod
 421     def _signal_names_by_number(cls):
 422         return dict(
 423             (k, v)
 424             for v, k in reversed(sorted(signal.__dict__.items()))
 425             if v.startswith("SIG") and not v.startswith("SIG_")
 426         )
 427
 428     def exceptional_exit_details(self, popen_status):
 429         signo = -popen_status
 430         signal_names_by_number = self._signal_names_by_number()
 431         signal_name = signal_names_by_number.get(signo, "")
 432         return (signo, signal_name)
 433
 434
 435 class WindowsProcessHelper(ProcessHelper):
 436     """Provides a Windows implementation of the ProcessHelper class."""
 437
 438     def __init__(self):
 439         super(WindowsProcessHelper, self).__init__()
 440
 441     def create_piped_process(self, command, new_process_group=True):
 442         if new_process_group:
 443             # We need this flag if we want os.kill() to work on the subprocess.
 444             creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP
 445         else:
 446             creation_flags = 0
 447
 448         return subprocess.Popen(
 449             command,
 450             stdin=subprocess.PIPE,
 451             stdout=subprocess.PIPE,
 452             stderr=subprocess.PIPE,
 453             universal_newlines=True,  # Elicits automatic byte -> string decoding in Py3
 454             creationflags=creation_flags,
 455         )
 456
 457     def was_hard_terminate(self, returncode):
 458         return returncode != 0
 459
 460
 461 class ProcessDriver(object):
 462     """Drives a child process, notifies on important events, and can timeout.
 463
 464     Clients are expected to derive from this class and override the
 465     on_process_started and on_process_exited methods if they want to
 466     hook either of those.
 467
 468     This class supports timing out the child process in a platform-agnostic
 469     way.  The on_process_exited method is informed if the exit was natural
 470     or if it was due to a timeout.
 471     """
 472
 473     def __init__(self, soft_terminate_timeout=10.0):
 474         super(ProcessDriver, self).__init__()
 475         self.process_helper = ProcessHelper.process_helper()
 476         self.pid = None
 477         # Create the synchronization event for notifying when the
 478         # inferior dotest process is complete.
 479         self.done_event = threading.Event()
 480         self.io_thread = None
 481         self.process = None
 482         # Number of seconds to wait for the soft terminate to
 483         # wrap up, before moving to more drastic measures.
 484         # Might want this longer if core dumps are generated and
 485         # take a long time to write out.
 486         self.soft_terminate_timeout = soft_terminate_timeout
 487         # Number of seconds to wait for the hard terminate to
 488         # wrap up, before giving up on the io thread.  This should
 489         # be fast.
 490         self.hard_terminate_timeout = 5.0
 491         self.returncode = None
 492
 493     # =============================================
 494     # Methods for subclasses to override if desired.
 495     # =============================================
 496
 497     def on_process_started(self):
 498         pass
 499
 500     def on_process_exited(self, command, output, was_timeout, exit_status):
 501         pass
 502
 503     def on_timeout_pre_kill(self):
 504         """Called after the timeout interval elapses but before killing it.
 505
 506         This method is added to enable derived classes the ability to do
 507         something to the process prior to it being killed.  For example,
 508         this would be a good spot to run a program that samples the process
 509         to see what it was doing (or not doing).
 510
 511         Do not attempt to reap the process (i.e. use wait()) in this method.
 512         That will interfere with the kill mechanism and return code processing.
 513         """
 514
 515     def write(self, content):
 516         # pylint: disable=no-self-use
 517         # Intended - we want derived classes to be able to override
 518         # this and use any self state they may contain.
 519         sys.stdout.write(content)
 520
 521     # ==============================================================
 522     # Operations used to drive processes.  Clients will want to call
 523     # one of these.
 524     # ==============================================================
 525
 526     def run_command(self, command):
 527         # Start up the child process and the thread that does the
 528         # communication pump.
 529         self._start_process_and_io_thread(command)
 530
 531         # Wait indefinitely for the child process to finish
 532         # communicating.  This indicates it has closed stdout/stderr
 533         # pipes and is done.
 534         self.io_thread.join()
 535         self.returncode = self.process.wait()
 536         if self.returncode is None:
 537             raise Exception(
 538                 "no exit status available for pid {} after the "
 539                 " inferior dotest.py should have completed".format(self.process.pid)
 540             )
 541
 542         # Notify of non-timeout exit.
 543         self.on_process_exited(command, self.io_thread.output, False, self.returncode)
 544
 545     def run_command_with_timeout(self, command, timeout, want_core):
 546         # Figure out how many seconds our timeout description is requesting.
 547         timeout_seconds = timeout_to_seconds(timeout)
 548
 549         # Start up the child process and the thread that does the
 550         # communication pump.
 551         self._start_process_and_io_thread(command)
 552
 553         self._wait_with_timeout(timeout_seconds, command, want_core)
 554
 555     # ================
 556     # Internal details.
 557     # ================
 558
 559     def _start_process_and_io_thread(self, command):
 560         # Create the process.
 561         self.process = self.process_helper.create_piped_process(command)
 562         self.pid = self.process.pid
 563         self.on_process_started()
 564
 565         # Ensure the event is cleared that is used for signaling
 566         # from the communication() thread when communication is
 567         # complete (i.e. the inferior process has finished).
 568         self.done_event.clear()
 569
 570         self.io_thread = CommunicatorThread(self.process, self.done_event, self.write)
 571         self.io_thread.start()
 572
 573     def _attempt_soft_kill(self, want_core):
 574         # The inferior dotest timed out.  Attempt to clean it
 575         # with a non-drastic method (so it can clean up properly
 576         # and/or generate a core dump).  Often the OS can't guarantee
 577         # that the process will really terminate after this.
 578         self.process_helper.soft_terminate(
 579             self.process, want_core=want_core, log_file=self
 580         )
 581
 582         # Now wait up to a certain timeout period for the io thread
 583         # to say that the communication ended.  If that wraps up
 584         # within our soft terminate timeout, we're all done here.
 585         self.io_thread.join(self.soft_terminate_timeout)
 586         if not self.io_thread.is_alive():
 587             # stdout/stderr were closed on the child process side. We
 588             # should be able to wait and reap the child process here.
 589             self.returncode = self.process.wait()
 590             # We terminated, and the done_trying result is n/a
 591             terminated = True
 592             done_trying = None
 593         else:
 594             self.write(
 595                 "soft kill attempt of process {} timed out "
 596                 "after {} seconds\n".format(
 597                     self.process.pid, self.soft_terminate_timeout
 598                 )
 599             )
 600             terminated = False
 601             done_trying = False
 602         return terminated, done_trying
 603
 604     def _attempt_hard_kill(self):
 605         # Instruct the process to terminate and really force it to
 606         # happen.  Don't give the process a chance to ignore.
 607         self.process_helper.hard_terminate(self.process, log_file=self)
 608
 609         # Reap the child process.  This should not hang as the
 610         # hard_kill() mechanism is supposed to really kill it.
 611         # Improvement option:
 612         # If this does ever hang, convert to a self.process.poll()
 613         # loop checking on self.process.returncode until it is not
 614         # None or the timeout occurs.
 615         self.returncode = self.process.wait()
 616
 617         # Wait a few moments for the io thread to finish...
 618         self.io_thread.join(self.hard_terminate_timeout)
 619         if self.io_thread.is_alive():
 620             # ... but this is not critical if it doesn't end for some
 621             # reason.
 622             self.write(
 623                 "hard kill of process {} timed out after {} seconds waiting "
 624                 "for the io thread (ignoring)\n".format(
 625                     self.process.pid, self.hard_terminate_timeout
 626                 )
 627             )
 628
 629         # Set if it terminated.  (Set up for optional improvement above).
 630         terminated = self.returncode is not None
 631         # Nothing else to try.
 632         done_trying = True
 633
 634         return terminated, done_trying
 635
 636     def _attempt_termination(self, attempt_count, want_core):
 637         if self.process_helper.supports_soft_terminate():
 638             # When soft termination is supported, we first try to stop
 639             # the process with a soft terminate.  Failing that, we try
 640             # the hard terminate option.
 641             if attempt_count == 1:
 642                 return self._attempt_soft_kill(want_core)
 643             elif attempt_count == 2:
 644                 return self._attempt_hard_kill()
 645             else:
 646                 # We don't have anything else to try.
 647                 terminated = self.returncode is not None
 648                 done_trying = True
 649                 return terminated, done_trying
 650         else:
 651             # We only try the hard terminate option when there
 652             # is no soft terminate available.
 653             if attempt_count == 1:
 654                 return self._attempt_hard_kill()
 655             else:
 656                 # We don't have anything else to try.
 657                 terminated = self.returncode is not None
 658                 done_trying = True
 659                 return terminated, done_trying
 660
 661     def _wait_with_timeout(self, timeout_seconds, command, want_core):
 662         # Allow up to timeout seconds for the io thread to wrap up.
 663         # If that completes, the child process should be done.
 664         completed_normally = self.done_event.wait(timeout_seconds)
 665         if completed_normally:
 666             # Reap the child process here.
 667             self.returncode = self.process.wait()
 668         else:
 669             # Allow derived classes to do some work after we detected
 670             # a timeout but before we touch the timed-out process.
 671             self.on_timeout_pre_kill()
 672
 673             # Prepare to stop the process
 674             process_terminated = completed_normally
 675             terminate_attempt_count = 0
 676
 677             # Try as many attempts as we support for trying to shut down
 678             # the child process if it's not already shut down.
 679             while not process_terminated:
 680                 terminate_attempt_count += 1
 681                 # Attempt to terminate.
 682                 process_terminated, done_trying = self._attempt_termination(
 683                     terminate_attempt_count, want_core
 684                 )
 685                 # Check if there's nothing more to try.
 686                 if done_trying:
 687                     # Break out of our termination attempt loop.
 688                     break
 689
 690         # At this point, we're calling it good.  The process
 691         # finished gracefully, was shut down after one or more
 692         # attempts, or we failed but gave it our best effort.
 693         self.on_process_exited(
 694             command, self.io_thread.output, not completed_normally, self.returncode
 695         )
 696
 697
 698 def patched_init(self, *args, **kwargs):
 699     self.original_init(*args, **kwargs)
 700     # Initialize our condition variable that protects wait()/poll().
 701     self.wait_condition = threading.Condition()
 702
 703
 704 def patched_wait(self, *args, **kwargs):
 705     self.wait_condition.acquire()
 706     try:
 707         result = self.original_wait(*args, **kwargs)
 708         # The process finished.  Signal the condition.
 709         self.wait_condition.notify_all()
 710         return result
 711     finally:
 712         self.wait_condition.release()
 713
 714
 715 def patched_poll(self, *args, **kwargs):
 716     self.wait_condition.acquire()
 717     try:
 718         result = self.original_poll(*args, **kwargs)
 719         if self.returncode is not None:
 720             # We did complete, and we have the return value.
 721             # Signal the event to indicate we're done.
 722             self.wait_condition.notify_all()
 723         return result
 724     finally:
 725         self.wait_condition.release()
 726
 727
 728 def patch_up_subprocess_popen():
 729     subprocess.Popen.original_init = subprocess.Popen.__init__
 730     subprocess.Popen.__init__ = patched_init
 731
 732     subprocess.Popen.original_wait = subprocess.Popen.wait
 733     subprocess.Popen.wait = patched_wait
 734
 735     subprocess.Popen.original_poll = subprocess.Popen.poll
 736     subprocess.Popen.poll = patched_poll
 737
 738
 739 # Replace key subprocess.Popen() threading-unprotected methods with
 740 # threading-protected versions.
 741 patch_up_subprocess_popen()