third_party/pycoverage/coverage/collector.py

   1 """Raw data collector for Coverage."""
   2
   3 import os, sys, threading
   4
   5 try:
   6     # Use the C extension code when we can, for speed.
   7     from coverage.tracer import CTracer         # pylint: disable=F0401,E0611
   8 except ImportError:
   9     # Couldn't import the C extension, maybe it isn't built.
  10     if os.getenv('COVERAGE_TEST_TRACER') == 'c':
  11         # During testing, we use the COVERAGE_TEST_TRACER env var to indicate
  12         # that we've fiddled with the environment to test this fallback code.
  13         # If we thought we had a C tracer, but couldn't import it, then exit
  14         # quickly and clearly instead of dribbling confusing errors. I'm using
  15         # sys.exit here instead of an exception because an exception here
  16         # causes all sorts of other noise in unittest.
  17         sys.stderr.write(
  18             "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n"
  19             )
  20         sys.exit(1)
  21     CTracer = None
  22
  23
  24 class PyTracer(object):
  25     """Python implementation of the raw data tracer."""
  26
  27     # Because of poor implementations of trace-function-manipulating tools,
  28     # the Python trace function must be kept very simple.  In particular, there
  29     # must be only one function ever set as the trace function, both through
  30     # sys.settrace, and as the return value from the trace function.  Put
  31     # another way, the trace function must always return itself.  It cannot
  32     # swap in other functions, or return None to avoid tracing a particular
  33     # frame.
  34     #
  35     # The trace manipulator that introduced this restriction is DecoratorTools,
  36     # which sets a trace function, and then later restores the pre-existing one
  37     # by calling sys.settrace with a function it found in the current frame.
  38     #
  39     # Systems that use DecoratorTools (or similar trace manipulations) must use
  40     # PyTracer to get accurate results.  The command-line --timid argument is
  41     # used to force the use of this tracer.
  42
  43     def __init__(self):
  44         self.data = None
  45         self.should_trace = None
  46         self.should_trace_cache = None
  47         self.warn = None
  48         self.cur_file_data = None
  49         self.last_line = 0
  50         self.data_stack = []
  51         self.last_exc_back = None
  52         self.last_exc_firstlineno = 0
  53         self.arcs = False
  54         self.thread = None
  55         self.stopped = False
  56
  57     def _trace(self, frame, event, arg_unused):
  58         """The trace function passed to sys.settrace."""
  59
  60         if self.stopped:
  61             return
  62
  63         if 0:
  64             sys.stderr.write("trace event: %s %r @%d\n" % (
  65                 event, frame.f_code.co_filename, frame.f_lineno
  66             ))
  67
  68         if self.last_exc_back:
  69             if frame == self.last_exc_back:
  70                 # Someone forgot a return event.
  71                 if self.arcs and self.cur_file_data:
  72                     pair = (self.last_line, -self.last_exc_firstlineno)
  73                     self.cur_file_data[pair] = None
  74                 self.cur_file_data, self.last_line = self.data_stack.pop()
  75             self.last_exc_back = None
  76
  77         if event == 'call':
  78             # Entering a new function context.  Decide if we should trace
  79             # in this file.
  80             self.data_stack.append((self.cur_file_data, self.last_line))
  81             filename = frame.f_code.co_filename
  82             if filename not in self.should_trace_cache:
  83                 tracename = self.should_trace(filename, frame)
  84                 self.should_trace_cache[filename] = tracename
  85             else:
  86                 tracename = self.should_trace_cache[filename]
  87             #print("called, stack is %d deep, tracename is %r" % (
  88             #               len(self.data_stack), tracename))
  89             if tracename:
  90                 if tracename not in self.data:
  91                     self.data[tracename] = {}
  92                 self.cur_file_data = self.data[tracename]
  93             else:
  94                 self.cur_file_data = None
  95             # Set the last_line to -1 because the next arc will be entering a
  96             # code block, indicated by (-1, n).
  97             self.last_line = -1
  98         elif event == 'line':
  99             # Record an executed line.
 100             if self.cur_file_data is not None:
 101                 if self.arcs:
 102                     #print("lin", self.last_line, frame.f_lineno)
 103                     self.cur_file_data[(self.last_line, frame.f_lineno)] = None
 104                 else:
 105                     #print("lin", frame.f_lineno)
 106                     self.cur_file_data[frame.f_lineno] = None
 107             self.last_line = frame.f_lineno
 108         elif event == 'return':
 109             if self.arcs and self.cur_file_data:
 110                 first = frame.f_code.co_firstlineno
 111                 self.cur_file_data[(self.last_line, -first)] = None
 112             # Leaving this function, pop the filename stack.
 113             self.cur_file_data, self.last_line = self.data_stack.pop()
 114             #print("returned, stack is %d deep" % (len(self.data_stack)))
 115         elif event == 'exception':
 116             #print("exc", self.last_line, frame.f_lineno)
 117             self.last_exc_back = frame.f_back
 118             self.last_exc_firstlineno = frame.f_code.co_firstlineno
 119         return self._trace
 120
 121     def start(self):
 122         """Start this Tracer.
 123
 124         Return a Python function suitable for use with sys.settrace().
 125
 126         """
 127         self.thread = threading.currentThread()
 128         sys.settrace(self._trace)
 129         return self._trace
 130
 131     def stop(self):
 132         """Stop this Tracer."""
 133         self.stopped = True
 134         if self.thread != threading.currentThread():
 135             # Called on a different thread than started us: we can't unhook
 136             # ourseves, but we've set the flag that we should stop, so we won't
 137             # do any more tracing.
 138             return
 139
 140         if hasattr(sys, "gettrace") and self.warn:
 141             if sys.gettrace() != self._trace:
 142                 msg = "Trace function changed, measurement is likely wrong: %r"
 143                 self.warn(msg % (sys.gettrace(),))
 144         #print("Stopping tracer on %s" % threading.current_thread().ident)
 145         sys.settrace(None)
 146
 147     def get_stats(self):
 148         """Return a dictionary of statistics, or None."""
 149         return None
 150
 151
 152 class Collector(object):
 153     """Collects trace data.
 154
 155     Creates a Tracer object for each thread, since they track stack
 156     information.  Each Tracer points to the same shared data, contributing
 157     traced data points.
 158
 159     When the Collector is started, it creates a Tracer for the current thread,
 160     and installs a function to create Tracers for each new thread started.
 161     When the Collector is stopped, all active Tracers are stopped.
 162
 163     Threads started while the Collector is stopped will never have Tracers
 164     associated with them.
 165
 166     """
 167
 168     # The stack of active Collectors.  Collectors are added here when started,
 169     # and popped when stopped.  Collectors on the stack are paused when not
 170     # the top, and resumed when they become the top again.
 171     _collectors = []
 172
 173     def __init__(self, should_trace, timid, branch, warn):
 174         """Create a collector.
 175
 176         `should_trace` is a function, taking a filename, and returning a
 177         canonicalized filename, or None depending on whether the file should
 178         be traced or not.
 179
 180         If `timid` is true, then a slower simpler trace function will be
 181         used.  This is important for some environments where manipulation of
 182         tracing functions make the faster more sophisticated trace function not
 183         operate properly.
 184
 185         If `branch` is true, then branches will be measured.  This involves
 186         collecting data on which statements followed each other (arcs).  Use
 187         `get_arc_data` to get the arc data.
 188
 189         `warn` is a warning function, taking a single string message argument,
 190         to be used if a warning needs to be issued.
 191
 192         """
 193         self.should_trace = should_trace
 194         self.warn = warn
 195         self.branch = branch
 196         self.reset()
 197
 198         if timid:
 199             # Being timid: use the simple Python trace function.
 200             self._trace_class = PyTracer
 201         else:
 202             # Being fast: use the C Tracer if it is available, else the Python
 203             # trace function.
 204             self._trace_class = CTracer or PyTracer
 205
 206     def __repr__(self):
 207         return "<Collector at 0x%x>" % id(self)
 208
 209     def tracer_name(self):
 210         """Return the class name of the tracer we're using."""
 211         return self._trace_class.__name__
 212
 213     def reset(self):
 214         """Clear collected data, and prepare to collect more."""
 215         # A dictionary mapping filenames to dicts with linenumber keys,
 216         # or mapping filenames to dicts with linenumber pairs as keys.
 217         self.data = {}
 218
 219         # A cache of the results from should_trace, the decision about whether
 220         # to trace execution in a file. A dict of filename to (filename or
 221         # None).
 222         self.should_trace_cache = {}
 223
 224         # Our active Tracers.
 225         self.tracers = []
 226
 227     def _start_tracer(self):
 228         """Start a new Tracer object, and store it in self.tracers."""
 229         tracer = self._trace_class()
 230         tracer.data = self.data
 231         tracer.arcs = self.branch
 232         tracer.should_trace = self.should_trace
 233         tracer.should_trace_cache = self.should_trace_cache
 234         tracer.warn = self.warn
 235         fn = tracer.start()
 236         self.tracers.append(tracer)
 237         return fn
 238
 239     # The trace function has to be set individually on each thread before
 240     # execution begins.  Ironically, the only support the threading module has
 241     # for running code before the thread main is the tracing function.  So we
 242     # install this as a trace function, and the first time it's called, it does
 243     # the real trace installation.
 244
 245     def _installation_trace(self, frame_unused, event_unused, arg_unused):
 246         """Called on new threads, installs the real tracer."""
 247         # Remove ourselves as the trace function
 248         sys.settrace(None)
 249         # Install the real tracer.
 250         fn = self._start_tracer()
 251         # Invoke the real trace function with the current event, to be sure
 252         # not to lose an event.
 253         if fn:
 254             fn = fn(frame_unused, event_unused, arg_unused)
 255         # Return the new trace function to continue tracing in this scope.
 256         return fn
 257
 258     def start(self):
 259         """Start collecting trace information."""
 260         if self._collectors:
 261             self._collectors[-1].pause()
 262         self._collectors.append(self)
 263         #print("Started: %r" % self._collectors, file=sys.stderr)
 264
 265         # Check to see whether we had a fullcoverage tracer installed.
 266         traces0 = []
 267         if hasattr(sys, "gettrace"):
 268             fn0 = sys.gettrace()
 269             if fn0:
 270                 tracer0 = getattr(fn0, '__self__', None)
 271                 if tracer0:
 272                     traces0 = getattr(tracer0, 'traces', [])
 273
 274         # Install the tracer on this thread.
 275         fn = self._start_tracer()
 276
 277         for args in traces0:
 278             (frame, event, arg), lineno = args
 279             try:
 280                 fn(frame, event, arg, lineno=lineno)
 281             except TypeError:
 282                 raise Exception(
 283                     "fullcoverage must be run with the C trace function."
 284                 )
 285
 286         # Install our installation tracer in threading, to jump start other
 287         # threads.
 288         threading.settrace(self._installation_trace)
 289
 290     def stop(self):
 291         """Stop collecting trace information."""
 292         #print >>sys.stderr, "Stopping: %r" % self._collectors
 293         assert self._collectors
 294         assert self._collectors[-1] is self
 295
 296         self.pause()
 297         self.tracers = []
 298
 299         # Remove this Collector from the stack, and resume the one underneath
 300         # (if any).
 301         self._collectors.pop()
 302         if self._collectors:
 303             self._collectors[-1].resume()
 304
 305     def pause(self):
 306         """Pause tracing, but be prepared to `resume`."""
 307         for tracer in self.tracers:
 308             tracer.stop()
 309             stats = tracer.get_stats()
 310             if stats:
 311                 print("\nCoverage.py tracer stats:")
 312                 for k in sorted(stats.keys()):
 313                     print("%16s: %s" % (k, stats[k]))
 314         threading.settrace(None)
 315
 316     def resume(self):
 317         """Resume tracing after a `pause`."""
 318         for tracer in self.tracers:
 319             tracer.start()
 320         threading.settrace(self._installation_trace)
 321
 322     def get_line_data(self):
 323         """Return the line data collected.
 324
 325         Data is { filename: { lineno: None, ...}, ...}
 326
 327         """
 328         if self.branch:
 329             # If we were measuring branches, then we have to re-build the dict
 330             # to show line data.
 331             line_data = {}
 332             for f, arcs in self.data.items():
 333                 line_data[f] = ldf = {}
 334                 for l1, _ in list(arcs.keys()):
 335                     if l1:
 336                         ldf[l1] = None
 337             return line_data
 338         else:
 339             return self.data
 340
 341     def get_arc_data(self):
 342         """Return the arc data collected.
 343
 344         Data is { filename: { (l1, l2): None, ...}, ...}
 345
 346         Note that no data is collected or returned if the Collector wasn't
 347         created with `branch` true.
 348
 349         """
 350         if self.branch:
 351             return self.data
 352         else:
 353             return {}