SCons/Executor.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """Execute actions with specific lists of target and source Nodes."""
  25
  26 import collections
  27 from typing import Dict
  28
  29 import SCons.Errors
  30 import SCons.Memoize
  31 import SCons.Util
  32 from SCons.compat import NoSlotsPyPy
  33 import SCons.Debug
  34 from SCons.Debug import logInstanceCreation
  35 from SCons.Util.sctyping import ExecutorType
  36
  37 class Batch:
  38     """Remembers exact association between targets
  39     and sources of executor."""
  40
  41     __slots__ = ('targets',
  42                  'sources')
  43
  44     def __init__(self, targets=[], sources=[]) -> None:
  45         self.targets = targets
  46         self.sources = sources
  47
  48
  49
  50 class TSList(collections.UserList):
  51     """A class that implements $TARGETS or $SOURCES expansions by wrapping
  52     an executor Method.  This class is used in the Executor.lvars()
  53     to delay creation of NodeList objects until they're needed.
  54
  55     Note that we subclass collections.UserList purely so that the
  56     is_Sequence() function will identify an object of this class as
  57     a list during variable expansion.  We're not really using any
  58     collections.UserList methods in practice.
  59     """
  60     def __init__(self, func) -> None:
  61         self.func = func
  62     def __getattr__(self, attr):
  63         nl = self.func()
  64         return getattr(nl, attr)
  65     def __getitem__(self, i):
  66         nl = self.func()
  67         return nl[i]
  68     def __str__(self) -> str:
  69         nl = self.func()
  70         return str(nl)
  71     def __repr__(self) -> str:
  72         nl = self.func()
  73         return repr(nl)
  74
  75 class TSObject:
  76     """A class that implements $TARGET or $SOURCE expansions by wrapping
  77     an Executor method.
  78     """
  79     def __init__(self, func) -> None:
  80         self.func = func
  81     def __getattr__(self, attr):
  82         n = self.func()
  83         return getattr(n, attr)
  84     def __str__(self) -> str:
  85         n = self.func()
  86         if n:
  87             return str(n)
  88         return ''
  89     def __repr__(self) -> str:
  90         n = self.func()
  91         if n:
  92             return repr(n)
  93         return ''
  94
  95 def rfile(node):
  96     """
  97     A function to return the results of a Node's rfile() method,
  98     if it exists, and the Node itself otherwise (if it's a Value
  99     Node, e.g.).
 100     """
 101     try:
 102         rfile = node.rfile
 103     except AttributeError:
 104         return node
 105     else:
 106         return rfile()
 107
 108
 109 def execute_nothing(obj, target, kw) -> int:
 110     return 0
 111
 112 def execute_action_list(obj, target, kw):
 113     """Actually execute the action list."""
 114     env = obj.get_build_env()
 115     kw = obj.get_kw(kw)
 116     status = 0
 117     for act in obj.get_action_list():
 118         args = ([], [], env)
 119         status = act(*args, **kw)
 120         if isinstance(status, SCons.Errors.BuildError):
 121             status.executor = obj
 122             raise status    # TODO pylint E0702: raising int not allowed
 123         elif status:
 124             msg = "Error %s" % status
 125             raise SCons.Errors.BuildError(
 126                 errstr=msg,
 127                 node=obj.batches[0].targets,
 128                 executor=obj,
 129                 action=act)
 130     return status
 131
 132 _do_execute_map = {0 : execute_nothing,
 133                    1 : execute_action_list}
 134
 135
 136 def execute_actions_str(obj):
 137     env = obj.get_build_env()
 138     return "\n".join([action.genstring(obj.get_all_targets(),
 139                                        obj.get_all_sources(),
 140                                        env)
 141                       for action in obj.get_action_list()])
 142
 143 def execute_null_str(obj) -> str:
 144     return ''
 145
 146 _execute_str_map = {0 : execute_null_str,
 147                     1 : execute_actions_str}
 148
 149
 150 class Executor(metaclass=NoSlotsPyPy):
 151     """A class for controlling instances of executing an action.
 152
 153     This largely exists to hold a single association of an action,
 154     environment, list of environment override dictionaries, targets
 155     and sources for later processing as needed.
 156     """
 157
 158     __slots__ = ('pre_actions',
 159                  'post_actions',
 160                  'env',
 161                  'overridelist',
 162                  'batches',
 163                  'builder_kw',
 164                  '_memo',
 165                  'lvars',
 166                  '_changed_sources_list',
 167                  '_changed_targets_list',
 168                  '_unchanged_sources_list',
 169                  '_unchanged_targets_list',
 170                  'action_list',
 171                  '_do_execute',
 172                  '_execute_str')
 173
 174     def __init__(self, action, env=None, overridelist=[{}],
 175                  targets=[], sources=[], builder_kw={}) -> None:
 176         if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor')
 177         self.set_action_list(action)
 178         self.pre_actions = []
 179         self.post_actions = []
 180         self.env = env
 181         self.overridelist = overridelist
 182         if targets or sources:
 183             self.batches = [Batch(targets[:], sources[:])]
 184         else:
 185             self.batches = []
 186         self.builder_kw = builder_kw
 187         self._do_execute = 1
 188         self._execute_str = 1
 189         self._memo = {}
 190
 191     def get_lvars(self):
 192         try:
 193             return self.lvars
 194         except AttributeError:
 195             self.lvars = {
 196                 'CHANGED_SOURCES' : TSList(self._get_changed_sources),
 197                 'CHANGED_TARGETS' : TSList(self._get_changed_targets),
 198                 'SOURCE' : TSObject(self._get_source),
 199                 'SOURCES' : TSList(self._get_sources),
 200                 'TARGET' : TSObject(self._get_target),
 201                 'TARGETS' : TSList(self._get_targets),
 202                 'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources),
 203                 'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets),
 204             }
 205             return self.lvars
 206
 207     def _get_changes(self) -> None:
 208         cs = []
 209         ct = []
 210         us = []
 211         ut = []
 212         for b in self.batches:
 213             # don't add targets marked always build to unchanged lists
 214             # add to changed list as they always need to build
 215             if not b.targets[0].always_build and b.targets[0].is_up_to_date():
 216                 us.extend(list(map(rfile, b.sources)))
 217                 ut.extend(b.targets)
 218             else:
 219                 cs.extend(list(map(rfile, b.sources)))
 220                 ct.extend(b.targets)
 221         self._changed_sources_list = SCons.Util.NodeList(cs)
 222         self._changed_targets_list = SCons.Util.NodeList(ct)
 223         self._unchanged_sources_list = SCons.Util.NodeList(us)
 224         self._unchanged_targets_list = SCons.Util.NodeList(ut)
 225
 226     def _get_changed_sources(self, *args, **kw):
 227         try:
 228             return self._changed_sources_list
 229         except AttributeError:
 230             self._get_changes()
 231             return self._changed_sources_list
 232
 233     def _get_changed_targets(self, *args, **kw):
 234         try:
 235             return self._changed_targets_list
 236         except AttributeError:
 237             self._get_changes()
 238             return self._changed_targets_list
 239
 240     def _get_source(self, *args, **kw):
 241         return rfile(self.batches[0].sources[0]).get_subst_proxy()
 242
 243     def _get_sources(self, *args, **kw):
 244         return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()])
 245
 246     def _get_target(self, *args, **kw):
 247         return self.batches[0].targets[0].get_subst_proxy()
 248
 249     def _get_targets(self, *args, **kw):
 250         return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()])
 251
 252     def _get_unchanged_sources(self, *args, **kw):
 253         try:
 254             return self._unchanged_sources_list
 255         except AttributeError:
 256             self._get_changes()
 257             return self._unchanged_sources_list
 258
 259     def _get_unchanged_targets(self, *args, **kw):
 260         try:
 261             return self._unchanged_targets_list
 262         except AttributeError:
 263             self._get_changes()
 264             return self._unchanged_targets_list
 265
 266     def get_action_targets(self):
 267         if not self.action_list:
 268             return []
 269         targets_string = self.action_list[0].get_targets(self.env, self)
 270         if targets_string[0] == '$':
 271             targets_string = targets_string[1:]
 272         return self.get_lvars()[targets_string]
 273
 274     def set_action_list(self, action):
 275         if not SCons.Util.is_List(action):
 276             if not action:
 277                 raise SCons.Errors.UserError("Executor must have an action.")
 278             action = [action]
 279         self.action_list = action
 280
 281     def get_action_list(self):
 282         if self.action_list is None:
 283             return []
 284         return self.pre_actions + self.action_list + self.post_actions
 285
 286     def get_all_targets(self):
 287         """Returns all targets for all batches of this Executor."""
 288         result = []
 289         for batch in self.batches:
 290             result.extend(batch.targets)
 291         return result
 292
 293     def get_all_sources(self):
 294         """Returns all sources for all batches of this Executor."""
 295         result = []
 296         for batch in self.batches:
 297             result.extend(batch.sources)
 298         return result
 299
 300     def get_all_children(self):
 301         """Returns all unique children (dependencies) for all batches
 302         of this Executor.
 303
 304         The Taskmaster can recognize when it's already evaluated a
 305         Node, so we don't have to make this list unique for its intended
 306         canonical use case, but we expect there to be a lot of redundancy
 307         (long lists of batched .cc files #including the same .h files
 308         over and over), so removing the duplicates once up front should
 309         save the Taskmaster a lot of work.
 310         """
 311         result = []
 312         for target in self.get_all_targets():
 313             result.extend(target.children())
 314         return SCons.Util.uniquer_hashables(result)
 315
 316     def get_all_prerequisites(self):
 317         """Returns all unique (order-only) prerequisites for all batches
 318         of this Executor.
 319         """
 320         result = []
 321         for target in self.get_all_targets():
 322             if target.prerequisites is not None:
 323                 result.extend(target.prerequisites)
 324         return SCons.Util.uniquer_hashables(result)
 325
 326     def get_action_side_effects(self):
 327
 328         """Returns all side effects for all batches of this
 329         Executor used by the underlying Action.
 330         """
 331         result = []
 332         for target in self.get_action_targets():
 333             result.extend(target.side_effects)
 334         return SCons.Util.uniquer_hashables(result)
 335
 336     @SCons.Memoize.CountMethodCall
 337     def get_build_env(self):
 338         """Fetch or create the appropriate build Environment
 339         for this Executor.
 340         """
 341         try:
 342             return self._memo['get_build_env']
 343         except KeyError:
 344             pass
 345
 346         # Create the build environment instance with appropriate
 347         # overrides.  These get evaluated against the current
 348         # environment's construction variables so that users can
 349         # add to existing values by referencing the variable in
 350         # the expansion.
 351         overrides = {}
 352         for odict in self.overridelist:
 353             overrides.update(odict)
 354
 355         import SCons.Defaults
 356         env = self.env or SCons.Defaults.DefaultEnvironment()
 357         build_env = env.Override(overrides)
 358
 359         self._memo['get_build_env'] = build_env
 360
 361         return build_env
 362
 363     def get_build_scanner_path(self, scanner):
 364         """Fetch the scanner path for this executor's targets and sources.
 365         """
 366         env = self.get_build_env()
 367         try:
 368             cwd = self.batches[0].targets[0].cwd
 369         except (IndexError, AttributeError):
 370             cwd = None
 371         return scanner.path(env, cwd,
 372                             self.get_all_targets(),
 373                             self.get_all_sources())
 374
 375     def get_kw(self, kw={}):
 376         result = self.builder_kw.copy()
 377         result.update(kw)
 378         result['executor'] = self
 379         return result
 380
 381     # use extra indirection because with new-style objects (Python 2.2
 382     # and above) we can't override special methods, and nullify() needs
 383     # to be able to do this.
 384
 385     def __call__(self, target, **kw):
 386         return _do_execute_map[self._do_execute](self, target, kw)
 387
 388     def cleanup(self) -> None:
 389         self._memo = {}
 390
 391     def add_sources(self, sources) -> None:
 392         """Add source files to this Executor's list.  This is necessary
 393         for "multi" Builders that can be called repeatedly to build up
 394         a source file list for a given target."""
 395         # TODO(batch):  extend to multiple batches
 396         assert (len(self.batches) == 1)
 397         # TODO(batch):  remove duplicates?
 398         sources = [x for x in sources if x not in self.batches[0].sources]
 399         self.batches[0].sources.extend(sources)
 400
 401     def get_sources(self):
 402         return self.batches[0].sources
 403
 404     def add_batch(self, targets, sources) -> None:
 405         """Add pair of associated target and source to this Executor's list.
 406         This is necessary for "batch" Builders that can be called repeatedly
 407         to build up a list of matching target and source files that will be
 408         used in order to update multiple target files at once from multiple
 409         corresponding source files, for tools like MSVC that support it."""
 410         self.batches.append(Batch(targets, sources))
 411
 412     def prepare(self):
 413         """
 414         Preparatory checks for whether this Executor can go ahead
 415         and (try to) build its targets.
 416         """
 417         for s in self.get_all_sources():
 418             if s.missing():
 419                 msg = "Source `%s' not found, needed by target `%s'."
 420                 raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0]))
 421
 422     def add_pre_action(self, action) -> None:
 423         self.pre_actions.append(action)
 424
 425     def add_post_action(self, action) -> None:
 426         self.post_actions.append(action)
 427
 428     # another extra indirection for new-style objects and nullify...
 429
 430     def __str__(self) -> str:
 431         return _execute_str_map[self._execute_str](self)
 432
 433     def nullify(self) -> None:
 434         self.cleanup()
 435         self._do_execute = 0
 436         self._execute_str = 0
 437
 438     @SCons.Memoize.CountMethodCall
 439     def get_contents(self):
 440         """Fetch the signature contents.  This is the main reason this
 441         class exists, so we can compute this once and cache it regardless
 442         of how many target or source Nodes there are.
 443
 444         Returns bytes
 445         """
 446         try:
 447             return self._memo['get_contents']
 448         except KeyError:
 449             pass
 450         env = self.get_build_env()
 451
 452         action_list = self.get_action_list()
 453         all_targets = self.get_all_targets()
 454         all_sources = self.get_all_sources()
 455
 456         result = bytearray("",'utf-8').join([action.get_contents(all_targets,
 457                                                                  all_sources,
 458                                                                  env)
 459                                              for action in action_list])
 460
 461         self._memo['get_contents'] = result
 462         return result
 463
 464     def get_timestamp(self) -> int:
 465         """Fetch a time stamp for this Executor.  We don't have one, of
 466         course (only files do), but this is the interface used by the
 467         timestamp module.
 468         """
 469         return 0
 470
 471     def scan_targets(self, scanner) -> None:
 472         # TODO(batch):  scan by batches
 473         self.scan(scanner, self.get_all_targets())
 474
 475     def scan_sources(self, scanner) -> None:
 476         # TODO(batch):  scan by batches
 477         if self.batches[0].sources:
 478             self.scan(scanner, self.get_all_sources())
 479
 480     def scan(self, scanner, node_list) -> None:
 481         """Scan a list of this Executor's files (targets or sources) for
 482         implicit dependencies and update all of the targets with them.
 483         This essentially short-circuits an N*M scan of the sources for
 484         each individual target, which is a hell of a lot more efficient.
 485         """
 486         env = self.get_build_env()
 487         path = self.get_build_scanner_path
 488         kw = self.get_kw()
 489
 490         # TODO(batch):  scan by batches)
 491         deps = []
 492
 493         for node in node_list:
 494             node.disambiguate()
 495             deps.extend(node.get_implicit_deps(env, scanner, path, kw))
 496
 497         deps.extend(self.get_implicit_deps())
 498
 499         for tgt in self.get_all_targets():
 500             tgt.add_to_implicit(deps)
 501
 502     def _get_unignored_sources_key(self, node, ignore=()):
 503         return (node,) + tuple(ignore)
 504
 505     @SCons.Memoize.CountDictCall(_get_unignored_sources_key)
 506     def get_unignored_sources(self, node, ignore=()):
 507         key = (node,) + tuple(ignore)
 508         try:
 509             memo_dict = self._memo['get_unignored_sources']
 510         except KeyError:
 511             memo_dict = {}
 512             self._memo['get_unignored_sources'] = memo_dict
 513         else:
 514             try:
 515                 return memo_dict[key]
 516             except KeyError:
 517                 pass
 518
 519         if node:
 520             # TODO:  better way to do this (it's a linear search,
 521             # but it may not be critical path)?
 522             sourcelist = []
 523             for b in self.batches:
 524                 if node in b.targets:
 525                     sourcelist = b.sources
 526                     break
 527         else:
 528             sourcelist = self.get_all_sources()
 529         if ignore:
 530             idict = {}
 531             for i in ignore:
 532                 idict[i] = 1
 533             sourcelist = [s for s in sourcelist if s not in idict]
 534
 535         memo_dict[key] = sourcelist
 536
 537         return sourcelist
 538
 539     def get_implicit_deps(self):
 540         """Return the executor's implicit dependencies, i.e. the nodes of
 541         the commands to be executed."""
 542         result = []
 543         build_env = self.get_build_env()
 544         for act in self.get_action_list():
 545             deps = act.get_implicit_deps(self.get_all_targets(),
 546                                          self.get_all_sources(),
 547                                          build_env)
 548             result.extend(deps)
 549         return result
 550
 551
 552
 553 _batch_executors: Dict[str, ExecutorType] = {}
 554
 555 def GetBatchExecutor(key: str) -> ExecutorType:
 556     return _batch_executors[key]
 557
 558 def AddBatchExecutor(key: str, executor: ExecutorType) -> None:
 559     assert key not in _batch_executors
 560     _batch_executors[key] = executor
 561
 562 nullenv = None
 563
 564
 565 class NullEnvironment(SCons.Util.Null):
 566     import SCons.CacheDir
 567     _CacheDir_path = None
 568     _CacheDir = SCons.CacheDir.CacheDir(None)
 569     def get_CacheDir(self):
 570         return self._CacheDir
 571
 572
 573 def get_NullEnvironment():
 574     """Use singleton pattern for Null Environments."""
 575     global nullenv
 576
 577     if nullenv is None:
 578         nullenv = NullEnvironment()
 579     return nullenv
 580
 581 class Null(metaclass=NoSlotsPyPy):
 582     """A null Executor, with a null build Environment, that does
 583     nothing when the rest of the methods call it.
 584
 585     This might be able to disappear when we refactor things to
 586     disassociate Builders from Nodes entirely, so we're not
 587     going to worry about unit tests for this--at least for now.
 588     """
 589
 590     __slots__ = ('pre_actions',
 591                  'post_actions',
 592                  'env',
 593                  'overridelist',
 594                  'batches',
 595                  'builder_kw',
 596                  '_memo',
 597                  'lvars',
 598                  '_changed_sources_list',
 599                  '_changed_targets_list',
 600                  '_unchanged_sources_list',
 601                  '_unchanged_targets_list',
 602                  'action_list',
 603                  '_do_execute',
 604                  '_execute_str')
 605
 606     def __init__(self, *args, **kw) -> None:
 607         if SCons.Debug.track_instances:
 608             logInstanceCreation(self, 'Executor.Null')
 609         self.batches = [Batch(kw['targets'][:], [])]
 610     def get_build_env(self):
 611         return get_NullEnvironment()
 612     def get_build_scanner_path(self):
 613         return None
 614     def cleanup(self) -> None:
 615         pass
 616     def prepare(self) -> None:
 617         pass
 618     def get_unignored_sources(self, *args, **kw):
 619         return tuple(())
 620     def get_action_targets(self):
 621         return []
 622     def get_action_list(self):
 623         return []
 624     def get_all_targets(self):
 625         return self.batches[0].targets
 626     def get_all_sources(self):
 627         return self.batches[0].targets[0].sources
 628     def get_all_children(self):
 629         return self.batches[0].targets[0].children()
 630     def get_all_prerequisites(self):
 631         return []
 632     def get_action_side_effects(self):
 633         return []
 634     def __call__(self, *args, **kw) -> int:
 635         return 0
 636     def get_contents(self) -> str:
 637         return ''
 638     def _morph(self) -> None:
 639         """Morph this Null executor to a real Executor object."""
 640         batches = self.batches
 641         self.__class__ = Executor
 642         self.__init__([])
 643         self.batches = batches
 644
 645     # The following methods require morphing this Null Executor to a
 646     # real Executor object.
 647
 648     def add_pre_action(self, action) -> None:
 649         self._morph()
 650         self.add_pre_action(action)
 651     def add_post_action(self, action) -> None:
 652         self._morph()
 653         self.add_post_action(action)
 654     def set_action_list(self, action) -> None:
 655         self._morph()
 656         self.set_action_list(action)
 657
 658 # Local Variables:
 659 # tab-width:4
 660 # indent-tabs-mode:nil
 661 # End:
 662 # vim: set expandtab tabstop=4 shiftwidth=4: