SCons/Subst.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """SCons string substitution."""
  25
  26 import collections
  27 import re
  28 from inspect import signature, Parameter
  29 from typing import Optional
  30
  31 import SCons.Errors
  32 from SCons.Util import is_String, is_Sequence
  33
  34 # Indexed by the SUBST_* constants below.
  35 _strconv = [
  36     SCons.Util.to_String_for_subst,
  37     SCons.Util.to_String_for_subst,
  38     SCons.Util.to_String_for_signature,
  39 ]
  40
  41 AllowableExceptions = (IndexError, NameError)
  42
  43
  44 def SetAllowableExceptions(*excepts) -> None:
  45     global AllowableExceptions
  46     AllowableExceptions = [_f for _f in excepts if _f]
  47
  48
  49 def raise_exception(exception, target, s):
  50     name = exception.__class__.__name__
  51     msg = "%s `%s' trying to evaluate `%s'" % (name, exception, s)
  52     if target:
  53         raise SCons.Errors.BuildError(target[0], msg)
  54     else:
  55         raise SCons.Errors.UserError(msg)
  56
  57
  58 class Literal:
  59     """A wrapper for a string.  If you use this object wrapped
  60     around a string, then it will be interpreted as literal.
  61     When passed to the command interpreter, all special
  62     characters will be escaped."""
  63     def __init__(self, lstr) -> None:
  64         self.lstr = lstr
  65
  66     def __str__(self) -> str:
  67         return self.lstr
  68
  69     def escape(self, escape_func):
  70         return escape_func(self.lstr)
  71
  72     def for_signature(self):
  73         return self.lstr
  74
  75     def is_literal(self) -> bool:
  76         return True
  77
  78     def __eq__(self, other):
  79         if not isinstance(other, Literal):
  80             return False
  81         return self.lstr == other.lstr
  82
  83     def __neq__(self, other) -> bool:
  84         return not self.__eq__(other)
  85
  86     def __hash__(self):
  87         return hash(self.lstr)
  88
  89 class SpecialAttrWrapper:
  90     """This is a wrapper for what we call a 'Node special attribute.'
  91     This is any of the attributes of a Node that we can reference from
  92     Environment variable substitution, such as $TARGET.abspath or
  93     $SOURCES[1].filebase.  We implement the same methods as Literal
  94     so we can handle special characters, plus a for_signature method,
  95     such that we can return some canonical string during signature
  96     calculation to avoid unnecessary rebuilds."""
  97
  98     def __init__(self, lstr, for_signature=None) -> None:
  99         """The for_signature parameter, if supplied, will be the
 100         canonical string we return from for_signature().  Else
 101         we will simply return lstr."""
 102         self.lstr = lstr
 103         if for_signature:
 104             self.forsig = for_signature
 105         else:
 106             self.forsig = lstr
 107
 108     def __str__(self) -> str:
 109         return self.lstr
 110
 111     def escape(self, escape_func):
 112         return escape_func(self.lstr)
 113
 114     def for_signature(self):
 115         return self.forsig
 116
 117     def is_literal(self) -> bool:
 118         return True
 119
 120 def quote_spaces(arg):
 121     """Generic function for putting double quotes around any string that
 122     has white space in it."""
 123     if ' ' in arg or '\t' in arg:
 124         return '"%s"' % arg
 125     else:
 126         return str(arg)
 127
 128 class CmdStringHolder(collections.UserString):
 129     """This is a special class used to hold strings generated by
 130     scons_subst() and scons_subst_list().  It defines a special method
 131     escape().  When passed a function with an escape algorithm for a
 132     particular platform, it will return the contained string with the
 133     proper escape sequences inserted.
 134     """
 135     def __init__(self, cmd, literal=None) -> None:
 136         super().__init__(cmd)
 137         self.literal = literal
 138
 139     def is_literal(self) -> bool:
 140         return self.literal
 141
 142     def escape(self, escape_func, quote_func=quote_spaces):
 143         """Escape the string with the supplied function.  The
 144         function is expected to take an arbitrary string, then
 145         return it with all special characters escaped and ready
 146         for passing to the command interpreter.
 147
 148         After calling this function, the next call to str() will
 149         return the escaped string.
 150         """
 151
 152         if self.is_literal():
 153             return escape_func(self.data)
 154         elif ' ' in self.data or '\t' in self.data:
 155             return quote_func(self.data)
 156         else:
 157             return self.data
 158
 159 def escape_list(mylist, escape_func):
 160     """Escape a list of arguments by running the specified escape_func
 161     on every object in the list that has an escape() method."""
 162     def escape(obj, escape_func=escape_func):
 163         try:
 164             e = obj.escape
 165         except AttributeError:
 166             return obj
 167         else:
 168             return e(escape_func)
 169     return list(map(escape, mylist))
 170
 171 class NLWrapper:
 172     """A wrapper class that delays turning a list of sources or targets
 173     into a NodeList until it's needed.  The specified function supplied
 174     when the object is initialized is responsible for turning raw nodes
 175     into proxies that implement the special attributes like .abspath,
 176     .source, etc.  This way, we avoid creating those proxies just
 177     "in case" someone is going to use $TARGET or the like, and only
 178     go through the trouble if we really have to.
 179
 180     In practice, this might be a wash performance-wise, but it's a little
 181     cleaner conceptually...
 182     """
 183
 184     def __init__(self, list, func) -> None:
 185         self.list = list
 186         self.func = func
 187     def _return_nodelist(self):
 188         return self.nodelist
 189     def _gen_nodelist(self):
 190         mylist = self.list
 191         if mylist is None:
 192             mylist = []
 193         elif not is_Sequence(mylist):
 194             mylist = [mylist]
 195         # The map(self.func) call is what actually turns
 196         # a list into appropriate proxies.
 197         self.nodelist = SCons.Util.NodeList(list(map(self.func, mylist)))
 198         self._create_nodelist = self._return_nodelist
 199         return self.nodelist
 200     _create_nodelist = _gen_nodelist
 201
 202
 203 class Targets_or_Sources(collections.UserList):
 204     """A class that implements $TARGETS or $SOURCES expansions by in turn
 205     wrapping a NLWrapper.  This class handles the different methods used
 206     to access the list, calling the NLWrapper to create proxies on demand.
 207
 208     Note that we subclass collections.UserList purely so that the
 209     is_Sequence() function will identify an object of this class as
 210     a list during variable expansion.  We're not really using any
 211     collections.UserList methods in practice.
 212     """
 213     def __init__(self, nl) -> None:
 214         self.nl = nl
 215     def __getattr__(self, attr):
 216         nl = self.nl._create_nodelist()
 217         return getattr(nl, attr)
 218     def __getitem__(self, i):
 219         nl = self.nl._create_nodelist()
 220         return nl[i]
 221     def __str__(self) -> str:
 222         nl = self.nl._create_nodelist()
 223         return str(nl)
 224     def __repr__(self) -> str:
 225         nl = self.nl._create_nodelist()
 226         return repr(nl)
 227
 228 class Target_or_Source:
 229     """A class that implements $TARGET or $SOURCE expansions by in turn
 230     wrapping a NLWrapper.  This class handles the different methods used
 231     to access an individual proxy Node, calling the NLWrapper to create
 232     a proxy on demand.
 233     """
 234     def __init__(self, nl) -> None:
 235         self.nl = nl
 236     def __getattr__(self, attr):
 237         nl = self.nl._create_nodelist()
 238         try:
 239             nl0 = nl[0]
 240         except IndexError:
 241             # If there is nothing in the list, then we have no attributes to
 242             # pass through, so raise AttributeError for everything.
 243             raise AttributeError("NodeList has no attribute: %s" % attr)
 244         return getattr(nl0, attr)
 245     def __str__(self) -> str:
 246         nl = self.nl._create_nodelist()
 247         if nl:
 248             return str(nl[0])
 249         return ''
 250     def __repr__(self) -> str:
 251         nl = self.nl._create_nodelist()
 252         if nl:
 253             return repr(nl[0])
 254         return ''
 255
 256 class NullNodeList(SCons.Util.NullSeq):
 257   def __call__(self, *args, **kwargs) -> str: return ''
 258   def __str__(self) -> str: return ''
 259
 260 NullNodesList = NullNodeList()
 261
 262 def subst_dict(target, source):
 263     """Create a dictionary for substitution of special
 264     construction variables.
 265
 266     This translates the following special arguments:
 267
 268     target - the target (object or array of objects),
 269              used to generate the TARGET and TARGETS
 270              construction variables
 271
 272     source - the source (object or array of objects),
 273              used to generate the SOURCES and SOURCE
 274              construction variables
 275     """
 276     dict = {}
 277
 278     if target:
 279         def get_tgt_subst_proxy(thing):
 280             try:
 281                 subst_proxy = thing.get_subst_proxy()
 282             except AttributeError:
 283                 subst_proxy = thing # probably a string, just return it
 284             return subst_proxy
 285         tnl = NLWrapper(target, get_tgt_subst_proxy)
 286         dict['TARGETS'] = Targets_or_Sources(tnl)
 287         dict['TARGET'] = Target_or_Source(tnl)
 288
 289         # This is a total cheat, but hopefully this dictionary goes
 290         # away soon anyway.  We just let these expand to $TARGETS
 291         # because that's "good enough" for the use of ToolSurrogates
 292         # (see test/ToolSurrogate.py) to generate documentation.
 293         dict['CHANGED_TARGETS'] = '$TARGETS'
 294         dict['UNCHANGED_TARGETS'] = '$TARGETS'
 295     else:
 296         dict['TARGETS'] = NullNodesList
 297         dict['TARGET'] = NullNodesList
 298
 299     if source:
 300         def get_src_subst_proxy(node):
 301             try:
 302                 rfile = node.rfile
 303             except AttributeError:
 304                 pass
 305             else:
 306                 node = rfile()
 307             try:
 308                 return node.get_subst_proxy()
 309             except AttributeError:
 310                 return node     # probably a String, just return it
 311         snl = NLWrapper(source, get_src_subst_proxy)
 312         dict['SOURCES'] = Targets_or_Sources(snl)
 313         dict['SOURCE'] = Target_or_Source(snl)
 314
 315         # This is a total cheat, but hopefully this dictionary goes
 316         # away soon anyway.  We just let these expand to $TARGETS
 317         # because that's "good enough" for the use of ToolSurrogates
 318         # (see test/ToolSurrogate.py) to generate documentation.
 319         dict['CHANGED_SOURCES'] = '$SOURCES'
 320         dict['UNCHANGED_SOURCES'] = '$SOURCES'
 321     else:
 322         dict['SOURCES'] = NullNodesList
 323         dict['SOURCE'] = NullNodesList
 324
 325     return dict
 326
 327
 328 _callable_args_set = {'target', 'source', 'env', 'for_signature'}
 329
 330 class StringSubber:
 331     """A class to construct the results of a scons_subst() call.
 332
 333     This binds a specific construction environment, mode, target and
 334     source with two methods (substitute() and expand()) that handle
 335     the expansion.
 336     """
 337
 338
 339     def __init__(self, env, mode, conv, gvars) -> None:
 340         self.env = env
 341         self.mode = mode
 342         self.conv = conv
 343         self.gvars = gvars
 344
 345     def expand(self, s, lvars):
 346         """Expand a single "token" as necessary, returning an
 347         appropriate string containing the expansion.
 348
 349         This handles expanding different types of things (strings,
 350         lists, callables) appropriately.  It calls the wrapper
 351         substitute() method to re-expand things as necessary, so that
 352         the results of expansions of side-by-side strings still get
 353         re-evaluated separately, not smushed together.
 354         """
 355         if is_String(s):
 356             try:
 357                 s0, s1 = s[:2]
 358             except (IndexError, ValueError):
 359                 return s
 360             if s0 != '$':
 361                 return s
 362             if s1 == '$':
 363                 # In this case keep the double $'s which we'll later
 364                 # swap for a single dollar sign as we need to retain
 365                 # this information to properly avoid matching "$("" when
 366                 # the actual text was "$$(""  (or "$)"" when "$$)"" )
 367                 return '$$'
 368             elif s1 in '()':
 369                 return s
 370             else:
 371                 key = s[1:]
 372                 if key[0] == '{' or '.' in key:
 373                     if key[0] == '{':
 374                         key = key[1:-1]
 375
 376                 # Store for error messages if we fail to expand the
 377                 # value
 378                 old_s = s
 379                 s = None
 380                 if key in lvars:
 381                      s = lvars[key]
 382                 elif key in self.gvars:
 383                      s = self.gvars[key]
 384                 else:
 385                      try:
 386                           s = eval(key, self.gvars, lvars)
 387                      except KeyboardInterrupt:
 388                           raise
 389                      except Exception as e:
 390                           if e.__class__ in AllowableExceptions:
 391                                return ''
 392                           raise_exception(e, lvars['TARGETS'], old_s)
 393
 394                 if s is None and NameError not in AllowableExceptions:
 395                      raise_exception(NameError(key), lvars['TARGETS'], old_s)
 396                 elif s is None:
 397                      return ''
 398
 399                 # Before re-expanding the result, handle
 400                 # recursive expansion by copying the local
 401                 # variable dictionary and overwriting a null
 402                 # string for the value of the variable name
 403                 # we just expanded.
 404                 #
 405                 # This could potentially be optimized by only
 406                 # copying lvars when s contains more expansions,
 407                 # but lvars is usually supposed to be pretty
 408                 # small, and deeply nested variable expansions
 409                 # are probably more the exception than the norm,
 410                 # so it should be tolerable for now.
 411                 lv = lvars.copy()
 412                 var = key.split('.')[0]
 413                 lv[var] = ''
 414                 return self.substitute(s, lv)
 415         elif is_Sequence(s):
 416             def func(l, conv=self.conv, substitute=self.substitute, lvars=lvars):
 417                 return conv(substitute(l, lvars))
 418             return list(map(func, s))
 419         elif callable(s):
 420
 421             # SCons has the unusual Null class where any __getattr__ call returns it's self,
 422             # which does not work the signature module, and the Null class returns an empty
 423             # string if called on, so we make an exception in this condition for Null class
 424             # Also allow callables where the only non default valued args match the expected defaults
 425             # this should also allow functools.partial's to work.
 426             if isinstance(s, SCons.Util.Null) or {k for k, v in signature(s).parameters.items() if
 427                                                   k in _callable_args_set or v.default == Parameter.empty} == _callable_args_set:
 428
 429                 s = s(target=lvars['TARGETS'],
 430                      source=lvars['SOURCES'],
 431                      env=self.env,
 432                      for_signature=(self.mode == SUBST_SIG))
 433             else:
 434                 # This probably indicates that it's a callable
 435                 # object that doesn't match our calling arguments
 436                 # (like an Action).
 437                 if self.mode == SUBST_RAW:
 438                     return s
 439                 s = self.conv(s)
 440             return self.substitute(s, lvars)
 441         elif s is None:
 442             return ''
 443         else:
 444             return s
 445
 446     def substitute(self, args, lvars):
 447         """Substitute expansions in an argument or list of arguments.
 448
 449         This serves as a wrapper for splitting up a string into
 450         separate tokens.
 451         """
 452         def sub_match(match):
 453             return self.conv(self.expand(match.group(1), lvars))
 454
 455         if is_String(args) and not isinstance(args, CmdStringHolder):
 456             args = str(args)        # In case it's a UserString.
 457             try:
 458                 result = _dollar_exps.sub(sub_match, args)
 459             except TypeError:
 460                 # If the internal conversion routine doesn't return
 461                 # strings (it could be overridden to return Nodes, for
 462                 # example), then the 1.5.2 re module will throw this
 463                 # exception.  Back off to a slower, general-purpose
 464                 # algorithm that works for all data types.
 465                 args = _separate_args.findall(args)
 466                 result = []
 467                 for a in args:
 468                     result.append(self.conv(self.expand(a, lvars)))
 469                 if len(result) == 1:
 470                     result = result[0]
 471                 else:
 472                     result = ''.join(map(str, result))
 473             return result
 474         else:
 475             return self.expand(args, lvars)
 476
 477
 478 class ListSubber(collections.UserList):
 479     """A class to construct the results of a scons_subst_list() call.
 480
 481     Like StringSubber, this class binds a specific construction
 482     environment, mode, target and source with two methods
 483     (substitute() and expand()) that handle the expansion.
 484
 485     In addition, however, this class is used to track the state of
 486     the result(s) we're gathering so we can do the appropriate thing
 487     whenever we have to append another word to the result--start a new
 488     line, start a new word, append to the current word, etc.  We do
 489     this by setting the "append" attribute to the right method so
 490     that our wrapper methods only need ever call ListSubber.append(),
 491     and the rest of the object takes care of doing the right thing
 492     internally.
 493     """
 494     def __init__(self, env, mode, conv, gvars) -> None:
 495         super().__init__([])
 496         self.env = env
 497         self.mode = mode
 498         self.conv = conv
 499         self.gvars = gvars
 500
 501         if self.mode == SUBST_RAW:
 502             self.add_strip = lambda x: self.append(x)
 503         else:
 504             self.add_strip = lambda x: None
 505         self.in_strip = None
 506         self.next_line()
 507
 508     def expanded(self, s) -> bool:
 509         """Determines if the string s requires further expansion.
 510
 511         Due to the implementation of ListSubber expand will call
 512         itself 2 additional times for an already expanded string. This
 513         method is used to determine if a string is already fully
 514         expanded and if so exit the loop early to prevent these
 515         recursive calls.
 516         """
 517         if not is_String(s) or isinstance(s, CmdStringHolder):
 518             return False
 519
 520         s = str(s)  # in case it's a UserString
 521         return _separate_args.findall(s) is None
 522
 523     def expand(self, s, lvars, within_list):
 524         """Expand a single "token" as necessary, appending the
 525         expansion to the current result.
 526
 527         This handles expanding different types of things (strings,
 528         lists, callables) appropriately.  It calls the wrapper
 529         substitute() method to re-expand things as necessary, so that
 530         the results of expansions of side-by-side strings still get
 531         re-evaluated separately, not smushed together.
 532         """
 533
 534         if is_String(s):
 535             try:
 536                 s0, s1 = s[:2]
 537             except (IndexError, ValueError):
 538                 self.append(s)
 539                 return
 540             if s0 != '$':
 541                 self.append(s)
 542                 return
 543             if s1 == '$':
 544                 self.append('$')
 545             elif s1 == '(':
 546                 self.open_strip('$(')
 547             elif s1 == ')':
 548                 self.close_strip('$)')
 549             else:
 550                 key = s[1:]
 551                 if key[0] == '{' or key.find('.') >= 0:
 552                     if key[0] == '{':
 553                         key = key[1:-1]
 554
 555                 # Store for error messages if we fail to expand the
 556                 # value
 557                 old_s = s
 558                 s = None
 559                 if key in lvars:
 560                      s = lvars[key]
 561                 elif key in self.gvars:
 562                      s = self.gvars[key]
 563                 else:
 564                      try:
 565                          s = eval(key, self.gvars, lvars)
 566                      except KeyboardInterrupt:
 567                          raise
 568                      except Exception as e:
 569                          if e.__class__ in AllowableExceptions:
 570                              return
 571                          raise_exception(e, lvars['TARGETS'], old_s)
 572
 573                 if s is None and NameError not in AllowableExceptions:
 574                      raise_exception(NameError(), lvars['TARGETS'], old_s)
 575                 elif s is None:
 576                      return
 577
 578                 # If the string is already full expanded there's no
 579                 # need to continue recursion.
 580                 if self.expanded(s):
 581                     self.append(s)
 582                     return
 583
 584                 # Before re-expanding the result, handle
 585                 # recursive expansion by copying the local
 586                 # variable dictionary and overwriting a null
 587                 # string for the value of the variable name
 588                 # we just expanded.
 589                 lv = lvars.copy()
 590                 var = key.split('.')[0]
 591                 lv[var] = ''
 592                 self.substitute(s, lv, 0)
 593                 self.this_word()
 594         elif is_Sequence(s):
 595             for a in s:
 596                 self.substitute(a, lvars, 1)
 597                 self.next_word()
 598         elif callable(s):
 599             # SCons has the unusual Null class where any __getattr__ call returns it's self,
 600             # which does not work the signature module, and the Null class returns an empty
 601             # string if called on, so we make an exception in this condition for Null class
 602             # Also allow callables where the only non default valued args match the expected defaults
 603             # this should also allow functools.partial's to work.
 604             if isinstance(s, SCons.Util.Null) or {k for k, v in signature(s).parameters.items() if
 605                                                   k in _callable_args_set or v.default == Parameter.empty} == _callable_args_set:
 606
 607                 s = s(target=lvars['TARGETS'],
 608                      source=lvars['SOURCES'],
 609                      env=self.env,
 610                      for_signature=(self.mode != SUBST_CMD))
 611             else:
 612                 # This probably indicates that it's a callable
 613                 # object that doesn't match our calling arguments
 614                 # (like an Action).
 615                 if self.mode == SUBST_RAW:
 616                     self.append(s)
 617                     return
 618                 s = self.conv(s)
 619             self.substitute(s, lvars, within_list)
 620         elif s is None:
 621             self.this_word()
 622         else:
 623             self.append(s)
 624
 625     def substitute(self, args, lvars, within_list) -> None:
 626         """Substitute expansions in an argument or list of arguments.
 627
 628         This serves as a wrapper for splitting up a string into
 629         separate tokens.
 630         """
 631
 632         if is_String(args) and not isinstance(args, CmdStringHolder):
 633             args = str(args)        # In case it's a UserString.
 634             args = _separate_args.findall(args)
 635             for a in args:
 636                 if a[0] in ' \t\n\r\f\v':
 637                     if '\n' in a:
 638                         self.next_line()
 639                     elif within_list:
 640                         self.append(a)
 641                     else:
 642                         self.next_word()
 643                 else:
 644                     self.expand(a, lvars, within_list)
 645         else:
 646             self.expand(args, lvars, within_list)
 647
 648     def next_line(self) -> None:
 649         """Arrange for the next word to start a new line.  This
 650         is like starting a new word, except that we have to append
 651         another line to the result."""
 652         collections.UserList.append(self, [])
 653         self.next_word()
 654
 655     def this_word(self) -> None:
 656         """Arrange for the next word to append to the end of the
 657         current last word in the result."""
 658         self.append = self.add_to_current_word
 659
 660     def next_word(self) -> None:
 661         """Arrange for the next word to start a new word."""
 662         self.append = self.add_new_word
 663
 664     def add_to_current_word(self, x) -> None:
 665         """Append the string x to the end of the current last word
 666         in the result.  If that is not possible, then just add
 667         it as a new word.  Make sure the entire concatenated string
 668         inherits the object attributes of x (in particular, the
 669         escape function) by wrapping it as CmdStringHolder."""
 670
 671         if not self.in_strip or self.mode != SUBST_SIG:
 672             try:
 673                 current_word = self[-1][-1]
 674             except IndexError:
 675                 self.add_new_word(x)
 676             else:
 677                 # All right, this is a hack and it should probably
 678                 # be refactored out of existence in the future.
 679                 # The issue is that we want to smoosh words together
 680                 # and make one file name that gets escaped if
 681                 # we're expanding something like foo$EXTENSION,
 682                 # but we don't want to smoosh them together if
 683                 # it's something like >$TARGET, because then we'll
 684                 # treat the '>' like it's part of the file name.
 685                 # So for now, just hard-code looking for the special
 686                 # command-line redirection characters...
 687                 try:
 688                     last_char = str(current_word)[-1]
 689                 except IndexError:
 690                     last_char = '\0'
 691                 if last_char in '<>|':
 692                     self.add_new_word(x)
 693                 else:
 694                     y = current_word + x
 695
 696                     # We used to treat a word appended to a literal
 697                     # as a literal itself, but this caused problems
 698                     # with interpreting quotes around space-separated
 699                     # targets on command lines.  Removing this makes
 700                     # none of the "substantive" end-to-end tests fail,
 701                     # so we'll take this out but leave it commented
 702                     # for now in case there's a problem not covered
 703                     # by the test cases and we need to resurrect this.
 704                     #literal1 = self.literal(self[-1][-1])
 705                     #literal2 = self.literal(x)
 706                     y = self.conv(y)
 707                     if is_String(y):
 708                         #y = CmdStringHolder(y, literal1 or literal2)
 709                         y = CmdStringHolder(y, None)
 710                     self[-1][-1] = y
 711
 712     def add_new_word(self, x) -> None:
 713         if not self.in_strip or self.mode != SUBST_SIG:
 714             literal = self.literal(x)
 715             x = self.conv(x)
 716             if is_String(x):
 717                 x = CmdStringHolder(x, literal)
 718             self[-1].append(x)
 719         self.append = self.add_to_current_word
 720
 721     def literal(self, x):
 722         try:
 723             l = x.is_literal
 724         except AttributeError:
 725             return None
 726         else:
 727             return l()
 728
 729     def open_strip(self, x) -> None:
 730         """Handle the "open strip" $( token."""
 731         self.add_strip(x)
 732         self.in_strip = 1
 733
 734     def close_strip(self, x) -> None:
 735         """Handle the "close strip" $) token."""
 736         self.add_strip(x)
 737         self.in_strip = None
 738
 739
 740 # Constants for the "mode" parameter to scons_subst_list() and
 741 # scons_subst().  SUBST_RAW gives the raw command line.  SUBST_CMD
 742 # gives a command line suitable for passing to a shell.  SUBST_SIG
 743 # gives a command line appropriate for calculating the signature
 744 # of a command line...if this changes, we should rebuild.
 745 SUBST_CMD = 0
 746 SUBST_RAW = 1
 747 SUBST_SIG = 2
 748
 749 _rm = re.compile(r'\$[()]')
 750
 751 # Note the pattern below only matches $( or $) when there is no
 752 # preceeding $. (Thus the (?<!\$))
 753 _rm_split = re.compile(r'(?<!\$)(\$[()])')
 754
 755 # Indexed by the SUBST_* constants above.
 756 _regex_remove = [ _rm, None, _rm_split ]
 757
 758 def _rm_list(list):
 759     return [l for l in list if l not in ('$(', '$)')]
 760
 761 def _remove_list(list):
 762     result = []
 763     depth = 0
 764     for l in list:
 765         if l == '$(':
 766             depth += 1
 767         elif l == '$)':
 768             depth -= 1
 769             if depth < 0:
 770                 break
 771         elif depth == 0:
 772             result.append(l)
 773     if depth != 0:
 774         return None
 775     return result
 776
 777 # Indexed by the SUBST_* constants above.
 778 _list_remove = [ _rm_list, None, _remove_list ]
 779
 780 # Regular expressions for splitting strings and handling substitutions,
 781 # for use by the scons_subst() and scons_subst_list() functions:
 782 #
 783 # The first expression compiled matches all of the $-introduced tokens
 784 # that we need to process in some way, and is used for substitutions.
 785 # The expressions it matches are:
 786 #
 787 #       "$$"
 788 #       "$("
 789 #       "$)"
 790 #       "$variable"             [must begin with alphabetic or underscore]
 791 #       "${any stuff}"
 792 #
 793 # The second expression compiled is used for splitting strings into tokens
 794 # to be processed, and it matches all of the tokens listed above, plus
 795 # the following that affect how arguments do or don't get joined together:
 796 #
 797 #       "   "                   [white space]
 798 #       "non-white-space"       [without any dollar signs]
 799 #       "$"                     [single dollar sign]
 800 #
 801 _dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
 802 _dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
 803 _separate_args = re.compile(r'(%s|\s+|[^\s$]+|\$)' % _dollar_exps_str)
 804
 805 # This regular expression is used to replace strings of multiple white
 806 # space characters in the string result from the scons_subst() function.
 807 _space_sep = re.compile(r'[\t ]+(?![^{]*})')
 808
 809
 810 def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides: Optional[dict] = None):
 811     """Expand a string or list containing construction variable
 812     substitutions.
 813
 814     This is the work-horse function for substitutions in file names
 815     and the like.  The companion scons_subst_list() function (below)
 816     handles separating command lines into lists of arguments, so see
 817     that function if that's what you're looking for.
 818     """
 819     if (isinstance(strSubst, str) and '$' not in strSubst) or isinstance(strSubst, CmdStringHolder):
 820         return strSubst
 821
 822     if conv is None:
 823         conv = _strconv[mode]
 824
 825     # Doing this every time is a bit of a waste, since the Executor
 826     # has typically already populated the OverrideEnvironment with
 827     # $TARGET/$SOURCE variables.  We're keeping this (for now), though,
 828     # because it supports existing behavior that allows us to call
 829     # an Action directly with an arbitrary target+source pair, which
 830     # we use in Tool/tex.py to handle calling $BIBTEX when necessary.
 831     # If we dropped that behavior (or found another way to cover it),
 832     # we could get rid of this call completely and just rely on the
 833     # Executor setting the variables.
 834     if 'TARGET' not in lvars:
 835         d = subst_dict(target, source)
 836         if d:
 837             lvars = lvars.copy()
 838             lvars.update(d)
 839
 840     # Allow last ditch chance to override lvars
 841     if overrides:
 842         lvars.update(overrides)
 843
 844     # We're (most likely) going to eval() things.  If Python doesn't
 845     # find a __builtins__ value in the global dictionary used for eval(),
 846     # it copies the current global values for you.  Avoid this by
 847     # setting it explicitly and then deleting, so we don't pollute the
 848     # construction environment Dictionary(ies) that are typically used
 849     # for expansion.
 850     gvars['__builtins__'] = __builtins__
 851
 852     ss = StringSubber(env, mode, conv, gvars)
 853     result = ss.substitute(strSubst, lvars)
 854
 855     try:
 856         del gvars['__builtins__']
 857     except KeyError:
 858         pass
 859
 860     res = result
 861     if is_String(result):
 862         # Remove $(-$) pairs and any stuff in between,
 863         # if that's appropriate.
 864         remove = _regex_remove[mode]
 865         if remove:
 866             if mode == SUBST_SIG:
 867                 result = _list_remove[mode](remove.split(result))
 868                 if result is None:
 869                     raise SCons.Errors.UserError("Unbalanced $(/$) in: " + res)
 870                 result = ' '.join(result)
 871             else:
 872                 result = remove.sub('', result)
 873         if mode != SUBST_RAW:
 874             # Compress strings of white space characters into
 875             # a single space.
 876             result = _space_sep.sub(' ', result).strip()
 877
 878         # Now replace escaped $'s currently "$$"
 879         # This is needed because we now retain $$ instead of
 880         # replacing them during substition to avoid
 881         # improperly trying to escape "$$(" as being "$("
 882         result = result.replace('$$','$')
 883     elif is_Sequence(result):
 884         remove = _list_remove[mode]
 885         if remove:
 886             result = remove(result)
 887             if result is None:
 888                 raise SCons.Errors.UserError("Unbalanced $(/$) in: " + str(res))
 889
 890     return result
 891
 892 def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides: Optional[dict] = None):
 893     """Substitute construction variables in a string (or list or other
 894     object) and separate the arguments into a command list.
 895
 896     The companion scons_subst() function (above) handles basic
 897     substitutions within strings, so see that function instead
 898     if that's what you're looking for.
 899     """
 900     if conv is None:
 901         conv = _strconv[mode]
 902
 903     # Doing this every time is a bit of a waste, since the Executor
 904     # has typically already populated the OverrideEnvironment with
 905     # $TARGET/$SOURCE variables.  We're keeping this (for now), though,
 906     # because it supports existing behavior that allows us to call
 907     # an Action directly with an arbitrary target+source pair, which
 908     # we use in Tool/tex.py to handle calling $BIBTEX when necessary.
 909     # If we dropped that behavior (or found another way to cover it),
 910     # we could get rid of this call completely and just rely on the
 911     # Executor setting the variables.
 912     if 'TARGET' not in lvars:
 913         d = subst_dict(target, source)
 914         if d:
 915             lvars = lvars.copy()
 916             lvars.update(d)
 917
 918     # Allow caller to specify last ditch override of lvars
 919     if overrides:
 920         lvars.update(overrides)
 921
 922     # We're (most likely) going to eval() things.  If Python doesn't
 923     # find a __builtins__ value in the global dictionary used for eval(),
 924     # it copies the current global values for you.  Avoid this by
 925     # setting it explicitly and then deleting, so we don't pollute the
 926     # construction environment Dictionary(ies) that are typically used
 927     # for expansion.
 928     gvars['__builtins__'] = __builtins__
 929
 930     ls = ListSubber(env, mode, conv, gvars)
 931     ls.substitute(strSubst, lvars, 0)
 932
 933     try:
 934         del gvars['__builtins__']
 935     except KeyError:
 936         pass
 937
 938     return ls.data
 939
 940 def scons_subst_once(strSubst, env, key):
 941     """Perform single (non-recursive) substitution of a single
 942     construction variable keyword.
 943
 944     This is used when setting a variable when copying or overriding values
 945     in an Environment.  We want to capture (expand) the old value before
 946     we override it, so people can do things like:
 947
 948         env2 = env.Clone(CCFLAGS = '$CCFLAGS -g')
 949
 950     We do this with some straightforward, brute-force code here...
 951     """
 952     if isinstance(strSubst, str) and strSubst.find('$') < 0:
 953         return strSubst
 954
 955     matchlist = ['$' + key, '${' + key + '}']
 956     val = env.get(key, '')
 957     def sub_match(match, val=val, matchlist=matchlist):
 958         a = match.group(1)
 959         if a in matchlist:
 960             a = val
 961         if is_Sequence(a):
 962             return ' '.join(map(str, a))
 963         else:
 964             return str(a)
 965
 966     if is_Sequence(strSubst):
 967         result = []
 968         for arg in strSubst:
 969             if is_String(arg):
 970                 if arg in matchlist:
 971                     arg = val
 972                     if is_Sequence(arg):
 973                         result.extend(arg)
 974                     else:
 975                         result.append(arg)
 976                 else:
 977                     result.append(_dollar_exps.sub(sub_match, arg))
 978             else:
 979                 result.append(arg)
 980         return result
 981     elif is_String(strSubst):
 982         return _dollar_exps.sub(sub_match, strSubst)
 983     else:
 984         return strSubst
 985
 986 # Local Variables:
 987 # tab-width:4
 988 # indent-tabs-mode:nil
 989 # End:
 990 # vim: set expandtab tabstop=4 shiftwidth=4: