SCons/Tool/ninja/Utils.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23 import os
  24 import shutil
  25 from os.path import join as joinpath
  26 from collections import OrderedDict
  27
  28 import SCons
  29 from SCons.Action import get_default_ENV, _string_from_cmd_list
  30 from SCons.Script import AddOption
  31 from SCons.Util import is_List, flatten_sequence
  32
  33 class NinjaExperimentalWarning(SCons.Warnings.WarningOnByDefault):
  34     pass
  35
  36
  37 def ninja_add_command_line_options() -> None:
  38     """
  39     Add additional command line arguments to SCons specific to the ninja tool
  40     """
  41     AddOption('--disable-execute-ninja',
  42               dest='disable_execute_ninja',
  43               metavar='BOOL',
  44               action="store_true",
  45               default=False,
  46               help='Disable automatically running ninja after scons')
  47
  48     AddOption('--disable-ninja',
  49               dest='disable_ninja',
  50               metavar='BOOL',
  51               action="store_true",
  52               default=False,
  53               help='Disable ninja generation and build with scons even if tool is loaded. '+
  54                    'Also used by ninja to build targets which only scons can build.')
  55
  56     AddOption('--skip-ninja-regen',
  57               dest='skip_ninja_regen',
  58               metavar='BOOL',
  59               action="store_true",
  60               default=False,
  61               help='Allow scons to skip regeneration of the ninja file and restarting of the daemon. ' +
  62                     'Care should be taken in cases where Glob is in use or SCons generated files are used in ' +
  63                     'command lines.')
  64
  65
  66 def is_valid_dependent_node(node) -> bool:
  67     """
  68     Return True if node is not an alias or is an alias that has children
  69
  70     This prevents us from making phony targets that depend on other
  71     phony targets that will never have an associated ninja build
  72     target.
  73
  74     We also have to specify that it's an alias when doing the builder
  75     check because some nodes (like src files) won't have builders but
  76     are valid implicit dependencies.
  77     """
  78     if isinstance(node, SCons.Node.Alias.Alias):
  79         return bool(node.children())
  80
  81     return not node.get_env().get("NINJA_SKIP")
  82
  83
  84 def alias_to_ninja_build(node):
  85     """Convert an Alias node into a Ninja phony target"""
  86     return {
  87         "outputs": get_outputs(node),
  88         "rule": "phony",
  89         "implicit": [
  90             get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n)
  91         ],
  92     }
  93
  94
  95 def check_invalid_ninja_node(node) -> bool:
  96     return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias))
  97
  98
  99 def filter_ninja_nodes(node_list):
 100     ninja_nodes = []
 101     for node in node_list:
 102         if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) and not node.get_env().get('NINJA_SKIP'):
 103             ninja_nodes.append(node)
 104         else:
 105             continue
 106     return ninja_nodes
 107
 108
 109 def get_input_nodes(node):
 110     if node.get_executor() is not None:
 111         inputs = node.get_executor().get_all_sources()
 112     else:
 113         inputs = node.sources
 114     return inputs
 115
 116
 117 def invalid_ninja_nodes(node, targets):
 118     result = False
 119     for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]:
 120         if node_list:
 121             result = result or any([check_invalid_ninja_node(node) for node in node_list])
 122     return result
 123
 124
 125 def get_order_only(node):
 126     """Return a list of order only dependencies for node."""
 127     if node.prerequisites is None:
 128         return []
 129     return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)]
 130
 131
 132 def get_dependencies(node, skip_sources: bool=False):
 133     """Return a list of dependencies for node."""
 134     if skip_sources:
 135         return [
 136             get_path(src_file(child))
 137             for child in filter_ninja_nodes(node.children())
 138             if child not in node.sources
 139         ]
 140     return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())]
 141
 142
 143 def get_inputs(node):
 144     """Collect the Ninja inputs for node."""
 145     return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))]
 146
 147
 148 def get_outputs(node):
 149     """Collect the Ninja outputs for node."""
 150     executor = node.get_executor()
 151     if executor is not None:
 152         outputs = executor.get_all_targets()
 153     else:
 154         if hasattr(node, "target_peers"):
 155             outputs = node.target_peers
 156         else:
 157             outputs = [node]
 158
 159     outputs = [get_path(o) for o in filter_ninja_nodes(outputs)]
 160
 161     return outputs
 162
 163
 164 def get_targets_sources(node):
 165     executor = node.get_executor()
 166     if executor is not None:
 167         tlist = executor.get_all_targets()
 168         slist = executor.get_all_sources()
 169     else:
 170         if hasattr(node, "target_peers"):
 171             tlist = node.target_peers
 172         else:
 173             tlist = [node]
 174         slist = node.sources
 175
 176     # Retrieve the repository file for all sources
 177     slist = [rfile(s) for s in slist]
 178     return tlist, slist
 179
 180
 181 def get_path(node):
 182     """
 183     Return a fake path if necessary.
 184
 185     As an example Aliases use this as their target name in Ninja.
 186     """
 187     if hasattr(node, "get_path"):
 188         return node.get_path()
 189     return str(node)
 190
 191
 192 def rfile(node):
 193     """
 194     Return the repository file for node if it has one. Otherwise return node
 195     """
 196     if hasattr(node, "rfile"):
 197         return node.rfile()
 198     return node
 199
 200
 201 def src_file(node):
 202     """Returns the src code file if it exists."""
 203     if hasattr(node, "srcnode"):
 204         src = node.srcnode()
 205         if src.stat() is not None:
 206             return src
 207     return get_path(node)
 208
 209
 210 def get_rule(node, rule):
 211     tlist, slist = get_targets_sources(node)
 212     if invalid_ninja_nodes(node, tlist):
 213         return "TEMPLATE"
 214     else:
 215         return rule
 216
 217
 218 def to_escaped_list(env, lst):
 219     """
 220     Ninja tool function for returning an escaped list of strings from a subst
 221     generator.
 222
 223     env_var arg can be a list or a subst generator which returns a list.
 224     """
 225
 226     # subst_list will take in either a raw list or a subst callable which generates
 227     # a list, and return a list of CmdStringHolders which can be converted into raw strings.
 228     # If a raw list was passed in, then scons_list will make a list of lists from the original
 229     # values and even subst items in the list if they are substitutable. Flatten will flatten
 230     # the list in that case, to ensure for either input we have a list of CmdStringHolders.
 231     deps_list = env.Flatten(env.subst_list(lst))
 232
 233     # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings
 234     # and make sure to escape the strings to handle spaces in paths. We also will sort the result
 235     # keep the order of the list consistent.
 236     return sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list])
 237
 238
 239 def generate_depfile(env, node, dependencies) -> None:
 240     """
 241     Ninja tool function for writing a depfile. The depfile should include
 242     the node path followed by all the dependent files in a makefile format.
 243
 244     dependencies arg can be a list or a subst generator which returns a list.
 245     """
 246
 247     depfile = os.path.join(get_path(env['NINJA_DIR']), str(node) + '.depfile')
 248
 249     depfile_contents = str(node) + ": " + ' '.join(to_escaped_list(env, dependencies))
 250
 251     need_rewrite = False
 252     try:
 253         with open(depfile, 'r') as f:
 254             need_rewrite = (f.read() != depfile_contents)
 255     except FileNotFoundError:
 256         need_rewrite = True
 257
 258     if need_rewrite:
 259         os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True)
 260         with open(depfile, 'w') as f:
 261             f.write(depfile_contents)
 262
 263
 264 def ninja_noop(*_args, **_kwargs):
 265     """
 266     A general purpose no-op function.
 267
 268     There are many things that happen in SCons that we don't need and
 269     also don't return anything. We use this to disable those functions
 270     instead of creating multiple definitions of the same thing.
 271     """
 272     return None
 273
 274 def ninja_recursive_sorted_dict(build):
 275     sorted_dict = OrderedDict()
 276     for key, val in sorted(build.items()):
 277         if isinstance(val, dict):
 278             sorted_dict[key] = ninja_recursive_sorted_dict(val)
 279         else:
 280             sorted_dict[key] = val
 281     return sorted_dict
 282
 283
 284 def ninja_sorted_build(ninja, **build) -> None:
 285     sorted_dict = ninja_recursive_sorted_dict(build)
 286     ninja.build(**sorted_dict)
 287
 288
 289 def get_command_env(env, target, source):
 290     """
 291     Return a string that sets the environment for any environment variables that
 292     differ between the OS environment and the SCons command ENV.
 293
 294     It will be compatible with the default shell of the operating system.
 295     """
 296     try:
 297         return env["NINJA_ENV_VAR_CACHE"]
 298     except KeyError:
 299         pass
 300
 301     # Scan the ENV looking for any keys which do not exist in
 302     # os.environ or differ from it. We assume if it's a new or
 303     # differing key from the process environment then it's
 304     # important to pass down to commands in the Ninja file.
 305     ENV = SCons.Action._resolve_shell_env(env, target, source)
 306     scons_specified_env = {
 307         key: value
 308         for key, value in ENV.items()
 309         # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's.
 310         if key not in os.environ or os.environ.get(key, None) != value
 311     }
 312
 313     windows = env["PLATFORM"] == "win32"
 314     command_env = ""
 315     scons_specified_env = SCons.Util.sanitize_shell_env(scons_specified_env)
 316     for key, value in scons_specified_env.items():
 317         if windows:
 318             command_env += "set '{}={}' && ".format(key, value)
 319         else:
 320             # We address here *only* the specific case that a user might have
 321             # an environment variable which somehow gets included and has
 322             # spaces in the value. These are escapes that Ninja handles. This
 323             # doesn't make builds on paths with spaces (Ninja and SCons issues)
 324             # nor expanding response file paths with spaces (Ninja issue) work.
 325             value = value.replace(r' ', r'$ ')
 326             command_env += "export {}='{}';".format(key, value)
 327
 328     env["NINJA_ENV_VAR_CACHE"] = command_env
 329     return command_env
 330
 331
 332 def get_comstr(env, action, targets, sources):
 333     """Get the un-substituted string for action."""
 334     # Despite being having "list" in it's name this member is not
 335     # actually a list. It's the pre-subst'd string of the command. We
 336     # use it to determine if the command we're about to generate needs
 337     # to use a custom Ninja rule. By default this redirects CC, CXX,
 338     # AR, SHLINK, and LINK commands to their respective rules but the
 339     # user can inject custom Ninja rules and tie them to commands by
 340     # using their pre-subst'd string.
 341     if hasattr(action, "process"):
 342         return action.cmd_list
 343
 344     return action.genstring(targets, sources, env)
 345
 346
 347 def generate_command(env, node, action, targets, sources, executor=None):
 348     # Actions like CommandAction have a method called process that is
 349     # used by SCons to generate the cmd_line they need to run. So
 350     # check if it's a thing like CommandAction and call it if we can.
 351     if hasattr(action, "process"):
 352         cmd_list, _, _ = action.process(targets, sources, env, executor=executor)
 353         cmd = _string_from_cmd_list(cmd_list[0])
 354     else:
 355         # Anything else works with genstring, this is most commonly hit by
 356         # ListActions which essentially call process on all of their
 357         # commands and concatenate it for us.
 358         genstring = action.genstring(targets, sources, env)
 359         if executor is not None:
 360             cmd = env.subst(genstring, executor=executor)
 361         else:
 362             cmd = env.subst(genstring, targets, sources)
 363
 364         cmd = cmd.replace("\n", " && ").strip()
 365         if cmd.endswith("&&"):
 366             cmd = cmd[0:-2].strip()
 367
 368     # Escape dollars as necessary
 369     return cmd.replace("$", "$$")
 370
 371
 372 def ninja_csig(original):
 373     """Return a dummy csig"""
 374
 375     def wrapper(self):
 376         if isinstance(self, SCons.Node.Node) and self.is_sconscript():
 377             return original(self)
 378         return "dummy_ninja_csig"
 379
 380     return wrapper
 381
 382
 383 def ninja_contents(original):
 384     """Return a dummy content without doing IO"""
 385
 386     def wrapper(self):
 387         if isinstance(self, SCons.Node.Node) and (self.is_sconscript() or self.is_conftest()):
 388             return original(self)
 389         return bytes("dummy_ninja_contents", encoding="utf-8")
 390
 391     return wrapper
 392
 393
 394 def ninja_stat(_self, path):
 395     """
 396     Eternally memoized stat call.
 397
 398     SCons is very aggressive about clearing out cached values. For our
 399     purposes everything should only ever call stat once since we're
 400     running in a no_exec build the file system state should not
 401     change. For these reasons we patch SCons.Node.FS.LocalFS.stat to
 402     use our eternal memoized dictionary.
 403     """
 404
 405     try:
 406         return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path]
 407     except KeyError:
 408         try:
 409             result = os.stat(path)
 410         except os.error:
 411             result = None
 412
 413         SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result
 414         return result
 415
 416
 417 def ninja_whereis(thing, *_args, **_kwargs):
 418     """Replace env.WhereIs with a much faster version"""
 419
 420     # Optimize for success, this gets called significantly more often
 421     # when the value is already memoized than when it's not.
 422     try:
 423         return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing]
 424     except KeyError:
 425         # TODO: Fix this to respect env['ENV']['PATH']... WPD
 426         # We do not honor any env['ENV'] or env[*] variables in the
 427         # generated ninja file. Ninja passes your raw shell environment
 428         # down to it's subprocess so the only sane option is to do the
 429         # same during generation. At some point, if and when we try to
 430         # upstream this, I'm sure a sticking point will be respecting
 431         # env['ENV'] variables and such but it's actually quite
 432         # complicated. I have a naive version but making it always work
 433         # with shell quoting is nigh impossible. So I've decided to
 434         # cross that bridge when it's absolutely required.
 435         path = shutil.which(thing)
 436         SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path
 437         return path
 438
 439
 440 def ninja_print_conf_log(s, target, source, env) -> None:
 441     """Command line print only for conftest to generate a correct conf log."""
 442     if target and target[0].is_conftest():
 443         action = SCons.Action._ActionAction()
 444         action.print_cmd_line(s, target, source, env)