site_scons/site_tools/component_builders.py

   1 #!/usr/bin/python2.4
   2 # Copyright 2009, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Software construction toolkit builders for SCons."""
  32
  33
  34 import SCons
  35
  36
  37 __component_list = {}
  38
  39
  40 def _InitializeComponentBuilders(env):
  41   """Re-initializes component builders module.
  42
  43   Args:
  44     env: Environment context
  45   """
  46   env = env     # Silence gpylint
  47
  48   __component_list.clear()
  49
  50
  51 def _RetrieveComponents(component_name, filter_components=None):
  52   """Get the list of all components required by the specified component.
  53
  54   Args:
  55     component_name: Name of the base component.
  56     filter_components: List of components NOT to include.
  57
  58   Returns:
  59     A list of the transitive closure of all components required by the base
  60     component.  That is, if A requires B and B requires C, this returns [B, C].
  61
  62   """
  63   if filter_components:
  64     filter_components = set(filter_components)
  65   else:
  66     filter_components = set()
  67
  68   components = set([component_name])    # Components always require themselves
  69   new_components = set(components)
  70   while new_components:
  71     # Take next new component and add it to the list we've already scanned.
  72     c = new_components.pop()
  73     components.add(c)
  74     # Add to the list of new components any of c's components that we haven't
  75     # seen before.
  76     new_components.update(__component_list.get(c, set())
  77                           - components - filter_components)
  78
  79   return list(components)
  80
  81
  82 def _StoreComponents(self, component_name):
  83   """Stores the list of child components for the specified component.
  84
  85   Args:
  86     self: Environment containing component.
  87     component_name: Name of the component.
  88
  89   Adds component references based on the LIBS and COMPONENTS variables in the
  90   current environment.  Should be called at primary SConscript execution time;
  91   use _RetrieveComponents() to get the final components lists in a Defer()'d
  92   function.
  93   """
  94
  95   components = set()
  96   for clist in ('LIBS', 'COMPONENTS'):
  97     components.update(map(self.subst, self.Flatten(self[clist])))
  98
  99   if component_name not in __component_list:
 100     __component_list[component_name] = set()
 101   __component_list[component_name].update(components)
 102
 103
 104 def _ComponentPlatformSetup(env, builder_name, **kwargs):
 105   """Modify an environment to work with a component builder.
 106
 107   Args:
 108     env: Environment to clone.
 109     builder_name: Name of the builder.
 110     kwargs: Keyword arguments.
 111
 112   Returns:
 113     A modified clone of the environment.
 114   """
 115   # Clone environment so we can modify it
 116   env = env.Clone()
 117
 118   # Add all keyword arguments to the environment
 119   for k, v in kwargs.items():
 120     env[k] = v
 121
 122   # Add compiler flags for included headers, if any
 123   env['INCLUDES'] = env.Flatten(env.subst_list(['$INCLUDES']))
 124   for h in env['INCLUDES']:
 125     env.Append(CCFLAGS=['${CCFLAG_INCLUDE}%s' % h])
 126
 127   # Call platform-specific component setup function, if any
 128   if env.get('COMPONENT_PLATFORM_SETUP'):
 129     env['COMPONENT_PLATFORM_SETUP'](env, builder_name)
 130
 131   # Return the modified environment
 132   return env
 133
 134 #------------------------------------------------------------------------------
 135
 136 # TODO: Should be possible to refactor programs, test programs, libs to all
 137 # publish as packages, for simplicity and code reuse.
 138
 139
 140 def ComponentPackageDeferred(env):
 141   """Deferred build steps for component package.
 142
 143   Args:
 144     env: Environment from ComponentPackage().
 145
 146   Sets up the aliases to build the package.
 147   """
 148   package_name = env['PACKAGE_NAME']
 149
 150   # Install program and resources
 151   all_outputs = []
 152   package_filter = env.Flatten(env.subst_list('$COMPONENT_PACKAGE_FILTER'))
 153   components = _RetrieveComponents(package_name, package_filter)
 154   for resource, dest_dir in env.get('COMPONENT_PACKAGE_RESOURCES').items():
 155     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 156
 157   # Add installed program and resources to the alias
 158   env.Alias(package_name, all_outputs)
 159
 160
 161 def ComponentPackage(self, package_name, dest_dir, **kwargs):
 162   """Pseudo-builder for package containing other components.
 163
 164   Args:
 165     self: Environment in which we were called.
 166     package_name: Name of package.
 167     dest_dir: Destination directory for package.
 168     kwargs: Keyword arguments.
 169
 170   Returns:
 171     The alias node for the package.
 172   """
 173   # Clone and modify environment
 174   env = _ComponentPlatformSetup(self, 'ComponentPackage', **kwargs)
 175
 176   env.Replace(
 177       PACKAGE_NAME=package_name,
 178       PACKAGE_DIR=dest_dir,
 179   )
 180
 181   # Add an empty alias for the package and add it to the right groups
 182   a = env.Alias(package_name, [])
 183   for group in env['COMPONENT_PACKAGE_GROUPS']:
 184     SCons.Script.Alias(group, a)
 185
 186   # Store list of components for this program
 187   env._StoreComponents(package_name)
 188
 189   # Let component_targets know this target is available in the current mode
 190   env.SetTargetProperty(package_name, TARGET_PATH=dest_dir)
 191
 192   # Set up deferred call to replicate resources
 193   env.Defer(ComponentPackageDeferred)
 194
 195   # Return the alias, since it's the only node we have
 196   return a
 197
 198 #------------------------------------------------------------------------------
 199
 200
 201 def ComponentObject(self, *args, **kwargs):
 202   """Pseudo-builder for object to handle platform-dependent type.
 203
 204   Args:
 205     self: Environment in which we were called.
 206     args: Positional arguments.
 207     kwargs: Keyword arguments.
 208
 209   Returns:
 210     Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
 211
 212   TODO: Perhaps this should be a generator builder, so it can take a list of
 213   inputs and return a list of outputs?
 214   """
 215   # Clone and modify environment
 216   env = _ComponentPlatformSetup(self, 'ComponentObject', **kwargs)
 217
 218   # Make appropriate object type
 219   if env.get('COMPONENT_STATIC'):
 220     o = env.StaticObject(*args, **kwargs)
 221   else:
 222     o = env.SharedObject(*args, **kwargs)
 223
 224   # Add dependencies on includes
 225   env.Depends(o, env['INCLUDES'])
 226
 227   return o
 228
 229 #------------------------------------------------------------------------------
 230
 231
 232 def ComponentLibrary(self, lib_name, *args, **kwargs):
 233   """Pseudo-builder for library to handle platform-dependent type.
 234
 235   Args:
 236     self: Environment in which we were called.
 237     lib_name: Library name.
 238     args: Positional arguments.
 239     kwargs: Keyword arguments.
 240
 241   Returns:
 242     Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
 243   """
 244   # Clone and modify environment
 245   env = _ComponentPlatformSetup(self, 'ComponentLibrary', **kwargs)
 246
 247   # Make appropriate library type
 248   if env.get('COMPONENT_STATIC'):
 249     lib_outputs = env.StaticLibrary(lib_name, *args, **kwargs)
 250   else:
 251     lib_outputs = env.SharedLibrary(lib_name, *args, **kwargs)
 252
 253   # Add dependencies on includes
 254   env.Depends(lib_outputs, env['INCLUDES'])
 255
 256   # Scan library outputs for files we need to link against this library, and
 257   # files we need to run executables linked against this library.
 258   need_for_link = []
 259   need_for_debug = []
 260   need_for_run = []
 261   for o in lib_outputs:
 262     if o.suffix in env['COMPONENT_LIBRARY_LINK_SUFFIXES']:
 263       need_for_link.append(o)
 264     if o.suffix in env['COMPONENT_LIBRARY_DEBUG_SUFFIXES']:
 265       need_for_debug.append(o)
 266     if o.suffix == env['SHLIBSUFFIX']:
 267       need_for_run.append(o)
 268   all_outputs = lib_outputs
 269
 270   # Install library in intermediate directory, so other libs and programs can
 271   # link against it
 272   all_outputs += env.Replicate('$LIB_DIR', need_for_link)
 273
 274   # Publish output
 275   env.Publish(lib_name, 'link', need_for_link)
 276   env.Publish(lib_name, 'run', need_for_run)
 277   env.Publish(lib_name, 'debug', need_for_debug)
 278
 279   # Add an alias to build and copy the library, and add it to the right groups
 280   a = self.Alias(lib_name, all_outputs)
 281   for group in env['COMPONENT_LIBRARY_GROUPS']:
 282     SCons.Script.Alias(group, a)
 283
 284   # Store list of components for this library
 285   env._StoreComponents(lib_name)
 286
 287   # Let component_targets know this target is available in the current mode.
 288   env.SetTargetProperty(lib_name, TARGET_PATH=lib_outputs[0])
 289
 290   # If library should publish itself, publish as if it was a program
 291   if env.get('COMPONENT_LIBRARY_PUBLISH'):
 292     env['PROGRAM_BASENAME'] = lib_name
 293     env.Defer(ComponentProgramDeferred)
 294
 295   # Return the library outputs
 296   return lib_outputs
 297
 298 #------------------------------------------------------------------------------
 299
 300
 301 def ComponentTestProgramDeferred(env):
 302   """Deferred build steps for test program.
 303
 304   Args:
 305     env: Environment from ComponentTestProgram().
 306
 307   Sets up the aliases to compile and run the test program.
 308   """
 309   prog_name = env['PROGRAM_BASENAME']
 310
 311   # Install program and resources
 312   all_outputs = []
 313   components = _RetrieveComponents(prog_name)
 314   for resource, dest_dir in env.get('COMPONENT_TEST_RESOURCES').items():
 315     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 316
 317   # Add installed program and resources to the alias
 318   env.Alias(prog_name, all_outputs)
 319
 320   # Add target properties
 321   env.SetTargetProperty(
 322       prog_name,
 323       # The copy of the program we care about is the one in the tests dir
 324       EXE='$TESTS_DIR/$PROGRAM_NAME',
 325       RUN_CMDLINE='$COMPONENT_TEST_CMDLINE',
 326       RUN_DIR='$TESTS_DIR',
 327       TARGET_PATH='$TESTS_DIR/$PROGRAM_NAME',
 328   )
 329
 330   # Add an alias for running the test in the test directory, if the test is
 331   # runnable and has a test command line.
 332   if env.get('COMPONENT_TEST_RUNNABLE') and env.get('COMPONENT_TEST_CMDLINE'):
 333     env.Replace(
 334         COMMAND_OUTPUT_CMDLINE=env['COMPONENT_TEST_CMDLINE'],
 335         COMMAND_OUTPUT_RUN_DIR='$TESTS_DIR',
 336     )
 337     test_out_name = '$TEST_OUTPUT_DIR/${PROGRAM_BASENAME}.out.txt'
 338     if (env.GetOption('component_test_retest')
 339         and env.File(test_out_name).exists()):
 340       # Delete old test results, so test will rerun.
 341       env.Execute(SCons.Script.Delete(test_out_name))
 342
 343     # Set timeout based on test size
 344     timeout = env.get('COMPONENT_TEST_TIMEOUT')
 345     if type(timeout) is dict:
 346       timeout = timeout.get(env.get('COMPONENT_TEST_SIZE'))
 347     if timeout:
 348       env['COMMAND_OUTPUT_TIMEOUT'] = timeout
 349
 350     # Test program is the first run resource we replicated.  (Duplicate
 351     # replicate is not harmful, and is a handy way to pick out the correct
 352     # file from all those we replicated above.)
 353     test_program = env.ReplicatePublished('$TESTS_DIR', prog_name, 'run')
 354
 355     # Run the test.  Note that we need to refer to the file by name, so that
 356     # SCons will recreate the file node after we've deleted it; if we used the
 357     # env.File() we created in the if statement above, SCons would still think
 358     # it exists and not rerun the test.
 359     test_out = env.CommandOutput(test_out_name, test_program)
 360
 361     # Running the test requires the test and its libs copied to the tests dir
 362     env.Depends(test_out, all_outputs)
 363     env.ComponentTestOutput('run_' + prog_name, test_out)
 364
 365     # Add target properties
 366     env.SetTargetProperty(prog_name, RUN_TARGET='run_' + prog_name)
 367
 368
 369 def ComponentTestProgram(self, prog_name, *args, **kwargs):
 370   """Pseudo-builder for test program to handle platform-dependent type.
 371
 372   Args:
 373     self: Environment in which we were called.
 374     prog_name: Test program name.
 375     args: Positional arguments.
 376     kwargs: Keyword arguments.
 377
 378   Returns:
 379     Output node list from env.Program().
 380   """
 381   # Clone and modify environment
 382   env = _ComponentPlatformSetup(self, 'ComponentTestProgram', **kwargs)
 383
 384   env['PROGRAM_BASENAME'] = prog_name
 385   env['PROGRAM_NAME'] = '$PROGPREFIX$PROGRAM_BASENAME$PROGSUFFIX'
 386
 387   # Call env.Program()
 388   out_nodes = env.Program(prog_name, *args, **kwargs)
 389
 390   # Add dependencies on includes
 391   env.Depends(out_nodes, env['INCLUDES'])
 392
 393   # Publish output
 394   env.Publish(prog_name, 'run', out_nodes[0])
 395   env.Publish(prog_name, 'debug', out_nodes[1:])
 396
 397   # Add an alias to build the program to the right groups
 398   a = env.Alias(prog_name, out_nodes)
 399   for group in env['COMPONENT_TEST_PROGRAM_GROUPS']:
 400     SCons.Script.Alias(group, a)
 401
 402   # Store list of components for this program
 403   env._StoreComponents(prog_name)
 404
 405   # Let component_targets know this target is available in the current mode
 406   env.SetTargetProperty(prog_name, TARGET_PATH=out_nodes[0])
 407
 408   # Set up deferred call to replicate resources and run test
 409   env.Defer(ComponentTestProgramDeferred)
 410
 411   # Return the output node
 412   return out_nodes
 413
 414 #------------------------------------------------------------------------------
 415
 416
 417 def ComponentProgramDeferred(env):
 418   """Deferred build steps for program.
 419
 420   Args:
 421     env: Environment from ComponentProgram().
 422
 423   Sets up the aliases to compile the program.
 424   """
 425   prog_name = env['PROGRAM_BASENAME']
 426
 427   # Install program and resources
 428   all_outputs = []
 429   components = _RetrieveComponents(prog_name)
 430   for resource, dest_dir in env.get('COMPONENT_PROGRAM_RESOURCES').items():
 431     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 432
 433   # Add installed program and resources to the alias
 434   env.Alias(prog_name, all_outputs)
 435
 436
 437 def ComponentProgram(self, prog_name, *args, **kwargs):
 438   """Pseudo-builder for program to handle platform-dependent type.
 439
 440   Args:
 441     self: Environment in which we were called.
 442     prog_name: Test program name.
 443     args: Positional arguments.
 444     kwargs: Keyword arguments.
 445
 446   Returns:
 447     Output node list from env.Program().
 448   """
 449   # Clone and modify environment
 450   env = _ComponentPlatformSetup(self, 'ComponentProgram', **kwargs)
 451
 452   env['PROGRAM_BASENAME'] = prog_name
 453
 454   # Call env.Program()
 455   out_nodes = env.Program(prog_name, *args, **kwargs)
 456
 457   # Add dependencies on includes
 458   env.Depends(out_nodes, env['INCLUDES'])
 459
 460   # Publish output
 461   env.Publish(prog_name, 'run', out_nodes[0])
 462   env.Publish(prog_name, 'debug', out_nodes[1:])
 463
 464   # Add an alias to build the program to the right groups
 465   a = env.Alias(prog_name, out_nodes)
 466   for group in env['COMPONENT_PROGRAM_GROUPS']:
 467     SCons.Script.Alias(group, a)
 468
 469   # Store list of components for this program
 470   env._StoreComponents(prog_name)
 471
 472   # Let component_targets know this target is available in the current mode
 473   env.SetTargetProperty(prog_name, TARGET_PATH=out_nodes[0])
 474
 475   # Set up deferred call to replicate resources
 476   env.Defer(ComponentProgramDeferred)
 477
 478   # Return the output nodes
 479   return out_nodes
 480
 481 #------------------------------------------------------------------------------
 482
 483
 484 def ComponentTestOutput(self, test_name, nodes, **kwargs):
 485   """Pseudo-builder for test output.
 486
 487   Args:
 488     self: Environment in which we were called.
 489     test_name: Test name.
 490     nodes: List of files/Nodes output by the test.
 491     kwargs: Keyword arguments.
 492
 493   Returns:
 494     Passthrough return code from env.Alias().
 495   """
 496
 497   # Clone and modify environment
 498   env = _ComponentPlatformSetup(self, 'ComponentTestObject', **kwargs)
 499
 500   # Add an alias for the test output
 501   a = env.Alias(test_name, nodes)
 502
 503   # Determine groups test belongs in
 504   if env.get('COMPONENT_TEST_ENABLED'):
 505     groups = env.SubstList2('$COMPONENT_TEST_OUTPUT_GROUPS')
 506     if env.get('COMPONENT_TEST_SIZE'):
 507       groups.append(env.subst('run_${COMPONENT_TEST_SIZE}_tests'))
 508   else:
 509     # Disabled tests only go in the explicit disabled tests group
 510     groups = ['run_disabled_tests']
 511
 512   for group in groups:
 513     SCons.Script.Alias(group, a)
 514
 515   # Let component_targets know this target is available in the current mode
 516   env.SetTargetProperty(test_name, TARGET_PATH=nodes[0])
 517
 518   # Return the output node
 519   return a
 520
 521 #------------------------------------------------------------------------------
 522
 523
 524 def generate(env):
 525   # NOTE: SCons requires the use of this name, which fails gpylint.
 526   """SCons entry point for this tool."""
 527
 528   env.Replace(
 529       LIB_DIR='$TARGET_ROOT/lib',
 530       # TODO: Remove legacy COMPONENT_LIBRARY_DIR, once all users have
 531       # transitioned to LIB_DIR
 532       COMPONENT_LIBRARY_DIR='$LIB_DIR',
 533       STAGING_DIR='$TARGET_ROOT/staging',
 534       TESTS_DIR='$TARGET_ROOT/tests',
 535       TEST_OUTPUT_DIR='$TARGET_ROOT/test_output',
 536       # Default command line for a test is just the name of the file.
 537       # TODO: Why doesn't the following work:
 538       # COMPONENT_TEST_CMDLINE='${SOURCE.abspath}',
 539       # (it generates a SCons error)
 540       COMPONENT_TEST_CMDLINE='${PROGRAM_NAME}',
 541       # Component tests are runnable by default.
 542       COMPONENT_TEST_RUNNABLE=True,
 543       # Default test size is large
 544       COMPONENT_TEST_SIZE='large',
 545       # Default timeouts for component tests
 546       COMPONENT_TEST_TIMEOUT={'large': 900, 'medium': 450, 'small': 180},
 547       # Tests are enabled by default
 548       COMPONENT_TEST_ENABLED=True,
 549       # Static linking is a sensible default
 550       COMPONENT_STATIC=True,
 551       # Don't publish libraries to the staging dir by themselves by default.
 552       COMPONENT_LIBRARY_PUBLISH=False,
 553   )
 554   env.Append(
 555       LIBPATH=['$LIB_DIR'],
 556       RPATH=['$LIB_DIR'],
 557
 558       # Default alias groups for component builders
 559       COMPONENT_PACKAGE_GROUPS=['all_packages'],
 560       COMPONENT_LIBRARY_GROUPS=['all_libraries'],
 561       COMPONENT_PROGRAM_GROUPS=['all_programs'],
 562       COMPONENT_TEST_PROGRAM_GROUPS=['all_test_programs'],
 563       COMPONENT_TEST_OUTPUT_GROUPS=['run_all_tests'],
 564
 565       # Additional components whose resources should be copied into program
 566       # directories, in addition to those from LIBS and the program itself.
 567       LIBS=[],
 568       COMPONENTS=[],
 569
 570       # Dicts of what resources should go in each destination directory for
 571       # programs and test programs.
 572       COMPONENT_PACKAGE_RESOURCES={
 573           'run': '$PACKAGE_DIR',
 574           'debug': '$PACKAGE_DIR',
 575       },
 576       COMPONENT_PROGRAM_RESOURCES={
 577           'run': '$STAGING_DIR',
 578           'debug': '$STAGING_DIR',
 579       },
 580       COMPONENT_TEST_RESOURCES={
 581           'run': '$TESTS_DIR',
 582           'debug': '$TESTS_DIR',
 583           'test_input': '$TESTS_DIR',
 584       },
 585   )
 586
 587   # Add command line option for retest
 588   SCons.Script.AddOption(
 589       '--retest',
 590       dest='component_test_retest',
 591       action='store_true',
 592       help='force all tests to rerun')
 593   SCons.Script.Help('  --retest                    '
 594                     'Rerun specified tests, ignoring cached results.\n')
 595
 596   # Defer per-environment initialization, but do before building SConscripts
 597   env.Defer(_InitializeComponentBuilders)
 598   env.Defer('BuildEnvironmentSConscripts', after=_InitializeComponentBuilders)
 599
 600   # Add our pseudo-builder methods
 601   env.AddMethod(_StoreComponents)
 602   env.AddMethod(ComponentPackage)
 603   env.AddMethod(ComponentObject)
 604   env.AddMethod(ComponentLibrary)
 605   env.AddMethod(ComponentProgram)
 606   env.AddMethod(ComponentTestProgram)
 607   env.AddMethod(ComponentTestOutput)
 608
 609   # Add our target groups
 610   AddTargetGroup('all_libraries', 'libraries can be built')
 611   AddTargetGroup('all_programs', 'programs can be built')
 612   AddTargetGroup('all_test_programs', 'tests can be built')
 613   AddTargetGroup('all_packages', 'packages can be built')
 614   AddTargetGroup('run_all_tests', 'tests can be run')
 615   AddTargetGroup('run_disabled_tests', 'tests are disabled')
 616   AddTargetGroup('run_small_tests', 'small tests can be run')
 617   AddTargetGroup('run_medium_tests', 'medium tests can be run')
 618   AddTargetGroup('run_large_tests', 'large tests can be run')
 619