site_scons/site_tools/component_builders.py

   1 #!/usr/bin/python2.4
   2 # Copyright 2008, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Software construction toolkit builders for SCons."""
  32
  33
  34 import SCons
  35
  36
  37 __component_list = {}
  38
  39
  40 def _InitializeComponentBuilders(env):
  41   """Re-initializes component builders module.
  42
  43   Args:
  44     env: Environment context
  45   """
  46   env = env     # Silence gpylint
  47
  48   __component_list.clear()
  49
  50
  51 def _RetrieveComponents(component_name, filter_components=None):
  52   """Get the list of all components required by the specified component.
  53
  54   Args:
  55     component_name: Name of the base component.
  56     filter_components: List of components NOT to include.
  57
  58   Returns:
  59     A list of the transitive closure of all components required by the base
  60     component.  That is, if A requires B and B requires C, this returns [B, C].
  61
  62   """
  63   if filter_components:
  64     filter_components = set(filter_components)
  65   else:
  66     filter_components = set()
  67
  68   components = set([component_name])    # Components always require themselves
  69   new_components = set(components)
  70   while new_components:
  71     # Take next new component and add it to the list we've already scanned.
  72     c = new_components.pop()
  73     components.add(c)
  74     # Add to the list of new components any of c's components that we haven't
  75     # seen before.
  76     new_components.update(__component_list.get(c, set())
  77                           - components - filter_components)
  78
  79   return list(components)
  80
  81
  82 def _StoreComponents(self, component_name):
  83   """Stores the list of child components for the specified component.
  84
  85   Args:
  86     self: Environment containing component.
  87     component_name: Name of the component.
  88
  89   Adds component references based on the LIBS and COMPONENTS variables in the
  90   current environment.  Should be called at primary SConscript execution time;
  91   use _RetrieveComponents() to get the final components lists in a Defer()'d
  92   function.
  93   """
  94
  95   components = set()
  96   for clist in ('LIBS', 'COMPONENTS'):
  97     components.update(map(self.subst, self.Flatten(self[clist])))
  98
  99   if component_name not in __component_list:
 100     __component_list[component_name] = set()
 101   __component_list[component_name].update(components)
 102
 103
 104 def _ComponentPlatformSetup(env, builder_name, **kwargs):
 105   """Modify an environment to work with a component builder.
 106
 107   Args:
 108     env: Environment to clone.
 109     builder_name: Name of the builder.
 110     kwargs: Keyword arguments.
 111
 112   Returns:
 113     A modified clone of the environment.
 114   """
 115   # Clone environment so we can modify it
 116   env = env.Clone()
 117
 118   # Add all keyword arguments to the environment
 119   for k, v in kwargs.items():
 120     env[k] = v
 121
 122   # Add compiler flags for included headers, if any
 123   env['INCLUDES'] = env.Flatten(env.subst_list(['$INCLUDES']))
 124   for h in env['INCLUDES']:
 125     env.Append(CCFLAGS = ['${CCFLAG_INCLUDE}%s' % h])
 126
 127   # Call platform-specific component setup function, if any
 128   if env.get('COMPONENT_PLATFORM_SETUP'):
 129     env['COMPONENT_PLATFORM_SETUP'](env, builder_name)
 130
 131   # Return the modified environment
 132   return env
 133
 134 #------------------------------------------------------------------------------
 135
 136 # TODO: Should be possible to refactor programs, test programs,
 137 # libs to all publish as packages, for simplicity and code reuse.
 138
 139
 140 def ComponentPackageDeferred(env):
 141   """Deferred build steps for component package.
 142
 143   Args:
 144     env: Environment from ComponentPackage().
 145
 146   Sets up the aliases to build the package.
 147   """
 148   package_name = env['PACKAGE_NAME']
 149
 150   # Install program and resources
 151   all_outputs = []
 152   filter = env.Flatten(env.subst_list('$COMPONENT_PACKAGE_FILTER'))
 153   components = _RetrieveComponents(package_name, filter)
 154   for resource, dest_dir in env.get('COMPONENT_PACKAGE_RESOURCES').items():
 155     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 156
 157   # Add installed program and resources to the alias
 158   env.Alias(package_name, all_outputs)
 159
 160
 161 def ComponentPackage(self, package_name, dest_dir, **kwargs):
 162   """Pseudo-builder for package containing other components.
 163
 164   Args:
 165     self: Environment in which we were called.
 166     package_name: Name of package.
 167     dest_dir: Destination directory for package.
 168     args: Positional arguments.
 169     kwargs: Keyword arguments.
 170
 171   Returns:
 172     The alias node for the package.
 173   """
 174   # Clone and modify environment
 175   env = _ComponentPlatformSetup(self, 'ComponentPackage', **kwargs)
 176
 177   env.Replace(
 178       PACKAGE_NAME=package_name,
 179       PACKAGE_DIR=dest_dir,
 180   )
 181
 182   # Add an empty alias for the package and add it to the right groups
 183   a = env.Alias(package_name, [])
 184   for group in env['COMPONENT_PACKAGE_GROUPS']:
 185     SCons.Script.Alias(group, a)
 186
 187   # Store list of components for this program
 188   env._StoreComponents(package_name)
 189
 190   # Let component_targets know this target is available in the current mode
 191   env.SetTargetProperty(package_name, TARGET_PATH=dest_dir)
 192
 193   # Set up deferred call to replicate resources
 194   env.Defer(ComponentPackageDeferred)
 195
 196   # Return the alias, since it's the only node we have
 197   return a
 198
 199 #------------------------------------------------------------------------------
 200
 201
 202 def ComponentObject(self, *args, **kwargs):
 203   """Pseudo-builder for object to handle platform-dependent type.
 204
 205   Args:
 206     self: Environment in which we were called.
 207     args: Positional arguments.
 208     kwargs: Keyword arguments.
 209
 210   Returns:
 211     Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
 212
 213   TODO: Perhaps this should be a generator builder, so it can take
 214   a list of inputs and return a list of outputs?
 215   """
 216   # Clone and modify environment
 217   env = _ComponentPlatformSetup(self, 'ComponentObject', **kwargs)
 218
 219   # Make appropriate object type
 220   if env.get('COMPONENT_STATIC'):
 221     o = env.StaticObject(*args, **kwargs)
 222   else:
 223     o = env.SharedObject(*args, **kwargs)
 224
 225   # Add dependencies on includes
 226   env.Depends(o, env['INCLUDES'])
 227
 228   return o
 229
 230 #------------------------------------------------------------------------------
 231
 232
 233 def ComponentLibrary(self, lib_name, *args, **kwargs):
 234   """Pseudo-builder for library to handle platform-dependent type.
 235
 236   Args:
 237     self: Environment in which we were called.
 238     lib_name: Library name.
 239     args: Positional arguments.
 240     kwargs: Keyword arguments.
 241
 242   Returns:
 243     Passthrough return code from env.StaticLibrary() or env.SharedLibrary().
 244   """
 245   # Clone and modify environment
 246   env = _ComponentPlatformSetup(self, 'ComponentLibrary', **kwargs)
 247
 248   # Make appropriate library type
 249   if env.get('COMPONENT_STATIC'):
 250     lib_outputs = env.StaticLibrary(lib_name, *args, **kwargs)
 251   else:
 252     lib_outputs = env.SharedLibrary(lib_name, *args, **kwargs)
 253
 254   # Add dependencies on includes
 255   env.Depends(lib_outputs, env['INCLUDES'])
 256
 257   # Scan library outputs for files we need to link against this library, and
 258   # files we need to run executables linked against this library.
 259   need_for_link = []
 260   need_for_debug = []
 261   need_for_run = []
 262   for o in lib_outputs:
 263     if o.suffix in env['COMPONENT_LIBRARY_LINK_SUFFIXES']:
 264       need_for_link.append(o)
 265     if o.suffix in env['COMPONENT_LIBRARY_DEBUG_SUFFIXES']:
 266       need_for_debug.append(o)
 267     if o.suffix == env['SHLIBSUFFIX']:
 268       need_for_run.append(o)
 269   all_outputs = lib_outputs
 270
 271   # Install library in intermediate directory, so other libs and programs can
 272   # link against it
 273   all_outputs += env.Replicate('$LIB_DIR', need_for_link)
 274
 275   # Publish output
 276   env.Publish(lib_name, 'run', need_for_run)
 277   env.Publish(lib_name, 'debug', need_for_debug)
 278
 279   # Add an alias to build and copy the library, and add it to the right groups
 280   a = self.Alias(lib_name, all_outputs)
 281   for group in env['COMPONENT_LIBRARY_GROUPS']:
 282     SCons.Script.Alias(group, a)
 283
 284   # Store list of components for this library
 285   env._StoreComponents(lib_name)
 286
 287   # Let component_targets know this target is available in the current mode.
 288   env.SetTargetProperty(lib_name, TARGET_PATH=lib_outputs[0])
 289
 290   # If library should publish itself, publish as if it was a program
 291   if env.get('COMPONENT_LIBRARY_PUBLISH'):
 292     env['PROGRAM_BASENAME'] = lib_name
 293     env.Defer(ComponentProgramDeferred)
 294
 295   # Return the library outputs
 296   return lib_outputs
 297
 298 #------------------------------------------------------------------------------
 299
 300
 301 def ComponentTestProgramDeferred(env):
 302   """Deferred build steps for test program.
 303
 304   Args:
 305     env: Environment from ComponentTestProgram().
 306
 307   Sets up the aliases to compile and run the test program.
 308   """
 309   prog_name = env['PROGRAM_BASENAME']
 310
 311   # Install program and resources
 312   all_outputs = []
 313   components = _RetrieveComponents(prog_name)
 314   for resource, dest_dir in env.get('COMPONENT_TEST_RESOURCES').items():
 315     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 316
 317   # Add installed program and resources to the alias
 318   env.Alias(prog_name, all_outputs)
 319
 320   # Add target properties
 321   env.SetTargetProperty(
 322       prog_name,
 323       # The copy of the program we care about is the one in the tests dir
 324       EXE='$TESTS_DIR/$PROGRAM_NAME',
 325       RUN_CMDLINE='$COMPONENT_TEST_CMDLINE',
 326       RUN_DIR='$TESTS_DIR',
 327       TARGET_PATH='$TESTS_DIR/$PROGRAM_NAME',
 328   )
 329
 330   # Add an alias for running the test in the test directory, if the test is
 331   # runnable and has a test command line.
 332   if env.get('COMPONENT_TEST_RUNNABLE') and env.get('COMPONENT_TEST_CMDLINE'):
 333     env.Replace(
 334         COMMAND_OUTPUT_CMDLINE=env['COMPONENT_TEST_CMDLINE'],
 335         COMMAND_OUTPUT_RUN_DIR='$TESTS_DIR',
 336     )
 337     test_out_name = '$TEST_OUTPUT_DIR/${PROGRAM_BASENAME}.out.txt'
 338     if (env.GetOption('component_test_retest')
 339         and env.File(test_out_name).exists()):
 340       # Delete old test results, so test will rerun.
 341       env.Execute(SCons.Script.Delete(test_out_name))
 342
 343     # Set timeout based on test size
 344     timeout = env.get('COMPONENT_TEST_TIMEOUT')
 345     if type(timeout) is dict:
 346       timeout = timeout.get(env.get('COMPONENT_TEST_SIZE'))
 347     if timeout:
 348       env['COMMAND_OUTPUT_TIMEOUT'] = timeout
 349
 350     # Test program is the first run resource we replicated.  (Duplicate
 351     # replicate is not harmful, and is a handy way to pick out the correct
 352     # file from all those we replicated above.)
 353     test_program = env.ReplicatePublished('$TESTS_DIR', prog_name, 'run')
 354
 355     # Run the test.  Note that we need to refer to the file by name, so that
 356     # SCons will recreate the file node after we've deleted it; if we used the
 357     # env.File() we created in the if statement above, SCons would still think
 358     # it exists and not rerun the test.
 359     test_out = env.CommandOutput(test_out_name, test_program)
 360
 361     # Running the test requires the test and its libs copied to the tests dir
 362     env.Depends(test_out, all_outputs)
 363     env.ComponentTestOutput('run_' + prog_name, test_out)
 364
 365     # Add target properties
 366     env.SetTargetProperty(prog_name, RUN_TARGET='run_' + prog_name)
 367
 368 def ComponentTestProgram(self, prog_name, *args, **kwargs):
 369   """Pseudo-builder for test program to handle platform-dependent type.
 370
 371   Args:
 372     self: Environment in which we were called.
 373     prog_name: Test program name.
 374     args: Positional arguments.
 375     kwargs: Keyword arguments.
 376
 377   Returns:
 378     Output node list from env.Program().
 379
 380   TODO: Should have some sort of support for S/M/L categorization
 381   """
 382   # Clone and modify environment
 383   env = _ComponentPlatformSetup(self, 'ComponentTestProgram', **kwargs)
 384
 385   env['PROGRAM_BASENAME'] = prog_name
 386   env['PROGRAM_NAME'] = '$PROGPREFIX$PROGRAM_BASENAME$PROGSUFFIX'
 387
 388   # Call env.Program()
 389   out_nodes = env.Program(prog_name, *args, **kwargs)
 390
 391   # Add dependencies on includes
 392   env.Depends(out_nodes, env['INCLUDES'])
 393
 394   # Publish output
 395   env.Publish(prog_name, 'run', out_nodes[0])
 396   env.Publish(prog_name, 'debug', out_nodes[1:])
 397
 398   # Add an alias to build the program to the right groups
 399   a = env.Alias(prog_name, out_nodes)
 400   for group in env['COMPONENT_TEST_PROGRAM_GROUPS']:
 401     SCons.Script.Alias(group, a)
 402
 403   # Store list of components for this program
 404   env._StoreComponents(prog_name)
 405
 406   # Let component_targets know this target is available in the current mode
 407   env.SetTargetProperty(prog_name, TARGET_PATH=out_nodes[0])
 408
 409   # Set up deferred call to replicate resources and run test
 410   env.Defer(ComponentTestProgramDeferred)
 411
 412   # Return the output node
 413   return out_nodes
 414
 415 #------------------------------------------------------------------------------
 416
 417
 418 def ComponentProgramDeferred(env):
 419   """Deferred build steps for program.
 420
 421   Args:
 422     env: Environment from ComponentProgram().
 423
 424   Sets up the aliases to compile the program.
 425   """
 426   prog_name = env['PROGRAM_BASENAME']
 427
 428   # Install program and resources
 429   all_outputs = []
 430   components = _RetrieveComponents(prog_name)
 431   for resource, dest_dir in env.get('COMPONENT_PROGRAM_RESOURCES').items():
 432     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 433
 434   # Add installed program and resources to the alias
 435   env.Alias(prog_name, all_outputs)
 436
 437
 438 def ComponentProgram(self, prog_name, *args, **kwargs):
 439   """Pseudo-builder for program to handle platform-dependent type.
 440
 441   Args:
 442     self: Environment in which we were called.
 443     prog_name: Test program name.
 444     args: Positional arguments.
 445     kwargs: Keyword arguments.
 446
 447   Returns:
 448     Output node list from env.Program().
 449   """
 450   # Clone and modify environment
 451   env = _ComponentPlatformSetup(self, 'ComponentProgram', **kwargs)
 452
 453   env['PROGRAM_BASENAME'] = prog_name
 454
 455   # Call env.Program()
 456   out_nodes = env.Program(prog_name, *args, **kwargs)
 457
 458   # Add dependencies on includes
 459   env.Depends(out_nodes, env['INCLUDES'])
 460
 461   # Publish output
 462   env.Publish(prog_name, 'run', out_nodes[0])
 463   env.Publish(prog_name, 'debug', out_nodes[1:])
 464
 465   # Add an alias to build the program to the right groups
 466   a = env.Alias(prog_name, out_nodes)
 467   for group in env['COMPONENT_PROGRAM_GROUPS']:
 468     SCons.Script.Alias(group, a)
 469
 470   # Store list of components for this program
 471   env._StoreComponents(prog_name)
 472
 473   # Let component_targets know this target is available in the current mode
 474   env.SetTargetProperty(prog_name)
 475
 476   # Set up deferred call to replicate resources
 477   env.Defer(ComponentProgramDeferred)
 478
 479   # Return the output nodes
 480   return out_nodes
 481
 482 #------------------------------------------------------------------------------
 483
 484
 485 def ComponentTestOutput(self, test_name, nodes):
 486   """Pseudo-builder for test output.
 487
 488   Args:
 489     self: Environment in which we were called.
 490     test_name: Test name.
 491     nodes: List of files/Nodes output by the test.
 492
 493   Returns:
 494     Passthrough return code from env.Alias().
 495   """
 496
 497   # Add an alias for the test output
 498   a = self.Alias(test_name, nodes)
 499
 500   groups = self.get('COMPONENT_TEST_OUTPUT_GROUPS')
 501   if not groups:
 502     # Output group not explicitly specified, so automatically add to groups
 503     if self.get('COMPONENT_TEST_ENABLED'):
 504       # Enabled tests go in all tests, and their size category
 505       groups = ['run_all_tests']
 506       if self.get('COMPONENT_TEST_SIZE'):
 507         groups.append(self.subst('run_${COMPONENT_TEST_SIZE}_tests'))
 508     else:
 509       # Disabled tests only go in their group
 510       groups = ['run_disabled_tests']
 511
 512   for group in groups:
 513     SCons.Script.Alias(group, a)
 514
 515   # Let component_targets know this target is available in the current mode
 516   self.SetTargetProperty(test_name, TARGET_PATH=nodes[0])
 517
 518   # Return the output node
 519   return a
 520
 521 #------------------------------------------------------------------------------
 522
 523
 524 def generate(env):
 525   # NOTE: SCons requires the use of this name, which fails gpylint.
 526   """SCons entry point for this tool."""
 527
 528   env.Replace(
 529       LIB_DIR='$TARGET_ROOT/lib',
 530       # TODO: Remove legacy COMPONENT_LIBRARY_DIR, once all users
 531       # have transitioned to LIB_DIR
 532       COMPONENT_LIBRARY_DIR='$LIB_DIR',
 533       STAGING_DIR='$TARGET_ROOT/staging',
 534       TESTS_DIR='$TARGET_ROOT/tests',
 535       TEST_OUTPUT_DIR='$TARGET_ROOT/test_output',
 536       # Default command line for a test is just the name of the file.
 537       # TODO: Why doesn't the following work:
 538       # COMPONENT_TEST_CMDLINE='${SOURCE.abspath}',
 539       # (it generates a SCons error)
 540       COMPONENT_TEST_CMDLINE='${PROGRAM_NAME}',
 541       # Component tests are runnable by default.
 542       COMPONENT_TEST_RUNNABLE=True,
 543       # Default test size is large
 544       COMPONENT_TEST_SIZE='large',
 545       # Default timeouts for component tests
 546       COMPONENT_TEST_TIMEOUT={'large': 900, 'medium': 450, 'small': 180},
 547       # Tests are enabled by default
 548       COMPONENT_TEST_ENABLED=True,
 549       # Static linking is a sensible default
 550       COMPONENT_STATIC=True,
 551       # Don't publish libraries to the staging dir by themselves by default.
 552       COMPONENT_LIBRARY_PUBLISH=False,
 553   )
 554   env.Append(
 555       LIBPATH=['$LIB_DIR'],
 556       RPATH=['$LIB_DIR'],
 557
 558       # Default alias groups for component builders
 559       COMPONENT_PACKAGE_GROUPS=['all_packages'],
 560       COMPONENT_LIBRARY_GROUPS=['all_libraries'],
 561       COMPONENT_PROGRAM_GROUPS=['all_programs'],
 562       COMPONENT_TEST_PROGRAM_GROUPS=['all_test_programs'],
 563
 564       # Additional components whose resources should be copied into program
 565       # directories, in addition to those from LIBS and the program itself.
 566       LIBS=[],
 567       COMPONENTS=[],
 568
 569       # Dicts of what resources should go in each destination directory for
 570       # programs and test programs.
 571       COMPONENT_PACKAGE_RESOURCES={
 572           'run': '$PACKAGE_DIR',
 573           'debug': '$PACKAGE_DIR',
 574       },
 575       COMPONENT_PROGRAM_RESOURCES={
 576           'run': '$STAGING_DIR',
 577           'debug': '$STAGING_DIR',
 578       },
 579       COMPONENT_TEST_RESOURCES={
 580           'run': '$TESTS_DIR',
 581           'debug': '$TESTS_DIR',
 582           'test_input': '$TESTS_DIR',
 583       },
 584   )
 585
 586   # Add command line option for retest
 587   SCons.Script.AddOption(
 588       '--retest',
 589       dest='component_test_retest',
 590       action='store_true',
 591       help='force all tests to rerun')
 592   SCons.Script.Help('  --retest                    '
 593                     'Rerun specified tests, ignoring cached results.\n')
 594
 595   # Defer per-environment initialization, but do before building SConscripts
 596   env.Defer(_InitializeComponentBuilders)
 597   env.Defer('BuildEnvironmentSConscripts', after=_InitializeComponentBuilders)
 598
 599   # Add our pseudo-builder methods
 600   env.AddMethod(_StoreComponents)
 601   env.AddMethod(ComponentPackage)
 602   env.AddMethod(ComponentObject)
 603   env.AddMethod(ComponentLibrary)
 604   env.AddMethod(ComponentProgram)
 605   env.AddMethod(ComponentTestProgram)
 606   env.AddMethod(ComponentTestOutput)
 607
 608   # Add our target groups
 609   AddTargetGroup('all_libraries', 'libraries can be built')
 610   AddTargetGroup('all_programs', 'programs can be built')
 611   AddTargetGroup('all_test_programs', 'tests can be built')
 612   AddTargetGroup('all_packages', 'packages can be built')
 613   AddTargetGroup('run_all_tests', 'tests can be run')
 614   AddTargetGroup('run_disabled_tests', 'tests are disabled')
 615   AddTargetGroup('run_small_tests', 'small tests can be run')
 616   AddTargetGroup('run_medium_tests', 'medium tests can be run')
 617   AddTargetGroup('run_large_tests', 'large tests can be run')
 618