tools/export_tarball/export_tarball.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """
   7 This tool creates a tarball with all the sources, but without .svn directories.
   8
   9 It can also remove files which are not strictly required for build, so that
  10 the resulting tarball can be reasonably small (last time it was ~110 MB).
  11
  12 Example usage:
  13
  14 export_tarball.py /foo/bar
  15
  16 The above will create file /foo/bar.tar.bz2.
  17 """
  18
  19 import optparse
  20 import os
  21 import subprocess
  22 import sys
  23 import tarfile
  24
  25
  26 NONESSENTIAL_DIRS = (
  27     'breakpad/src/processor/testdata',
  28     'chrome/browser/resources/tracing/tests',
  29     'chrome/common/extensions/docs',
  30     'chrome/tools/test/reference_build',
  31     'courgette/testdata',
  32     'data',
  33     'native_client/src/trusted/service_runtime/testdata',
  34     'src/chrome/test/data',
  35     'o3d/documentation',
  36     'o3d/samples',
  37     'o3d/tests',
  38     'ppapi/examples',
  39     'ppapi/native_client/tests',
  40     'third_party/angle/samples/gles2_book',
  41     'third_party/findbugs',
  42     'third_party/hunspell_dictionaries',
  43     'third_party/hunspell/tests',
  44     'third_party/lighttpd',
  45     'third_party/sqlite/src/test',
  46     'third_party/sqlite/test',
  47     'third_party/vc_80',
  48     'third_party/xdg-utils/tests',
  49     'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
  50     'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
  51     'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
  52     'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
  53     'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
  54     'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
  55     'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
  56     'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
  57     'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
  58     'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
  59     'third_party/WebKit/LayoutTests',
  60     'third_party/WebKit/Source/JavaScriptCore/tests',
  61     'third_party/WebKit/Source/WebCore/ChangeLog',
  62     'third_party/WebKit/Source/WebKit2',
  63     'third_party/WebKit/Tools/Scripts',
  64     'tools/gyp/test',
  65     'v8/test',
  66     'webkit/data/layout_tests',
  67     'webkit/tools/test/reference_build',
  68 )
  69
  70 TESTDIRS = (
  71     'chrome/test/data',
  72     'content/test/data',
  73     'media/test/data',
  74     'net/data',
  75 )
  76
  77 PRUNEDDIRS = (
  78     'courgette',
  79 )
  80
  81
  82 def GetSourceDirectory():
  83   return os.path.realpath(
  84     os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src'))
  85
  86
  87 # Workaround lack of the exclude parameter in add method in python-2.4.
  88 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
  89 class MyTarFile(tarfile.TarFile):
  90   def set_remove_nonessential_files(self, remove):
  91     self.__remove_nonessential_files = remove
  92
  93   def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
  94     head, tail = os.path.split(name)
  95     if tail in ('.svn', '.git'):
  96       return
  97
  98     if self.__remove_nonessential_files:
  99       # WebKit change logs take quite a lot of space. This saves ~10 MB
 100       # in a bzip2-compressed tarball.
 101       if 'ChangeLog' in name:
 102         return
 103
 104       # Remove contents of non-essential directories, but preserve gyp files,
 105       # so that build/gyp_chromium can work.
 106       for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS + PRUNEDDIRS):
 107         dir_path = os.path.join(GetSourceDirectory(), nonessential_dir)
 108         if (name.startswith(dir_path) and
 109             os.path.isfile(name) and
 110             'gyp' not in name):
 111           return
 112
 113     tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
 114
 115
 116 def main(argv):
 117   parser = optparse.OptionParser()
 118   parser.add_option("--basename")
 119   parser.add_option("--remove-nonessential-files",
 120                     dest="remove_nonessential_files",
 121                     action="store_true", default=False)
 122   parser.add_option("--test-data", action="store_true")
 123   # TODO(phajdan.jr): Remove --xz option when it's not needed for compatibility.
 124   parser.add_option("--xz", action="store_true")
 125
 126   options, args = parser.parse_args(argv)
 127
 128   if len(args) != 1:
 129     print 'You must provide only one argument: output file name'
 130     print '(without .tar.xz extension).'
 131     return 1
 132
 133   if not os.path.exists(GetSourceDirectory()):
 134     print 'Cannot find the src directory ' + GetSourceDirectory()
 135     return 1
 136
 137   # These two commands are from src/DEPS; please keep them in sync.
 138   if subprocess.call(['python', 'build/util/lastchange.py', '-o',
 139                       'build/util/LASTCHANGE'], cwd=GetSourceDirectory()) != 0:
 140     print 'Could not run build/util/lastchange.py to update LASTCHANGE.'
 141     return 1
 142   if subprocess.call(['python', 'build/util/lastchange.py', '-s',
 143                       'third_party/WebKit', '-o',
 144                       'build/util/LASTCHANGE.blink'],
 145                      cwd=GetSourceDirectory()) != 0:
 146     print 'Could not run build/util/lastchange.py to update LASTCHANGE.blink.'
 147     return 1
 148
 149   output_fullname = args[0] + '.tar'
 150   output_basename = options.basename or os.path.basename(args[0])
 151
 152   archive = MyTarFile.open(output_fullname, 'w')
 153   archive.set_remove_nonessential_files(options.remove_nonessential_files)
 154   try:
 155     if options.test_data:
 156       for directory in TESTDIRS:
 157         archive.add(os.path.join(GetSourceDirectory(), directory),
 158                     arcname=os.path.join(output_basename, directory))
 159     else:
 160       archive.add(GetSourceDirectory(), arcname=output_basename)
 161   finally:
 162     archive.close()
 163
 164   if subprocess.call(['xz', '-9', output_fullname]) != 0:
 165     print 'xz -9 failed!'
 166     return 1
 167
 168   return 0
 169
 170
 171 if __name__ == "__main__":
 172   sys.exit(main(sys.argv[1:]))