build/android/gyp/process_resources.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Process Android resources to generate R.java, and prepare for packaging.
   8
   9 This will crunch images and generate v14 compatible resources
  10 (see generate_v14_compatible_resources.py).
  11 """
  12
  13 import codecs
  14 import collections
  15 import optparse
  16 import os
  17 import re
  18 import shutil
  19 import sys
  20
  21 import generate_v14_compatible_resources
  22
  23 from util import build_utils
  24
  25 # Import jinja2 from third_party/jinja2
  26 sys.path.insert(1,
  27     os.path.join(os.path.dirname(__file__), '../../../third_party'))
  28 from jinja2 import Template # pylint: disable=F0401
  29
  30
  31 # Represents a line from a R.txt file.
  32 TextSymbolsEntry = collections.namedtuple('RTextEntry',
  33     ('java_type', 'resource_type', 'name', 'value'))
  34
  35
  36 def ParseArgs(args):
  37   """Parses command line options.
  38
  39   Returns:
  40     An options object as from optparse.OptionsParser.parse_args()
  41   """
  42   parser = optparse.OptionParser()
  43   build_utils.AddDepfileOption(parser)
  44
  45   parser.add_option('--android-sdk', help='path to the Android SDK folder')
  46   parser.add_option('--aapt-path',
  47                     help='path to the Android aapt tool')
  48   parser.add_option('--non-constant-id', action='store_true')
  49
  50   parser.add_option('--android-manifest', help='AndroidManifest.xml path')
  51   parser.add_option('--custom-package', help='Java package for R.java')
  52   parser.add_option(
  53       '--shared-resources',
  54       action='store_true',
  55       help='Make a resource package that can be loaded by a different'
  56       'application at runtime to access the package\'s resources.')
  57   parser.add_option(
  58       '--app-as-shared-lib',
  59       action='store_true',
  60       help='Make a resource package that can be loaded as shared library.')
  61
  62   parser.add_option('--resource-dirs',
  63                     help='Directories containing resources of this target.')
  64   parser.add_option('--dependencies-res-zips',
  65                     help='Resources from dependents.')
  66
  67   parser.add_option('--resource-zip-out',
  68                     help='Path for output zipped resources.')
  69
  70   parser.add_option('--R-dir',
  71                     help='directory to hold generated R.java.')
  72   parser.add_option('--srcjar-out',
  73                     help='Path to srcjar to contain generated R.java.')
  74   parser.add_option('--r-text-out',
  75                     help='Path to store the R.txt file generated by appt.')
  76
  77   parser.add_option('--proguard-file',
  78                     help='Path to proguard.txt generated file')
  79
  80   parser.add_option(
  81       '--v14-skip',
  82       action="store_true",
  83       help='Do not generate nor verify v14 resources')
  84
  85   parser.add_option(
  86       '--extra-res-packages',
  87       help='Additional package names to generate R.java files for')
  88   parser.add_option(
  89       '--extra-r-text-files',
  90       help='For each additional package, the R.txt file should contain a '
  91       'list of resources to be included in the R.java file in the format '
  92       'generated by aapt')
  93   parser.add_option(
  94       '--include-all-resources',
  95       action='store_true',
  96       help='Include every resource ID in every generated R.java file '
  97       '(ignoring R.txt).')
  98
  99   parser.add_option(
 100       '--all-resources-zip-out',
 101       help='Path for output of all resources. This includes resources in '
 102       'dependencies.')
 103
 104   parser.add_option('--stamp', help='File to touch on success')
 105
 106   (options, args) = parser.parse_args(args)
 107
 108   if args:
 109     parser.error('No positional arguments should be given.')
 110
 111   # Check that required options have been provided.
 112   required_options = (
 113       'android_sdk',
 114       'aapt_path',
 115       'android_manifest',
 116       'dependencies_res_zips',
 117       'resource_dirs',
 118       'resource_zip_out',
 119       )
 120   build_utils.CheckOptions(options, parser, required=required_options)
 121
 122   if (options.R_dir is None) == (options.srcjar_out is None):
 123     raise Exception('Exactly one of --R-dir or --srcjar-out must be specified.')
 124
 125   return options
 126
 127
 128 def CreateExtraRJavaFiles(
 129       r_dir, extra_packages, extra_r_text_files, shared_resources, include_all):
 130   if include_all:
 131     java_files = build_utils.FindInDirectory(r_dir, "R.java")
 132     if len(java_files) != 1:
 133       return
 134     r_java_file = java_files[0]
 135     r_java_contents = codecs.open(r_java_file, encoding='utf-8').read()
 136
 137     for package in extra_packages:
 138       package_r_java_dir = os.path.join(r_dir, *package.split('.'))
 139       build_utils.MakeDirectory(package_r_java_dir)
 140       package_r_java_path = os.path.join(package_r_java_dir, 'R.java')
 141       new_r_java = re.sub(r'package [.\w]*;', u'package %s;' % package,
 142                           r_java_contents)
 143       codecs.open(package_r_java_path, 'w', encoding='utf-8').write(new_r_java)
 144   else:
 145     if len(extra_packages) != len(extra_r_text_files):
 146       raise Exception('Need one R.txt file per extra package')
 147
 148     r_txt_file = os.path.join(r_dir, 'R.txt')
 149     if not os.path.exists(r_txt_file):
 150       return
 151
 152     # Map of (resource_type, name) -> Entry.
 153     # Contains the correct values for resources.
 154     all_resources = {}
 155     for entry in _ParseTextSymbolsFile(r_txt_file):
 156       all_resources[(entry.resource_type, entry.name)] = entry
 157
 158     # Map of package_name->resource_type->entry
 159     resources_by_package = (
 160         collections.defaultdict(lambda: collections.defaultdict(list)))
 161     # Build the R.java files using each package's R.txt file, but replacing
 162     # each entry's placeholder value with correct values from all_resources.
 163     for package, r_text_file in zip(extra_packages, extra_r_text_files):
 164       if not os.path.exists(r_text_file):
 165         continue
 166       if package in resources_by_package:
 167         raise Exception(('Package name "%s" appeared twice. All '
 168                          'android_resources() targets must use unique package '
 169                          'names, or no package name at all.') % package)
 170       resources_by_type = resources_by_package[package]
 171       # The sub-R.txt files have the wrong values at this point. Read them to
 172       # figure out which entries belong to them, but use the values from the
 173       # main R.txt file.
 174       for entry in _ParseTextSymbolsFile(r_text_file):
 175         entry = all_resources[(entry.resource_type, entry.name)]
 176         resources_by_type[entry.resource_type].append(entry)
 177
 178     for package, resources_by_type in resources_by_package.iteritems():
 179       package_r_java_dir = os.path.join(r_dir, *package.split('.'))
 180       build_utils.MakeDirectory(package_r_java_dir)
 181       package_r_java_path = os.path.join(package_r_java_dir, 'R.java')
 182       java_file_contents = _CreateExtraRJavaFile(
 183           package, resources_by_type, shared_resources)
 184       with open(package_r_java_path, 'w') as f:
 185         f.write(java_file_contents)
 186
 187
 188 def _ParseTextSymbolsFile(path):
 189   """Given an R.txt file, returns a list of TextSymbolsEntry."""
 190   ret = []
 191   with open(path) as f:
 192     for line in f:
 193       m = re.match(r'(int(?:\[\])?) (\w+) (\w+) (.+)$', line)
 194       if not m:
 195         raise Exception('Unexpected line in R.txt: %s' % line)
 196       java_type, resource_type, name, value = m.groups()
 197       ret.append(TextSymbolsEntry(java_type, resource_type, name, value))
 198   return ret
 199
 200
 201 def _CreateExtraRJavaFile(package, resources_by_type, shared_resources):
 202   """Generates the contents of a R.java file."""
 203   template = Template("""/* AUTO-GENERATED FILE.  DO NOT MODIFY. */
 204
 205 package {{ package }};
 206
 207 public final class R {
 208     {% for resource_type in resources %}
 209     public static final class {{ resource_type }} {
 210         {% for e in resources[resource_type] %}
 211         {% if shared_resources %}
 212         public static {{ e.java_type }} {{ e.name }} = {{ e.value }};
 213         {% else %}
 214         public static final {{ e.java_type }} {{ e.name }} = {{ e.value }};
 215         {% endif %}
 216         {% endfor %}
 217     }
 218     {% endfor %}
 219     {% if shared_resources %}
 220     public static void onResourcesLoaded(int packageId) {
 221         {% for resource_type in resources %}
 222         {% for e in resources[resource_type] %}
 223         {% if e.java_type == 'int[]' %}
 224         for(int i = 0; i < {{ e.resource_type }}.{{ e.name }}.length; ++i) {
 225             {{ e.resource_type }}.{{ e.name }}[i] =
 226                     ({{ e.resource_type }}.{{ e.name }}[i] & 0x00ffffff)
 227                     | (packageId << 24);
 228         }
 229         {% else %}
 230         {{ e.resource_type }}.{{ e.name }} =
 231                 ({{ e.resource_type }}.{{ e.name }} & 0x00ffffff)
 232                 | (packageId << 24);
 233         {% endif %}
 234         {% endfor %}
 235         {% endfor %}
 236     }
 237     {% endif %}
 238 }
 239 """, trim_blocks=True, lstrip_blocks=True)
 240
 241   return template.render(package=package, resources=resources_by_type,
 242                          shared_resources=shared_resources)
 243
 244
 245 def CrunchDirectory(aapt, input_dir, output_dir):
 246   """Crunches the images in input_dir and its subdirectories into output_dir.
 247
 248   If an image is already optimized, crunching often increases image size. In
 249   this case, the crunched image is overwritten with the original image.
 250   """
 251   aapt_cmd = [aapt,
 252               'crunch',
 253               '-C', output_dir,
 254               '-S', input_dir,
 255               '--ignore-assets', build_utils.AAPT_IGNORE_PATTERN]
 256   build_utils.CheckOutput(aapt_cmd, stderr_filter=FilterCrunchStderr,
 257                           fail_func=DidCrunchFail)
 258
 259   # Check for images whose size increased during crunching and replace them
 260   # with their originals (except for 9-patches, which must be crunched).
 261   for dir_, _, files in os.walk(output_dir):
 262     for crunched in files:
 263       if crunched.endswith('.9.png'):
 264         continue
 265       if not crunched.endswith('.png'):
 266         raise Exception('Unexpected file in crunched dir: ' + crunched)
 267       crunched = os.path.join(dir_, crunched)
 268       original = os.path.join(input_dir, os.path.relpath(crunched, output_dir))
 269       original_size = os.path.getsize(original)
 270       crunched_size = os.path.getsize(crunched)
 271       if original_size < crunched_size:
 272         shutil.copyfile(original, crunched)
 273
 274
 275 def FilterCrunchStderr(stderr):
 276   """Filters out lines from aapt crunch's stderr that can safely be ignored."""
 277   filtered_lines = []
 278   for line in stderr.splitlines(True):
 279     # Ignore this libpng warning, which is a known non-error condition.
 280     # http://crbug.com/364355
 281     if ('libpng warning: iCCP: Not recognizing known sRGB profile that has '
 282         + 'been edited' in line):
 283       continue
 284     filtered_lines.append(line)
 285   return ''.join(filtered_lines)
 286
 287
 288 def DidCrunchFail(returncode, stderr):
 289   """Determines whether aapt crunch failed from its return code and output.
 290
 291   Because aapt's return code cannot be trusted, any output to stderr is
 292   an indication that aapt has failed (http://crbug.com/314885).
 293   """
 294   return returncode != 0 or stderr
 295
 296
 297 def ZipResources(resource_dirs, zip_path):
 298   # Python zipfile does not provide a way to replace a file (it just writes
 299   # another file with the same name). So, first collect all the files to put
 300   # in the zip (with proper overriding), and then zip them.
 301   files_to_zip = dict()
 302   for d in resource_dirs:
 303     for root, _, files in os.walk(d):
 304       for f in files:
 305         archive_path = f
 306         parent_dir = os.path.relpath(root, d)
 307         if parent_dir != '.':
 308           archive_path = os.path.join(parent_dir, f)
 309         path = os.path.join(root, f)
 310         files_to_zip[archive_path] = path
 311   build_utils.DoZip(files_to_zip.iteritems(), zip_path)
 312
 313
 314 def CombineZips(zip_files, output_path):
 315   # When packaging resources, if the top-level directories in the zip file are
 316   # of the form 0, 1, ..., then each subdirectory will be passed to aapt as a
 317   # resources directory. While some resources just clobber others (image files,
 318   # etc), other resources (particularly .xml files) need to be more
 319   # intelligently merged. That merging is left up to aapt.
 320   def path_transform(name, src_zip):
 321     return '%d/%s' % (zip_files.index(src_zip), name)
 322
 323   build_utils.MergeZips(output_path, zip_files, path_transform=path_transform)
 324
 325
 326 def main():
 327   args = build_utils.ExpandFileArgs(sys.argv[1:])
 328
 329   options = ParseArgs(args)
 330   android_jar = os.path.join(options.android_sdk, 'android.jar')
 331   aapt = options.aapt_path
 332
 333   input_files = []
 334
 335   with build_utils.TempDir() as temp_dir:
 336     deps_dir = os.path.join(temp_dir, 'deps')
 337     build_utils.MakeDirectory(deps_dir)
 338     v14_dir = os.path.join(temp_dir, 'v14')
 339     build_utils.MakeDirectory(v14_dir)
 340
 341     gen_dir = os.path.join(temp_dir, 'gen')
 342     build_utils.MakeDirectory(gen_dir)
 343
 344     input_resource_dirs = build_utils.ParseGypList(options.resource_dirs)
 345
 346     if not options.v14_skip:
 347       for resource_dir in input_resource_dirs:
 348         generate_v14_compatible_resources.GenerateV14Resources(
 349             resource_dir,
 350             v14_dir)
 351
 352     dep_zips = build_utils.ParseGypList(options.dependencies_res_zips)
 353     input_files += dep_zips
 354     dep_subdirs = []
 355     for z in dep_zips:
 356       subdir = os.path.join(deps_dir, os.path.basename(z))
 357       if os.path.exists(subdir):
 358         raise Exception('Resource zip name conflict: ' + os.path.basename(z))
 359       build_utils.ExtractAll(z, path=subdir)
 360       dep_subdirs.append(subdir)
 361
 362     # Generate R.java. This R.java contains non-final constants and is used only
 363     # while compiling the library jar (e.g. chromium_content.jar). When building
 364     # an apk, a new R.java file with the correct resource -> ID mappings will be
 365     # generated by merging the resources from all libraries and the main apk
 366     # project.
 367     package_command = [aapt,
 368                        'package',
 369                        '-m',
 370                        '-M', options.android_manifest,
 371                        '--auto-add-overlay',
 372                        '-I', android_jar,
 373                        '--output-text-symbols', gen_dir,
 374                        '-J', gen_dir,
 375                        '--ignore-assets', build_utils.AAPT_IGNORE_PATTERN]
 376
 377     for d in input_resource_dirs:
 378       package_command += ['-S', d]
 379
 380     for d in dep_subdirs:
 381       package_command += ['-S', d]
 382
 383     if options.non_constant_id:
 384       package_command.append('--non-constant-id')
 385     if options.custom_package:
 386       package_command += ['--custom-package', options.custom_package]
 387     if options.proguard_file:
 388       package_command += ['-G', options.proguard_file]
 389     if options.shared_resources:
 390       package_command.append('--shared-lib')
 391     if options.app_as_shared_lib:
 392       package_command.append('--app-as-shared-lib')
 393     build_utils.CheckOutput(package_command, print_stderr=False)
 394
 395     if options.extra_res_packages:
 396       CreateExtraRJavaFiles(
 397           gen_dir,
 398           build_utils.ParseGypList(options.extra_res_packages),
 399           build_utils.ParseGypList(options.extra_r_text_files),
 400           options.shared_resources,
 401           options.include_all_resources)
 402
 403     # This is the list of directories with resources to put in the final .zip
 404     # file. The order of these is important so that crunched/v14 resources
 405     # override the normal ones.
 406     zip_resource_dirs = input_resource_dirs + [v14_dir]
 407
 408     base_crunch_dir = os.path.join(temp_dir, 'crunch')
 409
 410     # Crunch image resources. This shrinks png files and is necessary for
 411     # 9-patch images to display correctly. 'aapt crunch' accepts only a single
 412     # directory at a time and deletes everything in the output directory.
 413     for idx, input_dir in enumerate(input_resource_dirs):
 414       crunch_dir = os.path.join(base_crunch_dir, str(idx))
 415       build_utils.MakeDirectory(crunch_dir)
 416       zip_resource_dirs.append(crunch_dir)
 417       CrunchDirectory(aapt, input_dir, crunch_dir)
 418
 419     ZipResources(zip_resource_dirs, options.resource_zip_out)
 420
 421     if options.all_resources_zip_out:
 422       CombineZips([options.resource_zip_out] + dep_zips,
 423                   options.all_resources_zip_out)
 424
 425     if options.R_dir:
 426       build_utils.DeleteDirectory(options.R_dir)
 427       shutil.copytree(gen_dir, options.R_dir)
 428     else:
 429       build_utils.ZipDir(options.srcjar_out, gen_dir)
 430
 431     if options.r_text_out:
 432       r_text_path = os.path.join(gen_dir, 'R.txt')
 433       if os.path.exists(r_text_path):
 434         shutil.copyfile(r_text_path, options.r_text_out)
 435       else:
 436         open(options.r_text_out, 'w').close()
 437
 438   if options.depfile:
 439     input_files += build_utils.GetPythonDependencies()
 440     build_utils.WriteDepfile(options.depfile, input_files)
 441
 442   if options.stamp:
 443     build_utils.Touch(options.stamp)
 444
 445
 446 if __name__ == '__main__':
 447   main()