android_webview/buildbot/deps_whitelist.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 """Logic to generate lists of DEPS used by various parts of
   7 the android_webview continuous integration (buildbot) infrastructure.
   8
   9 Note: The root Chromium project (which is not explicitly listed here)
  10 has a couple of third_party libraries checked in directly into it. This means
  11 that the list of third parties present in this file is not a comprehensive
  12 list of third party android_webview dependencies.
  13 """
  14
  15 import argparse
  16 import json
  17 import logging
  18 import os
  19 import sys
  20
  21 # Add android_webview/tools to path to get at known_issues.
  22 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
  23 import known_issues
  24
  25
  26 class DepsWhitelist(object):
  27   def __init__(self):
  28     # If a new DEPS entry is needed for the AOSP bot to compile please add it
  29     # here first.
  30     # This is a staging area for deps that are accepted by the android_webview
  31     # team and are in the process of having the required branches being created
  32     # in the Android tree.
  33     self._compile_but_not_snapshot_dependencies = [
  34       'third_party/libaddressinput/src/cpp',
  35       'third_party/boringssl/src',
  36     ]
  37
  38     # Dependencies that need to be merged into the Android tree.
  39     self._snapshot_into_android_dependencies = [
  40       'sdch/open-vcdiff',
  41       'testing/gtest',
  42       'third_party/WebKit',
  43       'third_party/angle',
  44       'third_party/brotli/src',
  45       ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
  46        'braille'),
  47       'third_party/freetype',
  48       'third_party/icu',
  49       'third_party/leveldatabase/src',
  50       'third_party/libjingle/source/talk',
  51       'third_party/libphonenumber/src/phonenumbers',
  52       'third_party/libphonenumber/src/resources',
  53       'third_party/libsrtp',
  54       'third_party/libvpx',
  55       'third_party/libyuv',
  56       'third_party/mesa/src',
  57       'third_party/openmax_dl',
  58       'third_party/openssl',
  59       'third_party/opus/src',
  60       'third_party/ots',
  61       'third_party/sfntly/cpp/src',
  62       'third_party/skia',
  63       'third_party/smhasher/src',
  64       'third_party/usrsctp/usrsctplib',
  65       'third_party/webrtc',
  66       'third_party/yasm/source/patched-yasm',
  67       'tools/grit',
  68       'tools/gyp',
  69       'v8',
  70     ]
  71
  72     # We can save some time by not rsyncing code we don't use.
  73     self._prune_from_rsync_build = [
  74         'third_party/WebKit/LayoutTests',
  75     ]
  76
  77     # Dependencies required to build android_webview.
  78     self._compile_dependencies = (self._snapshot_into_android_dependencies +
  79                                   self._compile_but_not_snapshot_dependencies)
  80
  81     # Dependencies required to run android_webview tests but not required to
  82     # compile.
  83     self._test_data_dependencies = [
  84       'chrome/test/data/perf/third_party/octane',
  85     ]
  86
  87   @staticmethod
  88   def _read_deps_file(deps_file_path):
  89     class FileImplStub(object):
  90       """Stub for the File syntax."""
  91       def __init__(self, file_location):
  92         pass
  93
  94       @staticmethod
  95       def GetPath():
  96         return ''
  97
  98       @staticmethod
  99       def GetFilename():
 100         return ''
 101
 102       @staticmethod
 103       def GetRevision():
 104         return None
 105
 106     def from_stub(__, _=None):
 107       """Stub for the From syntax."""
 108       return ''
 109
 110     class VarImpl(object):
 111       def __init__(self, custom_vars, local_scope):
 112         self._custom_vars = custom_vars
 113         self._local_scope = local_scope
 114
 115       def Lookup(self, var_name):
 116         """Implements the Var syntax."""
 117         if var_name in self._custom_vars:
 118           return self._custom_vars[var_name]
 119         elif var_name in self._local_scope.get("vars", {}):
 120           return self._local_scope["vars"][var_name]
 121         raise Exception("Var is not defined: %s" % var_name)
 122
 123     local_scope = {}
 124     var = VarImpl({}, local_scope)
 125     global_scope = {
 126         'File': FileImplStub,
 127         'From': from_stub,
 128         'Var': var.Lookup,
 129         'deps_os': {},
 130     }
 131     execfile(deps_file_path, global_scope, local_scope)
 132     deps = local_scope.get('deps', {})
 133     deps_os = local_scope.get('deps_os', {})
 134     for os_specific_deps in deps_os.itervalues():
 135       deps.update(os_specific_deps)
 136     return deps.keys()
 137
 138   def _get_known_issues(self):
 139     issues = []
 140     for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
 141       for path in paths:
 142         issues.append(os.path.normpath(os.path.join(root, path)))
 143     return issues
 144
 145   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
 146     """Calculates the list of deps that need to be excluded from the deps_file
 147     so that the only deps left are the one in the whitelist."""
 148     all_deps = self._read_deps_file(deps_file_path)
 149     # The list of deps read from the DEPS file are prefixed with the source
 150     # tree root, which is 'src' for Chromium.
 151     def prepend_root(path):
 152       return os.path.join('src', path)
 153     whitelisted_deps = map(prepend_root, whitelisted_deps)
 154     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
 155     return dict(map(lambda(x): (x, None), deps_blacklist))
 156
 157   def _make_blacklist(self, deps_file_path, whitelisted_deps):
 158     """Calculates the list of paths we should exclude """
 159     all_deps = self._read_deps_file(deps_file_path)
 160     def remove_src_prefix(path):
 161       return path.replace('src/', '', 1)
 162     all_deps = map(remove_src_prefix, all_deps)
 163     # Ignore all deps except those whitelisted.
 164     blacklist = set(all_deps).difference(whitelisted_deps)
 165     # Ignore the 'known issues'. Typically these are the licence incompatible
 166     # things checked directly into Chromium.
 167     blacklist = blacklist.union(self._get_known_issues())
 168     # Ignore any other non-deps, non-licence paths we don't like.
 169     blacklist = blacklist.union(self._prune_from_rsync_build)
 170     return list(blacklist)
 171
 172   def get_deps_for_android_build(self, deps_file_path):
 173     """This is used to calculate the custom_deps list for the Android bot.
 174     """
 175     if not deps_file_path:
 176       raise Exception('You need to specify a DEPS file path.')
 177     return self._make_gclient_blacklist(deps_file_path,
 178                                         self._compile_dependencies)
 179
 180   def get_deps_for_android_build_and_test(self, deps_file_path):
 181     """This is used to calculate the custom_deps list for the Android perf bot.
 182     """
 183     if not deps_file_path:
 184       raise Exception('You need to specify a DEPS file path.')
 185     return self._make_gclient_blacklist(deps_file_path,
 186                                         self._compile_dependencies +
 187                                         self._test_data_dependencies)
 188
 189   def get_blacklist_for_android_rsync_build(self, deps_file_path):
 190     """Calculates the list of paths we should exclude when building Android
 191     either because of license compatibility or because they are large and
 192     uneeded.
 193     """
 194     if not deps_file_path:
 195       raise Exception('You need to specify a DEPS file path.')
 196     return self._make_blacklist(deps_file_path, self._compile_dependencies)
 197
 198   def get_deps_for_android_merge(self, _):
 199     """Calculates the list of deps that need to be merged into the Android tree
 200     in order to build the C++ and Java android_webview code."""
 201     return self._snapshot_into_android_dependencies
 202
 203   def get_deps_for_license_check(self, _):
 204     """Calculates the list of deps that need to be checked for Android license
 205     compatibility"""
 206     return self._compile_dependencies
 207
 208
 209   def execute_method(self, method_name, deps_file_path):
 210     methods = {
 211       'android_build': self.get_deps_for_android_build,
 212       'android_build_and_test':
 213         self.get_deps_for_android_build_and_test,
 214       'android_merge': self.get_deps_for_android_merge,
 215       'license_check': self.get_deps_for_license_check,
 216       'android_rsync_build': self.get_blacklist_for_android_rsync_build,
 217     }
 218     if not method_name in methods:
 219       raise Exception('Method name %s is not valid. Valid choices are %s' %
 220                       (method_name, methods.keys()))
 221     return methods[method_name](deps_file_path)
 222
 223 def main():
 224   parser = argparse.ArgumentParser()
 225   parser.add_argument('--method', help='Method to use to fetch from whitelist.',
 226                       required=True)
 227   parser.add_argument('--path-to-deps', help='Path to DEPS file.')
 228   parser.add_argument('--output-json', help='Name of file to write output to.')
 229   parser.add_argument('verbose', action='store_true', default=False)
 230   opts = parser.parse_args()
 231
 232   logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
 233
 234   deps_whitelist = DepsWhitelist()
 235   blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
 236
 237   if (opts.output_json):
 238     output_dict = {
 239         'blacklist' : blacklist
 240     }
 241     with open(opts.output_json, 'w') as output_json_file:
 242       json.dump(output_dict, output_json_file)
 243   else:
 244     print blacklist
 245
 246   return 0
 247
 248
 249 if __name__ == '__main__':
 250   sys.exit(main())