android_webview/buildbot/deps_whitelist.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 """Logic to generate lists of DEPS used by various parts of
   7 the android_webview continuous integration (buildbot) infrastructure.
   8
   9 Note: The root Chromium project (which is not explicitly listed here)
  10 has a couple of third_party libraries checked in directly into it. This means
  11 that the list of third parties present in this file is not a comprehensive
  12 list of third party android_webview dependencies.
  13 """
  14
  15 import argparse
  16 import json
  17 import logging
  18 import os
  19 import sys
  20
  21
  22 class DepsWhitelist(object):
  23   def __init__(self):
  24     # If a new DEPS entry is needed for the AOSP bot to compile please add it
  25     # here first.
  26     # This is a staging area for deps that are accepted by the android_webview
  27     # team and are in the process of having the required branches being created
  28     # in the Android tree.
  29     self._compile_but_not_snapshot_dependencies = [
  30       'third_party/brotli/src',
  31     ]
  32
  33     # Dependencies that need to be merged into the Android tree.
  34     self._snapshot_into_android_dependencies = [
  35       'sdch/open-vcdiff',
  36       'testing/gtest',
  37       'third_party/WebKit',
  38       'third_party/angle',
  39       ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
  40        'braille'),
  41       'third_party/freetype',
  42       'third_party/icu',
  43       'third_party/leveldatabase/src',
  44       'third_party/libjingle/source/talk',
  45       'third_party/libphonenumber/src/phonenumbers',
  46       'third_party/libphonenumber/src/resources',
  47       'third_party/mesa/src',
  48       'third_party/openssl',
  49       'third_party/opus/src',
  50       'third_party/ots',
  51       'third_party/sfntly/cpp/src',
  52       'third_party/skia/gyp',
  53       'third_party/skia/include',
  54       'third_party/skia/src',
  55       'third_party/smhasher/src',
  56       'third_party/v8-i18n',
  57       'third_party/yasm/source/patched-yasm',
  58       'tools/grit',
  59       'tools/gyp',
  60       'v8',
  61     ]
  62
  63     # Dependencies required to build android_webview.
  64     self._compile_dependencies = (self._snapshot_into_android_dependencies +
  65                                   self._compile_but_not_snapshot_dependencies)
  66
  67     # Dependencies required to run android_webview tests but not required to
  68     # compile.
  69     self._test_data_dependencies = [
  70       'chrome/test/data/perf/third_party/octane',
  71     ]
  72
  73   @staticmethod
  74   def _read_deps_file(deps_file_path):
  75     class FileImplStub(object):
  76       """Stub for the File syntax."""
  77       def __init__(self, file_location):
  78         pass
  79
  80       @staticmethod
  81       def GetPath():
  82         return ''
  83
  84       @staticmethod
  85       def GetFilename():
  86         return ''
  87
  88       @staticmethod
  89       def GetRevision():
  90         return None
  91
  92     def from_stub(__, _=None):
  93       """Stub for the From syntax."""
  94       return ''
  95
  96     class VarImpl(object):
  97       def __init__(self, custom_vars, local_scope):
  98         self._custom_vars = custom_vars
  99         self._local_scope = local_scope
 100
 101       def Lookup(self, var_name):
 102         """Implements the Var syntax."""
 103         if var_name in self._custom_vars:
 104           return self._custom_vars[var_name]
 105         elif var_name in self._local_scope.get("vars", {}):
 106           return self._local_scope["vars"][var_name]
 107         raise Exception("Var is not defined: %s" % var_name)
 108
 109     local_scope = {}
 110     var = VarImpl({}, local_scope)
 111     global_scope = {
 112         'File': FileImplStub,
 113         'From': from_stub,
 114         'Var': var.Lookup,
 115         'deps_os': {},
 116     }
 117     execfile(deps_file_path, global_scope, local_scope)
 118     deps = local_scope.get('deps', {})
 119     deps_os = local_scope.get('deps_os', {})
 120     for os_specific_deps in deps_os.itervalues():
 121       deps.update(os_specific_deps)
 122     return deps.keys()
 123
 124   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
 125     """Calculates the list of deps that need to be excluded from the deps_file
 126     so that the only deps left are the one in the whitelist."""
 127     all_deps = self._read_deps_file(deps_file_path)
 128     # The list of deps read from the DEPS file are prefixed with the source
 129     # tree root, which is 'src' for Chromium.
 130     def prepend_root(path):
 131       return os.path.join('src', path)
 132     whitelisted_deps = map(prepend_root, whitelisted_deps)
 133     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
 134     return dict(map(lambda(x): (x, None), deps_blacklist))
 135
 136   def get_deps_for_android_build(self, deps_file_path):
 137     """This is used to calculate the custom_deps list for the Android bot.
 138     """
 139     if not deps_file_path:
 140       raise Exception('You need to specify a DEPS file path.')
 141     return self._make_gclient_blacklist(deps_file_path,
 142                                         self._compile_dependencies)
 143
 144   def get_deps_for_android_build_and_test(self, deps_file_path):
 145     """This is used to calculate the custom_deps list for the Android perf bot.
 146     """
 147     if not deps_file_path:
 148       raise Exception('You need to specify a DEPS file path.')
 149     return self._make_gclient_blacklist(deps_file_path,
 150                                         self._compile_dependencies +
 151                                         self._test_data_dependencies)
 152
 153   def get_deps_for_android_merge(self, _):
 154     """Calculates the list of deps that need to be merged into the Android tree
 155     in order to build the C++ and Java android_webview code."""
 156     return self._snapshot_into_android_dependencies
 157
 158   def get_deps_for_license_check(self, _):
 159     """Calculates the list of deps that need to be checked for Android license
 160     compatibility"""
 161     return self._compile_dependencies
 162
 163   def execute_method(self, method_name, deps_file_path):
 164     methods = {
 165       'android_build': self.get_deps_for_android_build,
 166       'android_build_and_test':
 167         self.get_deps_for_android_build_and_test,
 168       'android_merge': self.get_deps_for_android_merge,
 169       'license_check': self.get_deps_for_license_check
 170     }
 171     if not method_name in methods:
 172       raise Exception('Method name %s is not valid. Valid choices are %s' %
 173                       (method_name, methods.keys()))
 174     return methods[method_name](deps_file_path)
 175
 176 def main():
 177   parser = argparse.ArgumentParser()
 178   parser.add_argument('--method', help='Method to use to fetch from whitelist.',
 179                       required=True)
 180   parser.add_argument('--path-to-deps', help='Path to DEPS file.')
 181   parser.add_argument('--output-json', help='Name of file to write output to.')
 182   parser.add_argument('verbose', action='store_true', default=False)
 183   opts = parser.parse_args()
 184
 185   logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
 186
 187   deps_whitelist = DepsWhitelist()
 188   blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
 189
 190   if (opts.output_json):
 191     output_dict = {
 192         'blacklist' : blacklist
 193     }
 194     with open(opts.output_json, 'w') as output_json_file:
 195       json.dump(output_dict, output_json_file)
 196   else:
 197     print blacklist
 198
 199   return 0
 200
 201
 202 if __name__ == '__main__':
 203   sys.exit(main())