chrome/common/extensions/docs/server2/availability_finder.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import posixpath
   6
   7 from api_models import GetNodeCategories
   8 from api_schema_graph import APISchemaGraph
   9 from branch_utility import BranchUtility, ChannelInfo
  10 from compiled_file_system import CompiledFileSystem, SingleFile, Unicode
  11 from extensions_paths import API_PATHS, JSON_TEMPLATES
  12 from features_bundle import FeaturesBundle
  13 from file_system import FileNotFoundError
  14 from schema_processor import SchemaProcessor
  15 from third_party.json_schema_compiler.memoize import memoize
  16 from third_party.json_schema_compiler.model import UnixName
  17
  18
  19 _DEVTOOLS_API = 'devtools_api.json'
  20 _EXTENSION_API = 'extension_api.json'
  21 # The version where api_features.json is first available.
  22 _API_FEATURES_MIN_VERSION = 28
  23 # The version where permission_ and manifest_features.json are available and
  24 # presented in the current format.
  25 _ORIGINAL_FEATURES_MIN_VERSION = 20
  26 # API schemas are aggregated in extension_api.json up to this version.
  27 _EXTENSION_API_MAX_VERSION = 17
  28 # The earliest version for which we have SVN data.
  29 _SVN_MIN_VERSION = 5
  30
  31
  32 def _GetChannelFromFeatures(api_name, features):
  33   '''Finds API channel information for |api_name| from |features|.
  34   Returns None if channel information for the API cannot be located.
  35   '''
  36   feature = features.Get().get(api_name)
  37   return feature.get('channel') if feature else None
  38
  39
  40 def _GetChannelFromAPIFeatures(api_name, features_bundle):
  41   return _GetChannelFromFeatures(api_name, features_bundle.GetAPIFeatures())
  42
  43
  44 def _GetChannelFromManifestFeatures(api_name, features_bundle):
  45   # _manifest_features.json uses unix_style API names.
  46   api_name = UnixName(api_name)
  47   return _GetChannelFromFeatures(api_name,
  48                                  features_bundle.GetManifestFeatures())
  49
  50
  51 def _GetChannelFromPermissionFeatures(api_name, features_bundle):
  52   return _GetChannelFromFeatures(api_name,
  53                                  features_bundle.GetPermissionFeatures())
  54
  55
  56 def _GetAPISchemaFilename(api_name, file_system, version):
  57   '''Gets the name of the file which may contain the schema for |api_name| in
  58   |file_system|, or None if the API is not found. Note that this may be the
  59   single _EXTENSION_API file which all APIs share in older versions of Chrome,
  60   in which case it is unknown whether the API actually exists there.
  61   '''
  62   if version == 'master' or version > _ORIGINAL_FEATURES_MIN_VERSION:
  63     # API schema filenames switch format to unix_hacker_style.
  64     api_name = UnixName(api_name)
  65
  66   # Devtools API names have 'devtools.' prepended to them.
  67   # The corresponding filenames do not.
  68   if 'devtools_' in api_name:
  69     api_name = api_name.replace('devtools_', '')
  70
  71   for api_path in API_PATHS:
  72     try:
  73       for base, _, filenames in file_system.Walk(api_path):
  74         for ext in ('json', 'idl'):
  75           filename = '%s.%s' % (api_name, ext)
  76           if filename in filenames:
  77             return posixpath.join(api_path, base, filename)
  78           if _EXTENSION_API in filenames:
  79             return posixpath.join(api_path, base, _EXTENSION_API)
  80     except FileNotFoundError:
  81       continue
  82   return None
  83
  84
  85 class AvailabilityInfo(object):
  86   '''Represents availability data for an API. |scheduled| is a version number
  87   specifying when dev and beta APIs will become stable, or None if that data
  88   is unknown.
  89   '''
  90   def __init__(self, channel_info, scheduled=None):
  91     assert isinstance(channel_info, ChannelInfo)
  92     assert isinstance(scheduled, int) or scheduled is None
  93     self.channel_info = channel_info
  94     self.scheduled = scheduled
  95
  96   def __eq__(self, other):
  97     return self.__dict__ == other.__dict__
  98
  99   def __ne__(self, other):
 100     return not (self == other)
 101
 102   def __repr__(self):
 103     return '%s%s' % (type(self).__name__, repr(self.__dict__))
 104
 105   def __str__(self):
 106     return repr(self)
 107
 108
 109 class AvailabilityFinder(object):
 110   '''Generates availability information for APIs by looking at API schemas and
 111   _features files over multiple release versions of Chrome.
 112   '''
 113
 114   def __init__(self,
 115                branch_utility,
 116                compiled_fs_factory,
 117                file_system_iterator,
 118                host_file_system,
 119                object_store_creator,
 120                platform,
 121                schema_processor_factory):
 122     self._branch_utility = branch_utility
 123     self._compiled_fs_factory = compiled_fs_factory
 124     self._file_system_iterator = file_system_iterator
 125     self._host_file_system = host_file_system
 126     self._object_store_creator = object_store_creator
 127     def create_object_store(category):
 128       return object_store_creator.Create(
 129           AvailabilityFinder, category='/'.join((platform, category)))
 130     self._top_level_object_store = create_object_store('top_level')
 131     self._node_level_object_store = create_object_store('node_level')
 132     self._json_fs = compiled_fs_factory.ForJson(self._host_file_system)
 133     self._platform = platform
 134     # When processing the API schemas, we retain inlined types in the schema
 135     # so that there are not missing nodes in the APISchemaGraphs when trying
 136     # to lookup availability.
 137     self._schema_processor = schema_processor_factory.Create(True)
 138
 139   def _GetPredeterminedNodeAvailability(self, node_name):
 140     '''Checks a configuration file for hardcoded (i.e. predetermined)
 141     availability information for an API node.
 142     '''
 143     node_info = self._json_fs.GetFromFile(
 144         JSON_TEMPLATES + 'api_availabilities.json').Get().get(node_name)
 145     if node_info is None:
 146       return None
 147     if node_info['channel'] == 'stable':
 148       return AvailabilityInfo(
 149           self._branch_utility.GetStableChannelInfo(node_info['version']))
 150     return AvailabilityInfo(
 151         self._branch_utility.GetChannelInfo(node_info['channel']))
 152
 153   @memoize
 154   def _CreateAPISchemaFileSystem(self, file_system):
 155     '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
 156     data and formatting it so that it can be used to create APISchemaGraphs.
 157     '''
 158     def process_schema(path, data):
 159       return self._schema_processor.Process(path, data)
 160     return self._compiled_fs_factory.Create(file_system,
 161                                             SingleFile(Unicode(process_schema)),
 162                                             CompiledFileSystem,
 163                                             category='api-schema')
 164
 165   def _GetAPISchema(self, api_name, file_system, version):
 166     '''Searches |file_system| for |api_name|'s API schema data, and processes
 167     and returns it if found.
 168     '''
 169     api_filename = _GetAPISchemaFilename(api_name, file_system, version)
 170     if api_filename is None:
 171       # No file for the API could be found in the given |file_system|.
 172       return None
 173
 174     schema_fs = self._CreateAPISchemaFileSystem(file_system)
 175     api_schemas = schema_fs.GetFromFile(api_filename).Get()
 176     matching_schemas = [api for api in api_schemas
 177                         if api and api['namespace'] == api_name]
 178     # There should only be a single matching schema per file, or zero in the
 179     # case of no API data being found in _EXTENSION_API.
 180     assert len(matching_schemas) <= 1
 181     return matching_schemas or None
 182
 183   def _HasAPISchema(self, api_name, file_system, version):
 184     '''Whether or not an API schema for |api_name| exists in the given
 185     |file_system|.
 186     '''
 187     filename = _GetAPISchemaFilename(api_name, file_system, version)
 188     if filename is None:
 189       return False
 190     if filename.endswith(_EXTENSION_API) or filename.endswith(_DEVTOOLS_API):
 191       return self._GetAPISchema(api_name, file_system, version) is not None
 192     return True
 193
 194   def _CheckStableAvailability(self,
 195                                api_name,
 196                                file_system,
 197                                version,
 198                                earliest_version=None):
 199     '''Checks for availability of an API, |api_name|, on the stable channel.
 200     Considers several _features.json files, file system existence, and
 201     extension_api.json depending on the given |version|.
 202     |earliest_version| is the version of Chrome at which |api_name| first became
 203     available. It should only be given when checking stable availability for
 204     API nodes, so it can be used as an alternative to the check for filesystem
 205     existence.
 206     '''
 207     earliest_version = earliest_version or _SVN_MIN_VERSION
 208     if version < earliest_version:
 209       # SVN data isn't available below this version.
 210       return False
 211     features_bundle = self._CreateFeaturesBundle(file_system)
 212     available_channel = None
 213     if version >= _API_FEATURES_MIN_VERSION:
 214       # The _api_features.json file first appears in version 28 and should be
 215       # the most reliable for finding API availability.
 216       available_channel = _GetChannelFromAPIFeatures(api_name,
 217                                                      features_bundle)
 218     if version >= _ORIGINAL_FEATURES_MIN_VERSION:
 219       # The _permission_features.json and _manifest_features.json files are
 220       # present in Chrome 20 and onwards. Use these if no information could be
 221       # found using _api_features.json.
 222       available_channel = (
 223           available_channel or
 224           _GetChannelFromPermissionFeatures(api_name, features_bundle) or
 225           _GetChannelFromManifestFeatures(api_name, features_bundle))
 226       if available_channel is not None:
 227         return available_channel == 'stable'
 228
 229     # |earliest_version| == _SVN_MIN_VERSION implies we're dealing with an API.
 230     # Fall back to a check for file system existence if the API is not
 231     # stable in any of the _features.json files, or if the _features files
 232     # do not exist (version 19 and earlier).
 233     if earliest_version == _SVN_MIN_VERSION:
 234       return self._HasAPISchema(api_name, file_system, version)
 235     # For API nodes, assume it's available if |version| is greater than the
 236     # version the node became available (which it is, because of the first
 237     # check).
 238     return True
 239
 240   def _CheckChannelAvailability(self, api_name, file_system, channel_info):
 241     '''Searches through the _features files in a given |file_system|, falling
 242     back to checking the file system for API schema existence, to determine
 243     whether or not an API is available on the given channel, |channel_info|.
 244     '''
 245     features_bundle = self._CreateFeaturesBundle(file_system)
 246     available_channel = (
 247         _GetChannelFromAPIFeatures(api_name, features_bundle) or
 248         _GetChannelFromPermissionFeatures(api_name, features_bundle) or
 249         _GetChannelFromManifestFeatures(api_name, features_bundle))
 250     if (available_channel is None and
 251         self._HasAPISchema(api_name, file_system, channel_info.version)):
 252       # If an API is not represented in any of the _features files, but exists
 253       # in the filesystem, then assume it is available in this version.
 254       # The chrome.windows API is an example of this.
 255       available_channel = channel_info.channel
 256     # If the channel we're checking is the same as or newer than the
 257     # |available_channel| then the API is available at this channel.
 258     newest = BranchUtility.NewestChannel((available_channel,
 259                                           channel_info.channel))
 260     return available_channel is not None and newest == channel_info.channel
 261
 262   def _CheckChannelAvailabilityForNode(self,
 263                                        node_name,
 264                                        file_system,
 265                                        channel_info,
 266                                        earliest_channel_info):
 267     '''Searches through the _features files in a given |file_system| to
 268     determine whether or not an API node is available on the given channel,
 269     |channel_info|. |earliest_channel_info| is the earliest channel the node
 270     was introduced.
 271     '''
 272     features_bundle = self._CreateFeaturesBundle(file_system)
 273     available_channel = None
 274     # Only API nodes can have their availability overriden on a per-node basis,
 275     # so we only need to check _api_features.json.
 276     if channel_info.version >= _API_FEATURES_MIN_VERSION:
 277       available_channel = _GetChannelFromAPIFeatures(node_name, features_bundle)
 278     if (available_channel is None and
 279         channel_info.version >= earliest_channel_info.version):
 280       # Most API nodes inherit their availabiltity from their parent, so don't
 281       # explicitly appear in _api_features.json. For example, "tabs.create"
 282       # isn't listed; it inherits from "tabs". Assume these are available at
 283       # |channel_info|.
 284       available_channel = channel_info.channel
 285     newest = BranchUtility.NewestChannel((available_channel,
 286                                           channel_info.channel))
 287     return available_channel is not None and newest == channel_info.channel
 288
 289   @memoize
 290   def _CreateFeaturesBundle(self, file_system):
 291     return FeaturesBundle(file_system,
 292                           self._compiled_fs_factory,
 293                           self._object_store_creator,
 294                           self._platform)
 295
 296   def _CheckAPIAvailability(self, api_name, file_system, channel_info):
 297     '''Determines the availability for an API at a certain version of Chrome.
 298     Two branches of logic are used depending on whether or not the API is
 299     determined to be 'stable' at the given version.
 300     '''
 301     if channel_info.channel == 'stable':
 302       return self._CheckStableAvailability(api_name,
 303                                            file_system,
 304                                            channel_info.version)
 305     return self._CheckChannelAvailability(api_name,
 306                                           file_system,
 307                                           channel_info)
 308
 309   def _FindScheduled(self, api_name, earliest_version=None):
 310     '''Determines the earliest version of Chrome where the API is stable.
 311     Unlike the code in GetAPIAvailability, this checks if the API is stable
 312     even when Chrome is in dev or beta, which shows that the API is scheduled
 313     to be stable in that verison of Chrome. |earliest_version| is the version
 314     |api_name| became first available. Only use it when finding scheduled
 315     availability for nodes.
 316     '''
 317     def check_scheduled(file_system, channel_info):
 318       return self._CheckStableAvailability(api_name,
 319                                            file_system,
 320                                            channel_info.version,
 321                                            earliest_version=earliest_version)
 322
 323     stable_channel = self._file_system_iterator.Descending(
 324         self._branch_utility.GetChannelInfo('dev'), check_scheduled)
 325
 326     return stable_channel.version if stable_channel else None
 327
 328   def _CheckAPINodeAvailability(self, node_name, earliest_channel_info):
 329     '''Gets availability data for a node by checking _features files.
 330     '''
 331     # Check for predetermined availability and cache this information if found.
 332     availability = self._GetPredeterminedNodeAvailability(node_name)
 333     if availability is not None:
 334       self._top_level_object_store.Set(node_name, availability)
 335       return availability
 336
 337     def check_node_availability(file_system, channel_info):
 338       return self._CheckChannelAvailabilityForNode(node_name,
 339                                                    file_system,
 340                                                    channel_info,
 341                                                    earliest_channel_info)
 342     channel_info = (self._file_system_iterator.Descending(
 343         self._branch_utility.GetChannelInfo('dev'), check_node_availability) or
 344         earliest_channel_info)
 345
 346     if channel_info.channel == 'stable':
 347       scheduled = None
 348     else:
 349       scheduled = self._FindScheduled(
 350           node_name,
 351           earliest_version=earliest_channel_info.version)
 352
 353     return AvailabilityInfo(channel_info, scheduled=scheduled)
 354
 355   def GetAPIAvailability(self, api_name):
 356     '''Performs a search for an API's top-level availability by using a
 357     HostFileSystemIterator instance to traverse multiple version of the
 358     SVN filesystem.
 359     '''
 360     availability = self._top_level_object_store.Get(api_name).Get()
 361     if availability is not None:
 362       return availability
 363
 364     # Check for predetermined availability and cache this information if found.
 365     availability = self._GetPredeterminedNodeAvailability(api_name)
 366     if availability is not None:
 367       self._top_level_object_store.Set(api_name, availability)
 368       return availability
 369
 370     def check_api_availability(file_system, channel_info):
 371       return self._CheckAPIAvailability(api_name, file_system, channel_info)
 372
 373     channel_info = self._file_system_iterator.Descending(
 374         self._branch_utility.GetChannelInfo('dev'),
 375         check_api_availability)
 376     if channel_info is None:
 377       # The API wasn't available on 'dev', so it must be a 'master'-only API.
 378       channel_info = self._branch_utility.GetChannelInfo('master')
 379
 380     # If the API is not stable, check when it will be scheduled to be stable.
 381     if channel_info.channel == 'stable':
 382       scheduled = None
 383     else:
 384       scheduled = self._FindScheduled(api_name)
 385
 386     availability = AvailabilityInfo(channel_info, scheduled=scheduled)
 387
 388     self._top_level_object_store.Set(api_name, availability)
 389     return availability
 390
 391   def GetAPINodeAvailability(self, api_name):
 392     '''Returns an APISchemaGraph annotated with each node's availability (the
 393     ChannelInfo at the oldest channel it's available in).
 394     '''
 395     availability_graph = self._node_level_object_store.Get(api_name).Get()
 396     if availability_graph is not None:
 397       return availability_graph
 398
 399     def assert_not_none(value):
 400       assert value is not None
 401       return value
 402
 403     availability_graph = APISchemaGraph()
 404     host_fs = self._host_file_system
 405     master_stat = assert_not_none(host_fs.Stat(_GetAPISchemaFilename(
 406         api_name, host_fs, 'master')))
 407
 408     # Weird object thing here because nonlocal is Python 3.
 409     previous = type('previous', (object,), {'stat': None, 'graph': None})
 410
 411     def update_availability_graph(file_system, channel_info):
 412       # If we can't find a filename, skip checking at this branch.
 413       # For example, something could have a predetermined availability of 23,
 414       # but it doesn't show up in the file system until 26.
 415       # We know that the file will become available at some point.
 416       #
 417       # The problem with this is that at the first version where the API file
 418       # exists, we'll get a huge chunk of new objects that don't match
 419       # the predetermined API availability.
 420       version_filename = _GetAPISchemaFilename(api_name,
 421                                                file_system,
 422                                                channel_info.version)
 423       if version_filename is None:
 424         # Continue the loop at the next version.
 425         return True
 426
 427       version_stat = assert_not_none(file_system.Stat(version_filename))
 428
 429       # Important optimisation: only re-parse the graph if the file changed in
 430       # the last revision. Parsing the same schema and forming a graph on every
 431       # iteration is really expensive.
 432       if version_stat == previous.stat:
 433         version_graph = previous.graph
 434       else:
 435         # Keep track of any new schema elements from this version by adding
 436         # them to |availability_graph|.
 437         #
 438         # Calling |availability_graph|.Lookup() on the nodes being updated
 439         # will return the |annotation| object -- the current |channel_info|.
 440         version_graph = APISchemaGraph(
 441             api_schema=self._GetAPISchema(api_name,
 442                                           file_system,
 443                                           channel_info.version))
 444         def annotator(node_name):
 445           return self._CheckAPINodeAvailability('%s.%s' % (api_name, node_name),
 446                                                 channel_info)
 447
 448         availability_graph.Update(version_graph.Subtract(availability_graph),
 449                                   annotator)
 450
 451       previous.stat = version_stat
 452       previous.graph = version_graph
 453
 454       # Continue looping until there are no longer differences between this
 455       # version and master.
 456       return version_stat != master_stat
 457
 458     self._file_system_iterator.Ascending(
 459         self.GetAPIAvailability(api_name).channel_info,
 460         update_availability_graph)
 461
 462     self._node_level_object_store.Set(api_name, availability_graph)
 463     return availability_graph