chrome/common/extensions/docs/server2/compiled_file_system.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import sys
   6
   7 import schema_util
   8 from docs_server_utils import ToUnicode
   9 from file_system import FileNotFoundError
  10 from future import Gettable, Future
  11 from third_party.handlebar import Handlebar
  12 from third_party.json_schema_compiler import json_parse
  13 from third_party.json_schema_compiler.memoize import memoize
  14
  15
  16 _SINGLE_FILE_FUNCTIONS = set()
  17
  18
  19 def SingleFile(fn):
  20   '''A decorator which can be optionally applied to the compilation function
  21   passed to CompiledFileSystem.Create, indicating that the function only
  22   needs access to the file which is given in the function's callback. When
  23   this is the case some optimisations can be done.
  24
  25   Note that this decorator must be listed first in any list of decorators to
  26   have any effect.
  27   '''
  28   _SINGLE_FILE_FUNCTIONS.add(fn)
  29   return fn
  30
  31
  32 def Unicode(fn):
  33   '''A decorator which can be optionally applied to the compilation function
  34   passed to CompiledFileSystem.Create, indicating that the function processes
  35   the file's data as Unicode text.
  36   '''
  37
  38   # The arguments passed to fn can be (self, path, data) or (path, data). In
  39   # either case the last argument is |data|, which should be converted to
  40   # Unicode.
  41   def convert_args(args):
  42     args = list(args)
  43     args[-1] = ToUnicode(args[-1])
  44     return args
  45
  46   return lambda *args: fn(*convert_args(args))
  47
  48
  49 class _CacheEntry(object):
  50   def __init__(self, cache_data, version):
  51
  52     self._cache_data = cache_data
  53     self.version = version
  54
  55
  56 class CompiledFileSystem(object):
  57   '''This class caches FileSystem data that has been processed.
  58   '''
  59
  60   class Factory(object):
  61     '''A class to build a CompiledFileSystem backed by |file_system|.
  62     '''
  63
  64     def __init__(self, object_store_creator):
  65       self._object_store_creator = object_store_creator
  66
  67     def Create(self, file_system, compilation_function, cls, category=None):
  68       '''Creates a CompiledFileSystem view over |file_system| that populates
  69       its cache by calling |compilation_function| with (path, data), where
  70       |data| is the data that was fetched from |path| in |file_system|.
  71
  72       The namespace for the compiled file system is derived similar to
  73       ObjectStoreCreator: from |cls| along with an optional |category|.
  74       '''
  75       assert isinstance(cls, type)
  76       assert not cls.__name__[0].islower()  # guard against non-class types
  77       full_name = [cls.__name__, file_system.GetIdentity()]
  78       if category is not None:
  79         full_name.append(category)
  80       def create_object_store(my_category):
  81         # The read caches can start populated (start_empty=False) because file
  82         # updates are picked up by the stat - but only if the compilation
  83         # function is affected by a single file. If the compilation function is
  84         # affected by other files (e.g. compiling a list of APIs available to
  85         # extensions may be affected by both a features file and the list of
  86         # files in the API directory) then this optimisation won't work.
  87         return self._object_store_creator.Create(
  88             CompiledFileSystem,
  89             category='/'.join(full_name + [my_category]),
  90             start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS)
  91       return CompiledFileSystem(file_system,
  92                                 compilation_function,
  93                                 create_object_store('file'),
  94                                 create_object_store('list'))
  95
  96     @memoize
  97     def ForJson(self, file_system):
  98       '''A CompiledFileSystem specifically for parsing JSON configuration data.
  99       These are memoized over file systems tied to different branches.
 100       '''
 101       return self.Create(file_system,
 102                          SingleFile(lambda _, data:
 103                              json_parse.Parse(ToUnicode(data))),
 104                          CompiledFileSystem,
 105                          category='json')
 106
 107     @memoize
 108     def ForApiSchema(self, file_system):
 109       '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
 110       data and formatting it so that it can be used by other classes, such
 111       as Model and APISchemaGraph.
 112       '''
 113       return self.Create(file_system,
 114                          SingleFile(Unicode(schema_util.ProcessSchema)),
 115                          CompiledFileSystem,
 116                          category='api-schema')
 117
 118     @memoize
 119     def ForTemplates(self, file_system):
 120       '''Creates a CompiledFileSystem for parsing templates.
 121       '''
 122       return self.Create(
 123           file_system,
 124           SingleFile(lambda path, text: Handlebar(ToUnicode(text), name=path)),
 125           CompiledFileSystem)
 126
 127     @memoize
 128     def ForUnicode(self, file_system):
 129       '''Creates a CompiledFileSystem for Unicode text processing.
 130       '''
 131       return self.Create(
 132         file_system,
 133         SingleFile(lambda _, text: ToUnicode(text)),
 134         CompiledFileSystem,
 135         category='text')
 136
 137   def __init__(self,
 138                file_system,
 139                compilation_function,
 140                file_object_store,
 141                list_object_store):
 142     self._file_system = file_system
 143     self._compilation_function = compilation_function
 144     self._file_object_store = file_object_store
 145     self._list_object_store = list_object_store
 146
 147   def _RecursiveList(self, path):
 148     '''Returns a Future containing the recursive directory listing of |path| as
 149     a flat list of paths.
 150     '''
 151     def split_dirs_from_files(paths):
 152       '''Returns a tuple (dirs, files) where |dirs| contains the directory
 153       names in |paths| and |files| contains the files.
 154       '''
 155       result = [], []
 156       for path in paths:
 157         result[0 if path.endswith('/') else 1].append(path)
 158       return result
 159
 160     def add_prefix(prefix, paths):
 161       return [prefix + path for path in paths]
 162
 163     # Read in the initial list of files. Do this eagerly (i.e. not part of the
 164     # asynchronous Future contract) because there's a greater chance to
 165     # parallelise fetching with the second layer (can fetch multiple paths).
 166     try:
 167       first_layer_dirs, first_layer_files = split_dirs_from_files(
 168           self._file_system.ReadSingle(path).Get())
 169     except FileNotFoundError:
 170       return Future(exc_info=sys.exc_info())
 171
 172     if not first_layer_dirs:
 173       return Future(value=first_layer_files)
 174
 175     second_layer_listing = self._file_system.Read(
 176         add_prefix(path, first_layer_dirs))
 177
 178     def resolve():
 179       def get_from_future_listing(futures):
 180         '''Recursively lists files from directory listing |futures|.
 181         '''
 182         dirs, files = [], []
 183         for dir_name, listing in futures.Get().iteritems():
 184           new_dirs, new_files = split_dirs_from_files(listing)
 185           # |dirs| are paths for reading. Add the full prefix relative to
 186           # |path| so that |file_system| can find the files.
 187           dirs += add_prefix(dir_name, new_dirs)
 188           # |files| are not for reading, they are for returning to the caller.
 189           # This entire function set (i.e. GetFromFileListing) is defined to
 190           # not include the fetched-path in the result, however, |dir_name|
 191           # will be prefixed with |path|. Strip it.
 192           assert dir_name.startswith(path)
 193           files += add_prefix(dir_name[len(path):], new_files)
 194         if dirs:
 195           files += get_from_future_listing(self._file_system.Read(dirs))
 196         return files
 197
 198       return first_layer_files + get_from_future_listing(second_layer_listing)
 199
 200     return Future(delegate=Gettable(resolve))
 201
 202   def GetFromFile(self, path):
 203     '''Calls |compilation_function| on the contents of the file at |path|.  If
 204     |binary| is True then the file will be read as binary - but this will only
 205     apply for the first time the file is fetched; if already cached, |binary|
 206     will be ignored.
 207     '''
 208     try:
 209       version = self._file_system.Stat(path).version
 210     except FileNotFoundError:
 211       return Future(exc_info=sys.exc_info())
 212
 213     cache_entry = self._file_object_store.Get(path).Get()
 214     if (cache_entry is not None) and (version == cache_entry.version):
 215       return Future(value=cache_entry._cache_data)
 216
 217     future_files = self._file_system.ReadSingle(path)
 218     def resolve():
 219       cache_data = self._compilation_function(path, future_files.Get())
 220       self._file_object_store.Set(path, _CacheEntry(cache_data, version))
 221       return cache_data
 222     return Future(delegate=Gettable(resolve))
 223
 224   def GetFromFileListing(self, path):
 225     '''Calls |compilation_function| on the listing of the files at |path|.
 226     Assumes that the path given is to a directory.
 227     '''
 228     if not path.endswith('/'):
 229       path += '/'
 230
 231     try:
 232       version = self._file_system.Stat(path).version
 233     except FileNotFoundError:
 234       return Future(exc_info=sys.exc_info())
 235
 236     cache_entry = self._list_object_store.Get(path).Get()
 237     if (cache_entry is not None) and (version == cache_entry.version):
 238       return Future(value=cache_entry._cache_data)
 239
 240     recursive_list_future = self._RecursiveList(path)
 241     def resolve():
 242       cache_data = self._compilation_function(path, recursive_list_future.Get())
 243       self._list_object_store.Set(path, _CacheEntry(cache_data, version))
 244       return cache_data
 245     return Future(delegate=Gettable(resolve))
 246
 247   def GetFileVersion(self, path):
 248     cache_entry = self._file_object_store.Get(path).Get()
 249     if cache_entry is not None:
 250       return cache_entry.version
 251     return self._file_system.Stat(path).version
 252
 253   def GetFileListingVersion(self, path):
 254     if not path.endswith('/'):
 255       path += '/'
 256     cache_entry = self._list_object_store.Get(path).Get()
 257     if cache_entry is not None:
 258       return cache_entry.version
 259     return self._file_system.Stat(path).version