chrome/common/extensions/docs/server2/compiled_file_system.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import sys
   6
   7 from docs_server_utils import ToUnicode
   8 from file_system import FileNotFoundError
   9 from future import Future
  10 from path_util import AssertIsDirectory, AssertIsFile, ToDirectory
  11 from third_party.json_schema_compiler import json_parse
  12 from third_party.json_schema_compiler.memoize import memoize
  13 from third_party.motemplate import Motemplate
  14
  15
  16 _CACHEABLE_FUNCTIONS = set()
  17 _SINGLE_FILE_FUNCTIONS = set()
  18
  19
  20 def _GetUnboundFunction(fn):
  21   '''Functions bound to an object are separate from the unbound
  22   defintion. This causes issues when checking for cache membership,
  23   so always get the unbound function, if possible.
  24   '''
  25   return getattr(fn, 'im_func', fn)
  26
  27
  28 def Cache(fn):
  29   '''A decorator which can be applied to the compilation function
  30   passed to CompiledFileSystem.Create, indicating that file/list data
  31   should be cached.
  32
  33   This decorator should be listed first in any list of decorators, along
  34   with the SingleFile decorator below.
  35   '''
  36   _CACHEABLE_FUNCTIONS.add(_GetUnboundFunction(fn))
  37   return fn
  38
  39
  40 def SingleFile(fn):
  41   '''A decorator which can be optionally applied to the compilation function
  42   passed to CompiledFileSystem.Create, indicating that the function only
  43   needs access to the file which is given in the function's callback. When
  44   this is the case some optimisations can be done.
  45
  46   Note that this decorator must be listed first in any list of decorators to
  47   have any effect.
  48   '''
  49   _SINGLE_FILE_FUNCTIONS.add(_GetUnboundFunction(fn))
  50   return fn
  51
  52
  53 def Unicode(fn):
  54   '''A decorator which can be optionally applied to the compilation function
  55   passed to CompiledFileSystem.Create, indicating that the function processes
  56   the file's data as Unicode text.
  57   '''
  58
  59   # The arguments passed to fn can be (self, path, data) or (path, data). In
  60   # either case the last argument is |data|, which should be converted to
  61   # Unicode.
  62   def convert_args(args):
  63     args = list(args)
  64     args[-1] = ToUnicode(args[-1])
  65     return args
  66
  67   return lambda *args: fn(*convert_args(args))
  68
  69
  70 class _CacheEntry(object):
  71   def __init__(self, cache_data, version):
  72
  73     self.cache_data = cache_data
  74     self.version = version
  75
  76
  77 class CompiledFileSystem(object):
  78   '''This class caches FileSystem data that has been processed.
  79   '''
  80
  81   class Factory(object):
  82     '''A class to build a CompiledFileSystem backed by |file_system|.
  83     '''
  84
  85     def __init__(self, object_store_creator):
  86       self._object_store_creator = object_store_creator
  87
  88     def Create(self, file_system, compilation_function, cls, category=None):
  89       '''Creates a CompiledFileSystem view over |file_system| that populates
  90       its cache by calling |compilation_function| with (path, data), where
  91       |data| is the data that was fetched from |path| in |file_system|.
  92
  93       The namespace for the compiled file system is derived similar to
  94       ObjectStoreCreator: from |cls| along with an optional |category|.
  95       '''
  96       assert isinstance(cls, type)
  97       assert not cls.__name__[0].islower()  # guard against non-class types
  98       full_name = [cls.__name__, file_system.GetIdentity()]
  99       if category is not None:
 100         full_name.append(category)
 101       def create_object_store(my_category):
 102         # The read caches can start populated (start_empty=False) because file
 103         # updates are picked up by the stat - but only if the compilation
 104         # function is affected by a single file. If the compilation function is
 105         # affected by other files (e.g. compiling a list of APIs available to
 106         # extensions may be affected by both a features file and the list of
 107         # files in the API directory) then this optimisation won't work.
 108         return self._object_store_creator.Create(
 109             CompiledFileSystem,
 110             category='/'.join(full_name + [my_category]),
 111             start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS)
 112       return CompiledFileSystem(file_system,
 113                                 compilation_function,
 114                                 create_object_store('file'),
 115                                 create_object_store('list'))
 116
 117     @memoize
 118     def ForJson(self, file_system):
 119       '''A CompiledFileSystem specifically for parsing JSON configuration data.
 120       These are memoized over file systems tied to different branches.
 121       '''
 122       return self.Create(file_system,
 123                          Cache(SingleFile(lambda _, data:
 124                                           json_parse.Parse(ToUnicode(data)))),
 125                          CompiledFileSystem,
 126                          category='json')
 127
 128     @memoize
 129     def ForTemplates(self, file_system):
 130       '''Creates a CompiledFileSystem for parsing templates.
 131       '''
 132       return self.Create(
 133           file_system,
 134           Cache(SingleFile(lambda path, text:
 135                            Motemplate(ToUnicode(text), name=path))),
 136           CompiledFileSystem)
 137
 138     @memoize
 139     def ForUnicode(self, file_system):
 140       '''Creates a CompiledFileSystem for Unicode text processing.
 141       '''
 142       return self.Create(
 143         file_system,
 144         SingleFile(lambda _, text: ToUnicode(text)),
 145         CompiledFileSystem,
 146         category='text')
 147
 148   def __init__(self,
 149                file_system,
 150                compilation_function,
 151                file_object_store,
 152                list_object_store):
 153     self._file_system = file_system
 154     self._compilation_function = compilation_function
 155     self._file_object_store = file_object_store
 156     self._list_object_store = list_object_store
 157
 158   def _Get(self, store, key):
 159     if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS:
 160       return store.Get(key)
 161     return Future(value=None)
 162
 163   def _Set(self, store, key, value):
 164     if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS:
 165       store.Set(key, value)
 166
 167   def _RecursiveList(self, path):
 168     '''Returns a Future containing the recursive directory listing of |path| as
 169     a flat list of paths.
 170     '''
 171     def split_dirs_from_files(paths):
 172       '''Returns a tuple (dirs, files) where |dirs| contains the directory
 173       names in |paths| and |files| contains the files.
 174       '''
 175       result = [], []
 176       for path in paths:
 177         result[0 if path.endswith('/') else 1].append(path)
 178       return result
 179
 180     def add_prefix(prefix, paths):
 181       return [prefix + path for path in paths]
 182
 183     # Read in the initial list of files. Do this eagerly (i.e. not part of the
 184     # asynchronous Future contract) because there's a greater chance to
 185     # parallelise fetching with the second layer (can fetch multiple paths).
 186     try:
 187       first_layer_dirs, first_layer_files = split_dirs_from_files(
 188           self._file_system.ReadSingle(path).Get())
 189     except FileNotFoundError:
 190       return Future(exc_info=sys.exc_info())
 191
 192     if not first_layer_dirs:
 193       return Future(value=first_layer_files)
 194
 195     def get_from_future_listing(listings):
 196       '''Recursively lists files from directory listing |futures|.
 197       '''
 198       dirs, files = [], []
 199       for dir_name, listing in listings.iteritems():
 200         new_dirs, new_files = split_dirs_from_files(listing)
 201         # |dirs| are paths for reading. Add the full prefix relative to
 202         # |path| so that |file_system| can find the files.
 203         dirs += add_prefix(dir_name, new_dirs)
 204         # |files| are not for reading, they are for returning to the caller.
 205         # This entire function set (i.e. GetFromFileListing) is defined to
 206         # not include the fetched-path in the result, however, |dir_name|
 207         # will be prefixed with |path|. Strip it.
 208         assert dir_name.startswith(path)
 209         files += add_prefix(dir_name[len(path):], new_files)
 210       if dirs:
 211         files += self._file_system.Read(dirs).Then(
 212             get_from_future_listing).Get()
 213       return files
 214
 215     return self._file_system.Read(add_prefix(path, first_layer_dirs)).Then(
 216         lambda results: first_layer_files + get_from_future_listing(results))
 217
 218   def GetFromFile(self, path, skip_not_found=False):
 219     '''Calls |compilation_function| on the contents of the file at |path|.
 220     If |skip_not_found| is True, then None is passed to |compilation_function|.
 221     '''
 222     AssertIsFile(path)
 223
 224     try:
 225       version = self._file_system.Stat(path).version
 226     except FileNotFoundError:
 227       if skip_not_found:
 228         version = None
 229       else:
 230         return Future(exc_info=sys.exc_info())
 231
 232     cache_entry = self._Get(self._file_object_store, path).Get()
 233     if (cache_entry is not None) and (version == cache_entry.version):
 234       return Future(value=cache_entry.cache_data)
 235
 236     def compile_(files):
 237       cache_data = self._compilation_function(path, files)
 238       self._Set(self._file_object_store, path, _CacheEntry(cache_data, version))
 239       return cache_data
 240
 241     return self._file_system.ReadSingle(
 242         path, skip_not_found=skip_not_found).Then(compile_)
 243
 244   def GetFromFileListing(self, path):
 245     '''Calls |compilation_function| on the listing of the files at |path|.
 246     Assumes that the path given is to a directory.
 247     '''
 248     AssertIsDirectory(path)
 249
 250     try:
 251       version = self._file_system.Stat(path).version
 252     except FileNotFoundError:
 253       return Future(exc_info=sys.exc_info())
 254
 255     cache_entry = self._Get(self._list_object_store, path).Get()
 256     if (cache_entry is not None) and (version == cache_entry.version):
 257       return Future(value=cache_entry.cache_data)
 258
 259     def compile_(files):
 260       cache_data = self._compilation_function(path, files)
 261       self._Set(self._list_object_store, path, _CacheEntry(cache_data, version))
 262       return cache_data
 263     return self._RecursiveList(path).Then(compile_)
 264
 265   # _GetFileVersionFromCache and _GetFileListingVersionFromCache are exposed
 266   # *only* so that ChainedCompiledFileSystem can optimise its caches. *Do not*
 267   # use these methods otherwise, they don't do what you want. Use
 268   # FileSystem.Stat on the FileSystem that this CompiledFileSystem uses.
 269
 270   def _GetFileVersionFromCache(self, path):
 271     cache_entry = self._Get(self._file_object_store, path).Get()
 272     if cache_entry is not None:
 273       return Future(value=cache_entry.version)
 274     stat_future = self._file_system.StatAsync(path)
 275     return Future(callback=lambda: stat_future.Get().version)
 276
 277   def _GetFileListingVersionFromCache(self, path):
 278     path = ToDirectory(path)
 279     cache_entry = self._Get(self._list_object_store, path).Get()
 280     if cache_entry is not None:
 281       return Future(value=cache_entry.version)
 282     stat_future = self._file_system.StatAsync(path)
 283     return Future(callback=lambda: stat_future.Get().version)
 284
 285   def GetIdentity(self):
 286     return self._file_system.GetIdentity()