third_party/jinja2/bccache.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.bccache
   4     ~~~~~~~~~~~~~~
   5
   6     This module implements the bytecode cache system Jinja is optionally
   7     using.  This is useful if you have very complex template situations and
   8     the compiliation of all those templates slow down your application too
   9     much.
  10
  11     Situations where this is useful are often forking web applications that
  12     are initialized on the first request.
  13
  14     :copyright: (c) 2010 by the Jinja Team.
  15     :license: BSD.
  16 """
  17 from os import path, listdir
  18 import sys
  19 import marshal
  20 import tempfile
  21 import fnmatch
  22 from hashlib import sha1
  23 from jinja2.utils import open_if_exists
  24 from jinja2._compat import BytesIO, pickle, PY2, text_type
  25
  26
  27 # marshal works better on 3.x, one hack less required
  28 if not PY2:
  29     marshal_dump = marshal.dump
  30     marshal_load = marshal.load
  31 else:
  32
  33     def marshal_dump(code, f):
  34         if isinstance(f, file):
  35             marshal.dump(code, f)
  36         else:
  37             f.write(marshal.dumps(code))
  38
  39     def marshal_load(f):
  40         if isinstance(f, file):
  41             return marshal.load(f)
  42         return marshal.loads(f.read())
  43
  44
  45 bc_version = 2
  46
  47 # magic version used to only change with new jinja versions.  With 2.6
  48 # we change this to also take Python version changes into account.  The
  49 # reason for this is that Python tends to segfault if fed earlier bytecode
  50 # versions because someone thought it would be a good idea to reuse opcodes
  51 # or make Python incompatible with earlier versions.
  52 bc_magic = 'j2'.encode('ascii') + \
  53     pickle.dumps(bc_version, 2) + \
  54     pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
  55
  56
  57 class Bucket(object):
  58     """Buckets are used to store the bytecode for one template.  It's created
  59     and initialized by the bytecode cache and passed to the loading functions.
  60
  61     The buckets get an internal checksum from the cache assigned and use this
  62     to automatically reject outdated cache material.  Individual bytecode
  63     cache subclasses don't have to care about cache invalidation.
  64     """
  65
  66     def __init__(self, environment, key, checksum):
  67         self.environment = environment
  68         self.key = key
  69         self.checksum = checksum
  70         self.reset()
  71
  72     def reset(self):
  73         """Resets the bucket (unloads the bytecode)."""
  74         self.code = None
  75
  76     def load_bytecode(self, f):
  77         """Loads bytecode from a file or file like object."""
  78         # make sure the magic header is correct
  79         magic = f.read(len(bc_magic))
  80         if magic != bc_magic:
  81             self.reset()
  82             return
  83         # the source code of the file changed, we need to reload
  84         checksum = pickle.load(f)
  85         if self.checksum != checksum:
  86             self.reset()
  87             return
  88         self.code = marshal_load(f)
  89
  90     def write_bytecode(self, f):
  91         """Dump the bytecode into the file or file like object passed."""
  92         if self.code is None:
  93             raise TypeError('can\'t write empty bucket')
  94         f.write(bc_magic)
  95         pickle.dump(self.checksum, f, 2)
  96         marshal_dump(self.code, f)
  97
  98     def bytecode_from_string(self, string):
  99         """Load bytecode from a string."""
 100         self.load_bytecode(BytesIO(string))
 101
 102     def bytecode_to_string(self):
 103         """Return the bytecode as string."""
 104         out = BytesIO()
 105         self.write_bytecode(out)
 106         return out.getvalue()
 107
 108
 109 class BytecodeCache(object):
 110     """To implement your own bytecode cache you have to subclass this class
 111     and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
 112     these methods are passed a :class:`~jinja2.bccache.Bucket`.
 113
 114     A very basic bytecode cache that saves the bytecode on the file system::
 115
 116         from os import path
 117
 118         class MyCache(BytecodeCache):
 119
 120             def __init__(self, directory):
 121                 self.directory = directory
 122
 123             def load_bytecode(self, bucket):
 124                 filename = path.join(self.directory, bucket.key)
 125                 if path.exists(filename):
 126                     with open(filename, 'rb') as f:
 127                         bucket.load_bytecode(f)
 128
 129             def dump_bytecode(self, bucket):
 130                 filename = path.join(self.directory, bucket.key)
 131                 with open(filename, 'wb') as f:
 132                     bucket.write_bytecode(f)
 133
 134     A more advanced version of a filesystem based bytecode cache is part of
 135     Jinja2.
 136     """
 137
 138     def load_bytecode(self, bucket):
 139         """Subclasses have to override this method to load bytecode into a
 140         bucket.  If they are not able to find code in the cache for the
 141         bucket, it must not do anything.
 142         """
 143         raise NotImplementedError()
 144
 145     def dump_bytecode(self, bucket):
 146         """Subclasses have to override this method to write the bytecode
 147         from a bucket back to the cache.  If it unable to do so it must not
 148         fail silently but raise an exception.
 149         """
 150         raise NotImplementedError()
 151
 152     def clear(self):
 153         """Clears the cache.  This method is not used by Jinja2 but should be
 154         implemented to allow applications to clear the bytecode cache used
 155         by a particular environment.
 156         """
 157
 158     def get_cache_key(self, name, filename=None):
 159         """Returns the unique hash key for this template name."""
 160         hash = sha1(name.encode('utf-8'))
 161         if filename is not None:
 162             filename = '|' + filename
 163             if isinstance(filename, text_type):
 164                 filename = filename.encode('utf-8')
 165             hash.update(filename)
 166         return hash.hexdigest()
 167
 168     def get_source_checksum(self, source):
 169         """Returns a checksum for the source."""
 170         return sha1(source.encode('utf-8')).hexdigest()
 171
 172     def get_bucket(self, environment, name, filename, source):
 173         """Return a cache bucket for the given template.  All arguments are
 174         mandatory but filename may be `None`.
 175         """
 176         key = self.get_cache_key(name, filename)
 177         checksum = self.get_source_checksum(source)
 178         bucket = Bucket(environment, key, checksum)
 179         self.load_bytecode(bucket)
 180         return bucket
 181
 182     def set_bucket(self, bucket):
 183         """Put the bucket into the cache."""
 184         self.dump_bytecode(bucket)
 185
 186
 187 class FileSystemBytecodeCache(BytecodeCache):
 188     """A bytecode cache that stores bytecode on the filesystem.  It accepts
 189     two arguments: The directory where the cache items are stored and a
 190     pattern string that is used to build the filename.
 191
 192     If no directory is specified the system temporary items folder is used.
 193
 194     The pattern can be used to have multiple separate caches operate on the
 195     same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
 196     is replaced with the cache key.
 197
 198     >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
 199
 200     This bytecode cache supports clearing of the cache using the clear method.
 201     """
 202
 203     def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
 204         if directory is None:
 205             directory = tempfile.gettempdir()
 206         self.directory = directory
 207         self.pattern = pattern
 208
 209     def _get_cache_filename(self, bucket):
 210         return path.join(self.directory, self.pattern % bucket.key)
 211
 212     def load_bytecode(self, bucket):
 213         f = open_if_exists(self._get_cache_filename(bucket), 'rb')
 214         if f is not None:
 215             try:
 216                 bucket.load_bytecode(f)
 217             finally:
 218                 f.close()
 219
 220     def dump_bytecode(self, bucket):
 221         f = open(self._get_cache_filename(bucket), 'wb')
 222         try:
 223             bucket.write_bytecode(f)
 224         finally:
 225             f.close()
 226
 227     def clear(self):
 228         # imported lazily here because google app-engine doesn't support
 229         # write access on the file system and the function does not exist
 230         # normally.
 231         from os import remove
 232         files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
 233         for filename in files:
 234             try:
 235                 remove(path.join(self.directory, filename))
 236             except OSError:
 237                 pass
 238
 239
 240 class MemcachedBytecodeCache(BytecodeCache):
 241     """This class implements a bytecode cache that uses a memcache cache for
 242     storing the information.  It does not enforce a specific memcache library
 243     (tummy's memcache or cmemcache) but will accept any class that provides
 244     the minimal interface required.
 245
 246     Libraries compatible with this class:
 247
 248     -   `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
 249     -   `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
 250     -   `cmemcache <http://gijsbert.org/cmemcache/>`_
 251
 252     (Unfortunately the django cache interface is not compatible because it
 253     does not support storing binary data, only unicode.  You can however pass
 254     the underlying cache client to the bytecode cache which is available
 255     as `django.core.cache.cache._client`.)
 256
 257     The minimal interface for the client passed to the constructor is this:
 258
 259     .. class:: MinimalClientInterface
 260
 261         .. method:: set(key, value[, timeout])
 262
 263             Stores the bytecode in the cache.  `value` is a string and
 264             `timeout` the timeout of the key.  If timeout is not provided
 265             a default timeout or no timeout should be assumed, if it's
 266             provided it's an integer with the number of seconds the cache
 267             item should exist.
 268
 269         .. method:: get(key)
 270
 271             Returns the value for the cache key.  If the item does not
 272             exist in the cache the return value must be `None`.
 273
 274     The other arguments to the constructor are the prefix for all keys that
 275     is added before the actual cache key and the timeout for the bytecode in
 276     the cache system.  We recommend a high (or no) timeout.
 277
 278     This bytecode cache does not support clearing of used items in the cache.
 279     The clear method is a no-operation function.
 280
 281     .. versionadded:: 2.7
 282        Added support for ignoring memcache errors through the
 283        `ignore_memcache_errors` parameter.
 284     """
 285
 286     def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
 287                  ignore_memcache_errors=True):
 288         self.client = client
 289         self.prefix = prefix
 290         self.timeout = timeout
 291         self.ignore_memcache_errors = ignore_memcache_errors
 292
 293     def load_bytecode(self, bucket):
 294         try:
 295             code = self.client.get(self.prefix + bucket.key)
 296         except Exception:
 297             if not self.ignore_memcache_errors:
 298                 raise
 299             code = None
 300         if code is not None:
 301             bucket.bytecode_from_string(code)
 302
 303     def dump_bytecode(self, bucket):
 304         args = (self.prefix + bucket.key, bucket.bytecode_to_string())
 305         if self.timeout is not None:
 306             args += (self.timeout,)
 307         try:
 308             self.client.set(*args)
 309         except Exception:
 310             if not self.ignore_memcache_errors:
 311                 raise