SCons/CacheDir.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """CacheDir support
  25 """
  26
  27 import atexit
  28 import json
  29 import os
  30 import stat
  31 import sys
  32 import uuid
  33
  34 import SCons.Action
  35 import SCons.Errors
  36 import SCons.Warnings
  37 import SCons
  38
  39 cache_enabled = True
  40 cache_debug = False
  41 cache_force = False
  42 cache_show = False
  43 cache_readonly = False
  44 cache_tmp_uuid = uuid.uuid4().hex
  45
  46 def CacheRetrieveFunc(target, source, env):
  47     t = target[0]
  48     fs = t.fs
  49     cd = env.get_CacheDir()
  50     cd.requests += 1
  51     cachedir, cachefile = cd.cachepath(t)
  52     if not fs.exists(cachefile):
  53         cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
  54         return 1
  55     cd.hits += 1
  56     cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
  57     if SCons.Action.execute_actions:
  58         if fs.islink(cachefile):
  59             fs.symlink(fs.readlink(cachefile), t.get_internal_path())
  60         else:
  61             cd.copy_from_cache(env, cachefile, t.get_internal_path())
  62             try:
  63                 os.utime(cachefile, None)
  64             except OSError:
  65                 pass
  66         st = fs.stat(cachefile)
  67         fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
  68     return 0
  69
  70 def CacheRetrieveString(target, source, env):
  71     t = target[0]
  72     fs = t.fs
  73     cd = env.get_CacheDir()
  74     cachedir, cachefile = cd.cachepath(t)
  75     if t.fs.exists(cachefile):
  76         return "Retrieved `%s' from cache" % t.get_internal_path()
  77     return None
  78
  79 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
  80
  81 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
  82
  83 def CachePushFunc(target, source, env):
  84     if cache_readonly:
  85         return
  86
  87     t = target[0]
  88     if t.nocache:
  89         return
  90     fs = t.fs
  91     cd = env.get_CacheDir()
  92     cachedir, cachefile = cd.cachepath(t)
  93     if fs.exists(cachefile):
  94         # Don't bother copying it if it's already there.  Note that
  95         # usually this "shouldn't happen" because if the file already
  96         # existed in cache, we'd have retrieved the file from there,
  97         # not built it.  This can happen, though, in a race, if some
  98         # other person running the same build pushes their copy to
  99         # the cache after we decide we need to build it but before our
 100         # build completes.
 101         cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
 102         return
 103
 104     cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
 105
 106     tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid)
 107     errfmt = "Unable to copy %s to cache. Cache file is %s"
 108
 109     try:
 110         fs.makedirs(cachedir, exist_ok=True)
 111     except OSError:
 112         msg = errfmt % (str(target), cachefile)
 113         raise SCons.Errors.SConsEnvironmentError(msg)
 114     try:
 115         if fs.islink(t.get_internal_path()):
 116             fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
 117         else:
 118             cd.copy_to_cache(env, t.get_internal_path(), tempfile)
 119         fs.rename(tempfile, cachefile)
 120
 121     except EnvironmentError:
 122         # It's possible someone else tried writing the file at the
 123         # same time we did, or else that there was some problem like
 124         # the CacheDir being on a separate file system that's full.
 125         # In any case, inability to push a file to cache doesn't affect
 126         # the correctness of the build, so just print a warning.
 127         msg = errfmt % (str(t), cachefile)
 128         cd.CacheDebug(errfmt + '\n', str(t), cachefile)
 129         SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
 130
 131 CachePush = SCons.Action.Action(CachePushFunc, None)
 132
 133
 134 class CacheDir:
 135
 136     def __init__(self, path):
 137         """
 138         Initialize a CacheDir object.
 139
 140         The cache configuration is stored in the object. It
 141         is read from the config file in the supplied path if
 142         one exists,  if not the config file is created and
 143         the default config is written, as well as saved in the object.
 144         """
 145         self.requests = 0
 146         self.hits = 0
 147         self.path = path
 148         self.current_cache_debug = None
 149         self.debugFP = None
 150         self.config = dict()
 151         if path is None:
 152             return
 153
 154         self._readconfig(path)
 155
 156
 157     def _readconfig(self, path):
 158         """
 159         Read the cache config.
 160
 161         If directory or config file do not exist, create.  Take advantage
 162         of Py3 capability in os.makedirs() and in file open(): just try
 163         the operation and handle failure appropriately.
 164
 165         Omit the check for old cache format, assume that's old enough
 166         there will be none of those left to worry about.
 167
 168         :param path: path to the cache directory
 169         """
 170         config_file = os.path.join(path, 'config')
 171         try:
 172             os.makedirs(path, exist_ok=True)
 173         except FileExistsError:
 174             pass
 175         except OSError:
 176             msg = "Failed to create cache directory " + path
 177             raise SCons.Errors.SConsEnvironmentError(msg)
 178
 179         try:
 180             with open(config_file, 'x') as config:
 181                 self.config['prefix_len'] = 2
 182                 try:
 183                     json.dump(self.config, config)
 184                 except Exception:
 185                     msg = "Failed to write cache configuration for " + path
 186                     raise SCons.Errors.SConsEnvironmentError(msg)
 187         except FileExistsError:
 188             try:
 189                 with open(config_file) as config:
 190                     self.config = json.load(config)
 191             except ValueError:
 192                 msg = "Failed to read cache configuration for " + path
 193                 raise SCons.Errors.SConsEnvironmentError(msg)
 194
 195     def CacheDebug(self, fmt, target, cachefile):
 196         if cache_debug != self.current_cache_debug:
 197             if cache_debug == '-':
 198                 self.debugFP = sys.stdout
 199             elif cache_debug:
 200                 def debug_cleanup(debugFP):
 201                     debugFP.close()
 202
 203                 self.debugFP = open(cache_debug, 'w')
 204                 atexit.register(debug_cleanup, self.debugFP)
 205             else:
 206                 self.debugFP = None
 207             self.current_cache_debug = cache_debug
 208         if self.debugFP:
 209             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
 210             self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
 211                                (self.requests, self.hits, self.misses, self.hit_ratio))
 212
 213     @classmethod
 214     def copy_from_cache(cls, env, src, dst):
 215         if env.cache_timestamp_newer:
 216             return env.fs.copy(src, dst)
 217         else:
 218             return env.fs.copy2(src, dst)
 219
 220     @classmethod
 221     def copy_to_cache(cls, env, src, dst):
 222         try:
 223             result = env.fs.copy2(src, dst)
 224             fs = env.File(src).fs
 225             st = fs.stat(src)
 226             fs.chmod(dst, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
 227             return result
 228         except AttributeError as ex:
 229             raise EnvironmentError from ex
 230
 231     @property
 232     def hit_ratio(self):
 233         return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
 234
 235     @property
 236     def misses(self):
 237         return self.requests - self.hits
 238
 239     def is_enabled(self):
 240         return cache_enabled and self.path is not None
 241
 242     def is_readonly(self):
 243         return cache_readonly
 244
 245     def get_cachedir_csig(self, node):
 246         cachedir, cachefile = self.cachepath(node)
 247         if cachefile and os.path.exists(cachefile):
 248             return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)
 249
 250     def cachepath(self, node):
 251         """
 252         """
 253         if not self.is_enabled():
 254             return None, None
 255
 256         sig = node.get_cachedir_bsig()
 257
 258         subdir = sig[:self.config['prefix_len']].upper()
 259
 260         dir = os.path.join(self.path, subdir)
 261         return dir, os.path.join(dir, sig)
 262
 263     def retrieve(self, node):
 264         """
 265         This method is called from multiple threads in a parallel build,
 266         so only do thread safe stuff here. Do thread unsafe stuff in
 267         built().
 268
 269         Note that there's a special trick here with the execute flag
 270         (one that's not normally done for other actions).  Basically
 271         if the user requested a no_exec (-n) build, then
 272         SCons.Action.execute_actions is set to 0 and when any action
 273         is called, it does its showing but then just returns zero
 274         instead of actually calling the action execution operation.
 275         The problem for caching is that if the file does NOT exist in
 276         cache then the CacheRetrieveString won't return anything to
 277         show for the task, but the Action.__call__ won't call
 278         CacheRetrieveFunc; instead it just returns zero, which makes
 279         the code below think that the file *was* successfully
 280         retrieved from the cache, therefore it doesn't do any
 281         subsequent building.  However, the CacheRetrieveString didn't
 282         print anything because it didn't actually exist in the cache,
 283         and no more build actions will be performed, so the user just
 284         sees nothing.  The fix is to tell Action.__call__ to always
 285         execute the CacheRetrieveFunc and then have the latter
 286         explicitly check SCons.Action.execute_actions itself.
 287         """
 288         if not self.is_enabled():
 289             return False
 290
 291         env = node.get_build_env()
 292         if cache_show:
 293             if CacheRetrieveSilent(node, [], env, execute=1) == 0:
 294                 node.build(presub=0, execute=0)
 295                 return True
 296         else:
 297             if CacheRetrieve(node, [], env, execute=1) == 0:
 298                 return True
 299
 300         return False
 301
 302     def push(self, node):
 303         if self.is_readonly() or not self.is_enabled():
 304             return
 305         return CachePush(node, [], node.get_build_env())
 306
 307     def push_if_forced(self, node):
 308         if cache_force:
 309             return self.push(node)
 310
 311 # Local Variables:
 312 # tab-width:4
 313 # indent-tabs-mode:nil
 314 # End:
 315 # vim: set expandtab tabstop=4 shiftwidth=4: