SCons/CacheDir.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """CacheDir support
  25 """
  26
  27 import atexit
  28 import json
  29 import os
  30 import stat
  31 import sys
  32 import uuid
  33
  34 import SCons.Action
  35 import SCons.Errors
  36 import SCons.Warnings
  37 import SCons
  38
  39 cache_enabled = True
  40 cache_debug = False
  41 cache_force = False
  42 cache_show = False
  43 cache_readonly = False
  44 cache_tmp_uuid = uuid.uuid4().hex
  45
  46 def CacheRetrieveFunc(target, source, env) -> int:
  47     t = target[0]
  48     fs = t.fs
  49     cd = env.get_CacheDir()
  50     cd.requests += 1
  51     cachedir, cachefile = cd.cachepath(t)
  52     if not fs.exists(cachefile):
  53         cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
  54         return 1
  55     cd.hits += 1
  56     cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
  57     if SCons.Action.execute_actions:
  58         if fs.islink(cachefile):
  59             fs.symlink(fs.readlink(cachefile), t.get_internal_path())
  60         else:
  61             cd.copy_from_cache(env, cachefile, t.get_internal_path())
  62             try:
  63                 os.utime(cachefile, None)
  64             except OSError:
  65                 pass
  66         st = fs.stat(cachefile)
  67         fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
  68     return 0
  69
  70 def CacheRetrieveString(target, source, env) -> None:
  71     t = target[0]
  72     fs = t.fs
  73     cd = env.get_CacheDir()
  74     cachedir, cachefile = cd.cachepath(t)
  75     if t.fs.exists(cachefile):
  76         return "Retrieved `%s' from cache" % t.get_internal_path()
  77     return None
  78
  79 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
  80
  81 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
  82
  83 def CachePushFunc(target, source, env):
  84     if cache_readonly:
  85         return
  86
  87     t = target[0]
  88     if t.nocache:
  89         return
  90     fs = t.fs
  91     cd = env.get_CacheDir()
  92     cachedir, cachefile = cd.cachepath(t)
  93     if fs.exists(cachefile):
  94         # Don't bother copying it if it's already there.  Note that
  95         # usually this "shouldn't happen" because if the file already
  96         # existed in cache, we'd have retrieved the file from there,
  97         # not built it.  This can happen, though, in a race, if some
  98         # other person running the same build pushes their copy to
  99         # the cache after we decide we need to build it but before our
 100         # build completes.
 101         cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
 102         return
 103
 104     cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
 105
 106     tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid)
 107     errfmt = "Unable to copy %s to cache. Cache file is %s"
 108
 109     try:
 110         fs.makedirs(cachedir, exist_ok=True)
 111     except OSError:
 112         msg = errfmt % (str(target), cachefile)
 113         raise SCons.Errors.SConsEnvironmentError(msg)
 114     try:
 115         if fs.islink(t.get_internal_path()):
 116             fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
 117         else:
 118             cd.copy_to_cache(env, t.get_internal_path(), tempfile)
 119         fs.rename(tempfile, cachefile)
 120
 121     except OSError:
 122         # It's possible someone else tried writing the file at the
 123         # same time we did, or else that there was some problem like
 124         # the CacheDir being on a separate file system that's full.
 125         # In any case, inability to push a file to cache doesn't affect
 126         # the correctness of the build, so just print a warning.
 127         msg = errfmt % (str(t), cachefile)
 128         cd.CacheDebug(errfmt + '\n', str(t), cachefile)
 129         SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
 130
 131 CachePush = SCons.Action.Action(CachePushFunc, None)
 132
 133
 134 class CacheDir:
 135
 136     def __init__(self, path) -> None:
 137         """
 138         Initialize a CacheDir object.
 139
 140         The cache configuration is stored in the object. It
 141         is read from the config file in the supplied path if
 142         one exists,  if not the config file is created and
 143         the default config is written, as well as saved in the object.
 144         """
 145         self.requests = 0
 146         self.hits = 0
 147         self.path = path
 148         self.current_cache_debug = None
 149         self.debugFP = None
 150         self.config = dict()
 151         if path is None:
 152             return
 153
 154         self._readconfig(path)
 155
 156
 157     def _readconfig(self, path):
 158         """
 159         Read the cache config.
 160
 161         If directory or config file do not exist, create.  Take advantage
 162         of Py3 capability in os.makedirs() and in file open(): just try
 163         the operation and handle failure appropriately.
 164
 165         Omit the check for old cache format, assume that's old enough
 166         there will be none of those left to worry about.
 167
 168         :param path: path to the cache directory
 169         """
 170         config_file = os.path.join(path, 'config')
 171         try:
 172             os.makedirs(path, exist_ok=True)
 173         except FileExistsError:
 174             pass
 175         except OSError:
 176             msg = "Failed to create cache directory " + path
 177             raise SCons.Errors.SConsEnvironmentError(msg)
 178
 179         try:
 180             with open(config_file, 'x') as config:
 181                 self.config['prefix_len'] = 2
 182                 try:
 183                     json.dump(self.config, config)
 184                 except Exception:
 185                     msg = "Failed to write cache configuration for " + path
 186                     raise SCons.Errors.SConsEnvironmentError(msg)
 187         except FileExistsError:
 188             try:
 189                 with open(config_file) as config:
 190                     self.config = json.load(config)
 191             except ValueError:
 192                 msg = "Failed to read cache configuration for " + path
 193                 raise SCons.Errors.SConsEnvironmentError(msg)
 194
 195     def CacheDebug(self, fmt, target, cachefile) -> None:
 196         if cache_debug != self.current_cache_debug:
 197             if cache_debug == '-':
 198                 self.debugFP = sys.stdout
 199             elif cache_debug:
 200                 def debug_cleanup(debugFP) -> None:
 201                     debugFP.close()
 202
 203                 self.debugFP = open(cache_debug, 'w')
 204                 atexit.register(debug_cleanup, self.debugFP)
 205             else:
 206                 self.debugFP = None
 207             self.current_cache_debug = cache_debug
 208         if self.debugFP:
 209             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
 210             self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
 211                                (self.requests, self.hits, self.misses, self.hit_ratio))
 212
 213     @classmethod
 214     def copy_from_cache(cls, env, src, dst) -> str:
 215         """Copy a file from cache."""
 216         if env.cache_timestamp_newer:
 217             return env.fs.copy(src, dst)
 218         else:
 219             return env.fs.copy2(src, dst)
 220
 221     @classmethod
 222     def copy_to_cache(cls, env, src, dst) -> str:
 223         """Copy a file to cache.
 224
 225         Just use the FS copy2 ("with metadata") method, except do an additional
 226         check and if necessary a chmod to ensure the cachefile is writeable,
 227         to forestall permission problems if the cache entry is later updated.
 228         """
 229         try:
 230             result = env.fs.copy2(src, dst)
 231             st = stat.S_IMODE(os.stat(result).st_mode)
 232             if not st | stat.S_IWRITE:
 233                 os.chmod(dst, st | stat.S_IWRITE)
 234             return result
 235         except AttributeError as ex:
 236             raise OSError from ex
 237
 238     @property
 239     def hit_ratio(self) -> float:
 240         return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
 241
 242     @property
 243     def misses(self) -> int:
 244         return self.requests - self.hits
 245
 246     def is_enabled(self) -> bool:
 247         return cache_enabled and self.path is not None
 248
 249     def is_readonly(self) -> bool:
 250         return cache_readonly
 251
 252     def get_cachedir_csig(self, node):
 253         cachedir, cachefile = self.cachepath(node)
 254         if cachefile and os.path.exists(cachefile):
 255             return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)
 256
 257     def cachepath(self, node) -> tuple:
 258         """Return where to cache a file.
 259
 260         Given a Node, obtain the configured cache directory and
 261         the path to the cached file, which is generated from the
 262         node's build signature. If caching is not enabled for the
 263         None, return a tuple of None.
 264         """
 265         if not self.is_enabled():
 266             return None, None
 267
 268         sig = node.get_cachedir_bsig()
 269         subdir = sig[:self.config['prefix_len']].upper()
 270         cachedir = os.path.join(self.path, subdir)
 271         return cachedir, os.path.join(cachedir, sig)
 272
 273     def retrieve(self, node) -> bool:
 274         """Retrieve a node from cache.
 275
 276         Returns True if a successful retrieval resulted.
 277
 278         This method is called from multiple threads in a parallel build,
 279         so only do thread safe stuff here. Do thread unsafe stuff in
 280         built().
 281
 282         Note that there's a special trick here with the execute flag
 283         (one that's not normally done for other actions).  Basically
 284         if the user requested a no_exec (-n) build, then
 285         SCons.Action.execute_actions is set to 0 and when any action
 286         is called, it does its showing but then just returns zero
 287         instead of actually calling the action execution operation.
 288         The problem for caching is that if the file does NOT exist in
 289         cache then the CacheRetrieveString won't return anything to
 290         show for the task, but the Action.__call__ won't call
 291         CacheRetrieveFunc; instead it just returns zero, which makes
 292         the code below think that the file *was* successfully
 293         retrieved from the cache, therefore it doesn't do any
 294         subsequent building.  However, the CacheRetrieveString didn't
 295         print anything because it didn't actually exist in the cache,
 296         and no more build actions will be performed, so the user just
 297         sees nothing.  The fix is to tell Action.__call__ to always
 298         execute the CacheRetrieveFunc and then have the latter
 299         explicitly check SCons.Action.execute_actions itself.
 300         """
 301         if not self.is_enabled():
 302             return False
 303
 304         env = node.get_build_env()
 305         if cache_show:
 306             if CacheRetrieveSilent(node, [], env, execute=1) == 0:
 307                 node.build(presub=0, execute=0)
 308                 return True
 309         else:
 310             if CacheRetrieve(node, [], env, execute=1) == 0:
 311                 return True
 312
 313         return False
 314
 315     def push(self, node):
 316         if self.is_readonly() or not self.is_enabled():
 317             return
 318         return CachePush(node, [], node.get_build_env())
 319
 320     def push_if_forced(self, node):
 321         if cache_force:
 322             return self.push(node)
 323
 324 # Local Variables:
 325 # tab-width:4
 326 # indent-tabs-mode:nil
 327 # End:
 328 # vim: set expandtab tabstop=4 shiftwidth=4: