SCons/CacheDir.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """CacheDir support
  25 """
  26
  27 import atexit
  28 import json
  29 import os
  30 import stat
  31 import sys
  32 import uuid
  33
  34 import SCons.Action
  35 import SCons.Errors
  36 import SCons.Warnings
  37 import SCons.Util
  38
  39 cache_enabled = True
  40 cache_debug = False
  41 cache_force = False
  42 cache_show = False
  43 cache_readonly = False
  44 cache_tmp_uuid = uuid.uuid4().hex
  45
  46 def CacheRetrieveFunc(target, source, env) -> int:
  47     t = target[0]
  48     fs = t.fs
  49     cd = env.get_CacheDir()
  50     cd.requests += 1
  51     cachedir, cachefile = cd.cachepath(t)
  52     if not fs.exists(cachefile):
  53         cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
  54         return 1
  55     cd.hits += 1
  56     cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
  57     if SCons.Action.execute_actions:
  58         if fs.islink(cachefile):
  59             fs.symlink(fs.readlink(cachefile), t.get_internal_path())
  60         else:
  61             cd.copy_from_cache(env, cachefile, t.get_internal_path())
  62             try:
  63                 os.utime(cachefile, None)
  64             except OSError:
  65                 pass
  66         st = fs.stat(cachefile)
  67         fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
  68     return 0
  69
  70 def CacheRetrieveString(target, source, env) -> None:
  71     t = target[0]
  72     fs = t.fs
  73     cd = env.get_CacheDir()
  74     cachedir, cachefile = cd.cachepath(t)
  75     if t.fs.exists(cachefile):
  76         return "Retrieved `%s' from cache" % t.get_internal_path()
  77     return None
  78
  79 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
  80
  81 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
  82
  83 def CachePushFunc(target, source, env):
  84     if cache_readonly:
  85         return
  86
  87     t = target[0]
  88     if t.nocache:
  89         return
  90     fs = t.fs
  91     cd = env.get_CacheDir()
  92     cachedir, cachefile = cd.cachepath(t)
  93     if fs.exists(cachefile):
  94         # Don't bother copying it if it's already there.  Note that
  95         # usually this "shouldn't happen" because if the file already
  96         # existed in cache, we'd have retrieved the file from there,
  97         # not built it.  This can happen, though, in a race, if some
  98         # other person running the same build pushes their copy to
  99         # the cache after we decide we need to build it but before our
 100         # build completes.
 101         cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
 102         return
 103
 104     cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
 105
 106     tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid)
 107     errfmt = "Unable to copy %s to cache. Cache file is %s"
 108
 109     try:
 110         fs.makedirs(cachedir, exist_ok=True)
 111     except OSError:
 112         msg = errfmt % (str(target), cachefile)
 113         raise SCons.Errors.SConsEnvironmentError(msg)
 114     try:
 115         if fs.islink(t.get_internal_path()):
 116             fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
 117         else:
 118             cd.copy_to_cache(env, t.get_internal_path(), tempfile)
 119         fs.rename(tempfile, cachefile)
 120
 121     except OSError:
 122         # It's possible someone else tried writing the file at the
 123         # same time we did, or else that there was some problem like
 124         # the CacheDir being on a separate file system that's full.
 125         # In any case, inability to push a file to cache doesn't affect
 126         # the correctness of the build, so just print a warning.
 127         msg = errfmt % (str(t), cachefile)
 128         cd.CacheDebug(errfmt + '\n', str(t), cachefile)
 129         SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
 130
 131 CachePush = SCons.Action.Action(CachePushFunc, None)
 132
 133
 134 class CacheDir:
 135
 136     def __init__(self, path) -> None:
 137         """
 138         Initialize a CacheDir object.
 139
 140         The cache configuration is stored in the object. It
 141         is read from the config file in the supplied path if
 142         one exists,  if not the config file is created and
 143         the default config is written, as well as saved in the object.
 144         """
 145         self.requests = 0
 146         self.hits = 0
 147         self.path = path
 148         self.current_cache_debug = None
 149         self.debugFP = None
 150         self.config = dict()
 151         if path is None:
 152             return
 153
 154         self._readconfig(path)
 155
 156
 157     def _readconfig(self, path):
 158         """
 159         Read the cache config.
 160
 161         If directory or config file do not exist, create.  Take advantage
 162         of Py3 capability in os.makedirs() and in file open(): just try
 163         the operation and handle failure appropriately.
 164
 165         Omit the check for old cache format, assume that's old enough
 166         there will be none of those left to worry about.
 167
 168         :param path: path to the cache directory
 169         """
 170         config_file = os.path.join(path, 'config')
 171         try:
 172             # still use a try block even with exist_ok, might have other fails
 173             os.makedirs(path, exist_ok=True)
 174         except OSError:
 175             msg = "Failed to create cache directory " + path
 176             raise SCons.Errors.SConsEnvironmentError(msg)
 177
 178         try:
 179             with SCons.Util.FileLock(config_file, timeout=5, writer=True), open(
 180                 config_file, "x"
 181             ) as config:
 182                 self.config['prefix_len'] = 2
 183                 try:
 184                     json.dump(self.config, config)
 185                 except Exception:
 186                     msg = "Failed to write cache configuration for " + path
 187                     raise SCons.Errors.SConsEnvironmentError(msg)
 188         except FileExistsError:
 189             try:
 190                 with SCons.Util.FileLock(config_file, timeout=5, writer=False), open(
 191                     config_file
 192                 ) as config:
 193                     self.config = json.load(config)
 194             except (ValueError, json.decoder.JSONDecodeError):
 195                 msg = "Failed to read cache configuration for " + path
 196                 raise SCons.Errors.SConsEnvironmentError(msg)
 197
 198     def CacheDebug(self, fmt, target, cachefile) -> None:
 199         if cache_debug != self.current_cache_debug:
 200             if cache_debug == '-':
 201                 self.debugFP = sys.stdout
 202             elif cache_debug:
 203                 def debug_cleanup(debugFP) -> None:
 204                     debugFP.close()
 205
 206                 self.debugFP = open(cache_debug, 'w')
 207                 atexit.register(debug_cleanup, self.debugFP)
 208             else:
 209                 self.debugFP = None
 210             self.current_cache_debug = cache_debug
 211         if self.debugFP:
 212             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
 213             self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
 214                                (self.requests, self.hits, self.misses, self.hit_ratio))
 215
 216     @classmethod
 217     def copy_from_cache(cls, env, src, dst) -> str:
 218         """Copy a file from cache."""
 219         if env.cache_timestamp_newer:
 220             return env.fs.copy(src, dst)
 221         else:
 222             return env.fs.copy2(src, dst)
 223
 224     @classmethod
 225     def copy_to_cache(cls, env, src, dst) -> str:
 226         """Copy a file to cache.
 227
 228         Just use the FS copy2 ("with metadata") method, except do an additional
 229         check and if necessary a chmod to ensure the cachefile is writeable,
 230         to forestall permission problems if the cache entry is later updated.
 231         """
 232         try:
 233             result = env.fs.copy2(src, dst)
 234             st = stat.S_IMODE(os.stat(result).st_mode)
 235             if not st | stat.S_IWRITE:
 236                 os.chmod(dst, st | stat.S_IWRITE)
 237             return result
 238         except AttributeError as ex:
 239             raise OSError from ex
 240
 241     @property
 242     def hit_ratio(self) -> float:
 243         return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
 244
 245     @property
 246     def misses(self) -> int:
 247         return self.requests - self.hits
 248
 249     def is_enabled(self) -> bool:
 250         return cache_enabled and self.path is not None
 251
 252     def is_readonly(self) -> bool:
 253         return cache_readonly
 254
 255     def get_cachedir_csig(self, node):
 256         cachedir, cachefile = self.cachepath(node)
 257         if cachefile and os.path.exists(cachefile):
 258             return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)
 259
 260     def cachepath(self, node) -> tuple:
 261         """Return where to cache a file.
 262
 263         Given a Node, obtain the configured cache directory and
 264         the path to the cached file, which is generated from the
 265         node's build signature. If caching is not enabled for the
 266         None, return a tuple of None.
 267         """
 268         if not self.is_enabled():
 269             return None, None
 270
 271         sig = node.get_cachedir_bsig()
 272         subdir = sig[:self.config['prefix_len']].upper()
 273         cachedir = os.path.join(self.path, subdir)
 274         return cachedir, os.path.join(cachedir, sig)
 275
 276     def retrieve(self, node) -> bool:
 277         """Retrieve a node from cache.
 278
 279         Returns True if a successful retrieval resulted.
 280
 281         This method is called from multiple threads in a parallel build,
 282         so only do thread safe stuff here. Do thread unsafe stuff in
 283         built().
 284
 285         Note that there's a special trick here with the execute flag
 286         (one that's not normally done for other actions).  Basically
 287         if the user requested a no_exec (-n) build, then
 288         SCons.Action.execute_actions is set to 0 and when any action
 289         is called, it does its showing but then just returns zero
 290         instead of actually calling the action execution operation.
 291         The problem for caching is that if the file does NOT exist in
 292         cache then the CacheRetrieveString won't return anything to
 293         show for the task, but the Action.__call__ won't call
 294         CacheRetrieveFunc; instead it just returns zero, which makes
 295         the code below think that the file *was* successfully
 296         retrieved from the cache, therefore it doesn't do any
 297         subsequent building.  However, the CacheRetrieveString didn't
 298         print anything because it didn't actually exist in the cache,
 299         and no more build actions will be performed, so the user just
 300         sees nothing.  The fix is to tell Action.__call__ to always
 301         execute the CacheRetrieveFunc and then have the latter
 302         explicitly check SCons.Action.execute_actions itself.
 303         """
 304         if not self.is_enabled():
 305             return False
 306
 307         env = node.get_build_env()
 308         if cache_show:
 309             if CacheRetrieveSilent(node, [], env, execute=1) == 0:
 310                 node.build(presub=0, execute=0)
 311                 return True
 312         else:
 313             if CacheRetrieve(node, [], env, execute=1) == 0:
 314                 return True
 315
 316         return False
 317
 318     def push(self, node):
 319         if self.is_readonly() or not self.is_enabled():
 320             return
 321         return CachePush(node, [], node.get_build_env())
 322
 323     def push_if_forced(self, node):
 324         if cache_force:
 325             return self.push(node)
 326
 327 # Local Variables:
 328 # tab-width:4
 329 # indent-tabs-mode:nil
 330 # End:
 331 # vim: set expandtab tabstop=4 shiftwidth=4: