SCons/CacheDir.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """CacheDir support
  25 """
  26
  27 import atexit
  28 import json
  29 import os
  30 import stat
  31 import sys
  32 import tempfile
  33 import uuid
  34
  35 import SCons.Action
  36 import SCons.Errors
  37 import SCons.Warnings
  38 import SCons.Util
  39
  40 CACHE_PREFIX_LEN = 2  # first two characters used as subdirectory name
  41 CACHE_TAG = (
  42     b"Signature: 8a477f597d28d172789f06886806bc55\n"
  43     b"# SCons cache directory - see https://bford.info/cachedir/\n"
  44 )
  45
  46 cache_enabled = True
  47 cache_debug = False
  48 cache_force = False
  49 cache_show = False
  50 cache_readonly = False
  51 cache_tmp_uuid = uuid.uuid4().hex
  52
  53 def CacheRetrieveFunc(target, source, env) -> int:
  54     t = target[0]
  55     fs = t.fs
  56     cd = env.get_CacheDir()
  57     cd.requests += 1
  58     cachedir, cachefile = cd.cachepath(t)
  59     if not fs.exists(cachefile):
  60         cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
  61         return 1
  62     cd.hits += 1
  63     cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
  64     if SCons.Action.execute_actions:
  65         if fs.islink(cachefile):
  66             fs.symlink(fs.readlink(cachefile), t.get_internal_path())
  67         else:
  68             cd.copy_from_cache(env, cachefile, t.get_internal_path())
  69             try:
  70                 os.utime(cachefile, None)
  71             except OSError:
  72                 pass
  73         st = fs.stat(cachefile)
  74         fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
  75     return 0
  76
  77 def CacheRetrieveString(target, source, env) -> str:
  78     t = target[0]
  79     fs = t.fs
  80     cd = env.get_CacheDir()
  81     cachedir, cachefile = cd.cachepath(t)
  82     if t.fs.exists(cachefile):
  83         return "Retrieved `%s' from cache" % t.get_internal_path()
  84     return ""
  85
  86 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
  87
  88 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
  89
  90 def CachePushFunc(target, source, env) -> None:
  91     if cache_readonly:
  92         return
  93
  94     t = target[0]
  95     if t.nocache:
  96         return
  97     fs = t.fs
  98     cd = env.get_CacheDir()
  99     cachedir, cachefile = cd.cachepath(t)
 100     if fs.exists(cachefile):
 101         # Don't bother copying it if it's already there.  Note that
 102         # usually this "shouldn't happen" because if the file already
 103         # existed in cache, we'd have retrieved the file from there,
 104         # not built it.  This can happen, though, in a race, if some
 105         # other person running the same build pushes their copy to
 106         # the cache after we decide we need to build it but before our
 107         # build completes.
 108         cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
 109         return
 110
 111     cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
 112
 113     tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid)
 114     errfmt = "Unable to copy %s to cache. Cache file is %s"
 115
 116     try:
 117         fs.makedirs(cachedir, exist_ok=True)
 118     except OSError:
 119         msg = errfmt % (str(target), cachefile)
 120         raise SCons.Errors.SConsEnvironmentError(msg)
 121     try:
 122         if fs.islink(t.get_internal_path()):
 123             fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
 124         else:
 125             cd.copy_to_cache(env, t.get_internal_path(), tempfile)
 126         fs.rename(tempfile, cachefile)
 127
 128     except OSError:
 129         # It's possible someone else tried writing the file at the
 130         # same time we did, or else that there was some problem like
 131         # the CacheDir being on a separate file system that's full.
 132         # In any case, inability to push a file to cache doesn't affect
 133         # the correctness of the build, so just print a warning.
 134         msg = errfmt % (str(t), cachefile)
 135         cd.CacheDebug(errfmt + '\n', str(t), cachefile)
 136         SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
 137
 138 CachePush = SCons.Action.Action(CachePushFunc, None)
 139
 140
 141 class CacheDir:
 142
 143     def __init__(self, path) -> None:
 144         """Initialize a CacheDir object.
 145
 146         The cache configuration is stored in the object. It
 147         is read from the config file in the supplied path if
 148         one exists,  if not the config file is created and
 149         the default config is written, as well as saved in the object.
 150         """
 151         self.requests = 0
 152         self.hits = 0
 153         self.path = path
 154         self.current_cache_debug = None
 155         self.debugFP = None
 156         self.config = {}
 157         if path is not None:
 158             self._readconfig(path)
 159
 160     def _add_config(self, path: str) -> None:
 161         """Create the cache config file in *path*.
 162
 163         Locking isn't necessary in the normal case - when the cachedir is
 164         being created - because it's written to a unique directory first,
 165         before the directory is renamed. But it is legal to call CacheDir
 166         with an existing directory, which may be missing the config file,
 167         and in that case we do need locking. Simpler to always lock.
 168         """
 169         config_file = os.path.join(path, 'config')
 170         # TODO: this breaks the "unserializable config object" test which
 171         #   does some crazy stuff, so for now don't use setdefault. It does
 172         #   seem like it would be better to preserve an exisiting value.
 173         # self.config.setdefault('prefix_len', CACHE_PREFIX_LEN)
 174         self.config['prefix_len'] = CACHE_PREFIX_LEN
 175         with SCons.Util.FileLock(config_file, timeout=5, writer=True), open(
 176             config_file, "x"
 177         ) as config:
 178             try:
 179                 json.dump(self.config, config)
 180             except Exception:
 181                 msg = "Failed to write cache configuration for " + path
 182                 raise SCons.Errors.SConsEnvironmentError(msg)
 183
 184         # Add the tag file "carelessly" - the contents are not used by SCons
 185         # so we don't care about the chance of concurrent writes.
 186         try:
 187             tagfile = os.path.join(path, "CACHEDIR.TAG")
 188             with open(tagfile, 'xb') as cachedir_tag:
 189                 cachedir_tag.write(CACHE_TAG)
 190         except FileExistsError:
 191             pass
 192
 193     def _mkdir_atomic(self, path: str) -> bool:
 194         """Create cache directory at *path*.
 195
 196         Uses directory renaming to avoid races.  If we are actually
 197         creating the dir, populate it with the metadata files at the
 198         same time as that's the safest way. But it's not illegal to point
 199         CacheDir at an existing directory that wasn't a cache previously,
 200         so we may have to do that elsewhere, too.
 201
 202         Returns:
 203             ``True`` if it we created the dir, ``False`` if already existed,
 204
 205         Raises:
 206             SConsEnvironmentError: if we tried and failed to create the cache.
 207         """
 208         directory = os.path.abspath(path)
 209         if os.path.exists(directory):
 210             return False
 211
 212         try:
 213             tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(directory))
 214         except OSError as e:
 215             msg = "Failed to create cache directory " + path
 216             raise SCons.Errors.SConsEnvironmentError(msg) from e
 217         self._add_config(tempdir.name)
 218         with tempdir:
 219             try:
 220                 os.rename(tempdir.name, directory)
 221                 return True
 222             except Exception as e:
 223                 # did someone else get there first?
 224                 if os.path.isdir(directory):
 225                     return False
 226                 msg = "Failed to create cache directory " + path
 227                 raise SCons.Errors.SConsEnvironmentError(msg) from e
 228
 229     def _readconfig(self, path: str) -> None:
 230         """Read the cache config from *path*.
 231
 232         If directory or config file do not exist, create and populate.
 233         """
 234         config_file = os.path.join(path, 'config')
 235         created = self._mkdir_atomic(path)
 236         if not created and not os.path.isfile(config_file):
 237             # Could have been passed an empty directory
 238             self._add_config(path)
 239         try:
 240             with SCons.Util.FileLock(config_file, timeout=5, writer=False), open(
 241                 config_file
 242             ) as config:
 243                 self.config = json.load(config)
 244         except (ValueError, json.decoder.JSONDecodeError):
 245             msg = "Failed to read cache configuration for " + path
 246             raise SCons.Errors.SConsEnvironmentError(msg)
 247
 248     def CacheDebug(self, fmt, target, cachefile) -> None:
 249         if cache_debug != self.current_cache_debug:
 250             if cache_debug == '-':
 251                 self.debugFP = sys.stdout
 252             elif cache_debug:
 253                 def debug_cleanup(debugFP) -> None:
 254                     debugFP.close()
 255
 256                 self.debugFP = open(cache_debug, 'w')
 257                 atexit.register(debug_cleanup, self.debugFP)
 258             else:
 259                 self.debugFP = None
 260             self.current_cache_debug = cache_debug
 261         if self.debugFP:
 262             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
 263             self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
 264                                (self.requests, self.hits, self.misses, self.hit_ratio))
 265
 266     @classmethod
 267     def copy_from_cache(cls, env, src, dst) -> str:
 268         """Copy a file from cache."""
 269         if env.cache_timestamp_newer:
 270             return env.fs.copy(src, dst)
 271         else:
 272             return env.fs.copy2(src, dst)
 273
 274     @classmethod
 275     def copy_to_cache(cls, env, src, dst) -> str:
 276         """Copy a file to cache.
 277
 278         Just use the FS copy2 ("with metadata") method, except do an additional
 279         check and if necessary a chmod to ensure the cachefile is writeable,
 280         to forestall permission problems if the cache entry is later updated.
 281         """
 282         try:
 283             result = env.fs.copy2(src, dst)
 284             st = stat.S_IMODE(os.stat(result).st_mode)
 285             if not st | stat.S_IWRITE:
 286                 os.chmod(dst, st | stat.S_IWRITE)
 287             return result
 288         except AttributeError as ex:
 289             raise OSError from ex
 290
 291     @property
 292     def hit_ratio(self) -> float:
 293         return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
 294
 295     @property
 296     def misses(self) -> int:
 297         return self.requests - self.hits
 298
 299     def is_enabled(self) -> bool:
 300         return cache_enabled and self.path is not None
 301
 302     def is_readonly(self) -> bool:
 303         return cache_readonly
 304
 305     def get_cachedir_csig(self, node) -> str:
 306         cachedir, cachefile = self.cachepath(node)
 307         if cachefile and os.path.exists(cachefile):
 308             return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)
 309
 310     def cachepath(self, node) -> tuple:
 311         """Return where to cache a file.
 312
 313         Given a Node, obtain the configured cache directory and
 314         the path to the cached file, which is generated from the
 315         node's build signature. If caching is not enabled for the
 316         None, return a tuple of None.
 317         """
 318         if not self.is_enabled():
 319             return None, None
 320
 321         sig = node.get_cachedir_bsig()
 322         subdir = sig[:self.config['prefix_len']].upper()
 323         cachedir = os.path.join(self.path, subdir)
 324         return cachedir, os.path.join(cachedir, sig)
 325
 326     def retrieve(self, node) -> bool:
 327         """Retrieve a node from cache.
 328
 329         Returns True if a successful retrieval resulted.
 330
 331         This method is called from multiple threads in a parallel build,
 332         so only do thread safe stuff here. Do thread unsafe stuff in
 333         built().
 334
 335         Note that there's a special trick here with the execute flag
 336         (one that's not normally done for other actions).  Basically
 337         if the user requested a no_exec (-n) build, then
 338         SCons.Action.execute_actions is set to 0 and when any action
 339         is called, it does its showing but then just returns zero
 340         instead of actually calling the action execution operation.
 341         The problem for caching is that if the file does NOT exist in
 342         cache then the CacheRetrieveString won't return anything to
 343         show for the task, but the Action.__call__ won't call
 344         CacheRetrieveFunc; instead it just returns zero, which makes
 345         the code below think that the file *was* successfully
 346         retrieved from the cache, therefore it doesn't do any
 347         subsequent building.  However, the CacheRetrieveString didn't
 348         print anything because it didn't actually exist in the cache,
 349         and no more build actions will be performed, so the user just
 350         sees nothing.  The fix is to tell Action.__call__ to always
 351         execute the CacheRetrieveFunc and then have the latter
 352         explicitly check SCons.Action.execute_actions itself.
 353         """
 354         if not self.is_enabled():
 355             return False
 356
 357         env = node.get_build_env()
 358         if cache_show:
 359             if CacheRetrieveSilent(node, [], env, execute=1) == 0:
 360                 node.build(presub=0, execute=0)
 361                 return True
 362         else:
 363             if CacheRetrieve(node, [], env, execute=1) == 0:
 364                 return True
 365
 366         return False
 367
 368     def push(self, node):
 369         if self.is_readonly() or not self.is_enabled():
 370             return
 371         return CachePush(node, [], node.get_build_env())
 372
 373     def push_if_forced(self, node):
 374         if cache_force:
 375             return self.push(node)
 376
 377 # Local Variables:
 378 # tab-width:4
 379 # indent-tabs-mode:nil
 380 # End:
 381 # vim: set expandtab tabstop=4 shiftwidth=4: