SCons/SConsign.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """Operations on signature database files (.sconsign). """
  25
  26 import SCons.compat  # pylint: disable=wrong-import-order
  27
  28 import os
  29 import pickle
  30 import time
  31
  32 import SCons.dblite
  33 import SCons.Warnings
  34 from SCons.compat import PICKLE_PROTOCOL
  35 from SCons.Util import print_time
  36
  37
  38 def corrupt_dblite_warning(filename) -> None:
  39     SCons.Warnings.warn(
  40         SCons.Warnings.CorruptSConsignWarning,
  41         "Ignoring corrupt .sconsign file: %s" % filename,
  42     )
  43
  44 SCons.dblite.IGNORE_CORRUPT_DBFILES = True
  45 SCons.dblite.corruption_warning = corrupt_dblite_warning
  46
  47 # XXX Get rid of the global array so this becomes re-entrant.
  48 sig_files = []
  49
  50 # Info for the database SConsign implementation (now the default):
  51 # "DataBase" is a dictionary that maps top-level SConstruct directories
  52 # to open database handles.
  53 # "DB_Module" is the Python database module to create the handles.
  54 # "DB_Name" is the base name of the database file (minus any
  55 # extension the underlying DB module will add).
  56 DataBase = {}
  57 DB_Module = SCons.dblite
  58 DB_Name = None
  59 DB_sync_list = []
  60
  61 def current_sconsign_filename():
  62     hash_format = SCons.Util.get_hash_format()
  63     current_hash_algorithm = SCons.Util.get_current_hash_algorithm_used()
  64     # if the user left the options defaulted AND the default algorithm set by
  65     # SCons is md5, then set the database name to be the special default name
  66     #
  67     # otherwise, if it defaults to something like 'sha1' or the user explicitly
  68     # set 'md5' as the hash format, set the database name to .sconsign_<algorithm>
  69     # eg .sconsign_sha1, etc.
  70     if hash_format is None and current_hash_algorithm == 'md5':
  71         return ".sconsign"
  72     return ".sconsign_" + current_hash_algorithm
  73
  74 def Get_DataBase(dir):
  75     global DB_Name
  76
  77     if DB_Name is None:
  78         DB_Name = current_sconsign_filename()
  79
  80     top = dir.fs.Top
  81     if not os.path.isabs(DB_Name) and top.repositories:
  82         mode = "c"
  83         for d in [top] + top.repositories:
  84             if dir.is_under(d):
  85                 try:
  86                     return DataBase[d], mode
  87                 except KeyError:
  88                     path = d.entry_abspath(DB_Name)
  89                     try: db = DataBase[d] = DB_Module.open(path, mode)
  90                     except OSError:
  91                         pass
  92                     else:
  93                         if mode != "r":
  94                             DB_sync_list.append(db)
  95                         return db, mode
  96             mode = "r"
  97     try:
  98         return DataBase[top], "c"
  99     except KeyError:
 100         db = DataBase[top] = DB_Module.open(DB_Name, "c")
 101         DB_sync_list.append(db)
 102         return db, "c"
 103     except TypeError:
 104         print("DataBase =", DataBase)
 105         raise
 106
 107
 108 def Reset() -> None:
 109     """Reset global state.  Used by unit tests that end up using
 110     SConsign multiple times to get a clean slate for each test."""
 111     global sig_files, DB_sync_list
 112     sig_files = []
 113     DB_sync_list = []
 114
 115
 116 normcase = os.path.normcase
 117
 118
 119 def write() -> None:
 120     if print_time():
 121         start_time = time.perf_counter()
 122
 123     for sig_file in sig_files:
 124         sig_file.write(sync=0)
 125     for db in DB_sync_list:
 126         try:
 127             syncmethod = db.sync
 128         except AttributeError:
 129             pass # Not all dbm modules have sync() methods.
 130         else:
 131             syncmethod()
 132         try:
 133             closemethod = db.close
 134         except AttributeError:
 135             pass # Not all dbm modules have close() methods.
 136         else:
 137             closemethod()
 138
 139     if print_time():
 140         elapsed = time.perf_counter() - start_time
 141         print('Total SConsign sync time: %f seconds' % elapsed)
 142
 143
 144 class SConsignEntry:
 145     """
 146     Wrapper class for the generic entry in a .sconsign file.
 147     The Node subclass populates it with attributes as it pleases.
 148
 149     XXX As coded below, we do expect a '.binfo' attribute to be added,
 150     but we'll probably generalize this in the next refactorings.
 151     """
 152     __slots__ = ("binfo", "ninfo", "__weakref__")
 153     current_version_id = 2
 154
 155     def __init__(self) -> None:
 156         # Create an object attribute from the class attribute so it ends up
 157         # in the pickled data in the .sconsign file.
 158         #_version_id = self.current_version_id
 159         pass
 160
 161     def convert_to_sconsign(self) -> None:
 162         self.binfo.convert_to_sconsign()
 163
 164     def convert_from_sconsign(self, dir, name) -> None:
 165         self.binfo.convert_from_sconsign(dir, name)
 166
 167     def __getstate__(self):
 168         state = getattr(self, '__dict__', {}).copy()
 169         for obj in type(self).mro():
 170             for name in getattr(obj, '__slots__', ()):
 171                 if hasattr(self, name):
 172                     state[name] = getattr(self, name)
 173
 174         state['_version_id'] = self.current_version_id
 175         try:
 176             del state['__weakref__']
 177         except KeyError:
 178             pass
 179         return state
 180
 181     def __setstate__(self, state) -> None:
 182         for key, value in state.items():
 183             if key not in ('_version_id', '__weakref__'):
 184                 setattr(self, key, value)
 185
 186
 187 class Base:
 188     """
 189     This is the controlling class for the signatures for the collection of
 190     entries associated with a specific directory.  The actual directory
 191     association will be maintained by a subclass that is specific to
 192     the underlying storage method.  This class provides a common set of
 193     methods for fetching and storing the individual bits of information
 194     that make up signature entry.
 195     """
 196     def __init__(self) -> None:
 197         self.entries = {}
 198         self.dirty = False
 199         self.to_be_merged = {}
 200
 201     def get_entry(self, filename):
 202         """
 203         Fetch the specified entry attribute.
 204         """
 205         return self.entries[filename]
 206
 207     def set_entry(self, filename, obj) -> None:
 208         """
 209         Set the entry.
 210         """
 211         self.entries[filename] = obj
 212         self.dirty = True
 213
 214     def do_not_set_entry(self, filename, obj) -> None:
 215         pass
 216
 217     def store_info(self, filename, node) -> None:
 218         entry = node.get_stored_info()
 219         entry.binfo.merge(node.get_binfo())
 220         self.to_be_merged[filename] = node
 221         self.dirty = True
 222
 223     def do_not_store_info(self, filename, node) -> None:
 224         pass
 225
 226     def merge(self) -> None:
 227         for key, node in self.to_be_merged.items():
 228             entry = node.get_stored_info()
 229             try:
 230                 ninfo = entry.ninfo
 231             except AttributeError:
 232                 # This happens with SConf Nodes, because the configuration
 233                 # subsystem takes direct control over how the build decision
 234                 # is made and its information stored.
 235                 pass
 236             else:
 237                 ninfo.merge(node.get_ninfo())
 238             self.entries[key] = entry
 239         self.to_be_merged = {}
 240
 241
 242 class DB(Base):
 243     """
 244     A Base subclass that reads and writes signature information
 245     from a global .sconsign.db* file--the actual file suffix is
 246     determined by the database module.
 247     """
 248     def __init__(self, dir) -> None:
 249         super().__init__()
 250
 251         self.dir = dir
 252
 253         db, mode = Get_DataBase(dir)
 254
 255         # Read using the path relative to the top of the Repository
 256         # (self.dir.tpath) from which we're fetching the signature
 257         # information.
 258         path = normcase(dir.get_tpath())
 259         try:
 260             rawentries = db[path]
 261         except KeyError:
 262             pass
 263         else:
 264             try:
 265                 self.entries = pickle.loads(rawentries)
 266                 if not isinstance(self.entries, dict):
 267                     self.entries = {}
 268                     raise TypeError
 269             except KeyboardInterrupt:
 270                 raise
 271             except Exception as e:
 272                 SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
 273                                     "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e))
 274             for key, entry in self.entries.items():
 275                 entry.convert_from_sconsign(dir, key)
 276
 277         if mode == "r":
 278             # This directory is actually under a repository, which means
 279             # likely they're reaching in directly for a dependency on
 280             # a file there.  Don't actually set any entry info, so we
 281             # won't try to write to that .sconsign.dblite file.
 282             self.set_entry = self.do_not_set_entry
 283             self.store_info = self.do_not_store_info
 284
 285         sig_files.append(self)
 286
 287     def write(self, sync: int=1) -> None:
 288         if not self.dirty:
 289             return
 290
 291         self.merge()
 292
 293         db, mode = Get_DataBase(self.dir)
 294
 295         # Write using the path relative to the top of the SConstruct
 296         # directory (self.dir.path), not relative to the top of
 297         # the Repository; we only write to our own .sconsign file,
 298         # not to .sconsign files in Repositories.
 299         path = normcase(self.dir.get_internal_path())
 300         for key, entry in self.entries.items():
 301             entry.convert_to_sconsign()
 302         db[path] = pickle.dumps(self.entries, PICKLE_PROTOCOL)
 303
 304         if sync:
 305             try:
 306                 syncmethod = db.sync
 307             except AttributeError:
 308                 # Not all anydbm modules have sync() methods.
 309                 pass
 310             else:
 311                 syncmethod()
 312
 313
 314 class Dir(Base):
 315     def __init__(self, fp=None, dir=None) -> None:
 316         """fp - file pointer to read entries from."""
 317         super().__init__()
 318
 319         if not fp:
 320             return
 321
 322         self.entries = pickle.load(fp)
 323         if not isinstance(self.entries, dict):
 324             self.entries = {}
 325             raise TypeError
 326
 327         if dir:
 328             for key, entry in self.entries.items():
 329                 entry.convert_from_sconsign(dir, key)
 330
 331
 332 class DirFile(Dir):
 333     """Encapsulates reading and writing a per-directory .sconsign file."""
 334     def __init__(self, dir) -> None:
 335         """dir - the directory for the file."""
 336
 337         self.dir = dir
 338         self.sconsign = os.path.join(dir.get_internal_path(), current_sconsign_filename())
 339
 340         try:
 341             fp = open(self.sconsign, 'rb')
 342         except OSError:
 343             fp = None
 344
 345         try:
 346             super().__init__(fp, dir)
 347         except KeyboardInterrupt:
 348             raise
 349         except Exception:
 350             SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
 351                                 "Ignoring corrupt .sconsign file: %s"%self.sconsign)
 352
 353         try:
 354             fp.close()
 355         except AttributeError:
 356             pass
 357
 358         sig_files.append(self)
 359
 360     def write(self, sync: int=1) -> None:
 361         """Write the .sconsign file to disk.
 362
 363         Try to write to a temporary file first, and rename it if we
 364         succeed.  If we can't write to the temporary file, it's
 365         probably because the directory isn't writable (and if so,
 366         how did we build anything in this directory, anyway?), so
 367         try to write directly to the .sconsign file as a backup.
 368         If we can't rename, try to copy the temporary contents back
 369         to the .sconsign file.  Either way, always try to remove
 370         the temporary file at the end.
 371         """
 372         if not self.dirty:
 373             return
 374
 375         self.merge()
 376
 377         temp = os.path.join(self.dir.get_internal_path(), '.scons%d' % os.getpid())
 378         try:
 379             file = open(temp, 'wb')
 380             fname = temp
 381         except OSError:
 382             try:
 383                 file = open(self.sconsign, 'wb')
 384                 fname = self.sconsign
 385             except OSError:
 386                 return
 387         for key, entry in self.entries.items():
 388             entry.convert_to_sconsign()
 389         pickle.dump(self.entries, file, PICKLE_PROTOCOL)
 390         file.close()
 391         if fname != self.sconsign:
 392             try:
 393                 mode = os.stat(self.sconsign)[0]
 394                 os.chmod(self.sconsign, 0o666)
 395                 os.unlink(self.sconsign)
 396             except OSError:
 397                 # Try to carry on in the face of either OSError
 398                 # (things like permission issues) or IOError (disk
 399                 # or network issues).  If there's a really dangerous
 400                 # issue, it should get re-raised by the calls below.
 401                 pass
 402             try:
 403                 os.rename(fname, self.sconsign)
 404             except OSError:
 405                 # An OSError failure to rename may indicate something
 406                 # like the directory has no write permission, but
 407                 # the .sconsign file itself might still be writable,
 408                 # so try writing on top of it directly.  An IOError
 409                 # here, or in any of the following calls, would get
 410                 # raised, indicating something like a potentially
 411                 # serious disk or network issue.
 412                 with open(self.sconsign, 'wb') as f, open(fname, 'rb') as f2:
 413                     f.write(f2.read())
 414                 os.chmod(self.sconsign, mode)
 415         try:
 416             os.unlink(temp)
 417         except OSError:
 418             pass
 419
 420 ForDirectory = DB
 421
 422
 423 def File(name, dbm_module=None) -> None:
 424     """
 425     Arrange for all signatures to be stored in a global .sconsign.db*
 426     file.
 427     """
 428     global ForDirectory, DB_Name, DB_Module
 429     if name is None:
 430         ForDirectory = DirFile
 431         DB_Module = None
 432     else:
 433         ForDirectory = DB
 434         DB_Name = name
 435         if dbm_module is not None:
 436             DB_Module = dbm_module
 437
 438 # Local Variables:
 439 # tab-width:4
 440 # indent-tabs-mode:nil
 441 # End:
 442 # vim: set expandtab tabstop=4 shiftwidth=4: