packages/maintenance/diskspace

   1 #!/usr/bin/env python
   2
   3 import os, sys, time, random, glob, traceback
   4
   5 opt_verbose = '-v' in sys.argv
   6 opt_force   = '-f' in sys.argv
   7
   8 omd_site = os.environ['OMD_SITE']
   9 omd_root = os.environ['OMD_ROOT']
  10
  11 config_file       = omd_root + '/etc/diskspace.conf'
  12 plugin_dir        = omd_root + '/share/diskspace'
  13 plugin_dir_local  = omd_root + '/local/share/diskspace'
  14
  15 # Initial configuration
  16 min_free_bytes = None
  17 max_file_age   = None
  18 min_file_age   = None
  19
  20 plugins = {}
  21
  22 def error(s):
  23     sys.stderr.write('ERROR: %s\n' % s)
  24
  25 def terminate(s):
  26     error(s)
  27     sys.exit(1)
  28
  29 def log(s):
  30     sys.stdout.write('%s\n' % s)
  31
  32 def verbose(s):
  33     if opt_verbose:
  34         log(s)
  35
  36 def read_config():
  37     try:
  38         execfile(config_file, globals(), globals())
  39     except IOError:
  40         pass # ignore non existant config
  41     except Exception, e:
  42         terminate('Invalid configuration: %s' % e)
  43
  44 def resolve_paths():
  45     for plugin in plugins.values():
  46         resolved = []
  47         for path in plugin.get('cleanup_paths', []):
  48             # Make relative paths absolute ones
  49             if path[0] != '/':
  50                 path = omd_root + '/' + path
  51
  52             # This resolves given path patterns to really existing files.
  53             # It also ensures that the files in the resolved list do really exist.
  54             resolved += glob.glob(path)
  55
  56         if resolved:
  57             plugin['cleanup_paths'] = resolved
  58         elif 'cleanup_paths' in plugin:
  59             del plugin['cleanup_paths']
  60
  61 def load_plugins():
  62     try:
  63         local_plugins = os.listdir(plugin_dir_local)
  64     except OSError:
  65         local_plugins = [] # this is optional
  66
  67     plugin_files = [ p for p in os.listdir(plugin_dir) if p not in local_plugins ]
  68
  69     for base_dir, file_list in [ (plugin_dir, plugin_files), (plugin_dir_local, local_plugins) ]:
  70         for f in file_list:
  71             if f[0] == '.':
  72                 continue
  73
  74             plugins[f] = {}
  75
  76             path = base_dir + '/' + f
  77             verbose('Loading plugin: %s' % path)
  78             try:
  79                 execfile(path, plugins[f], plugins[f])
  80             except Exception, e:
  81                 error('Exception while loading plugin "%s": %s' % (path, e))
  82
  83     # Now transform all path patterns to absolute paths for really existing files
  84     resolve_paths()
  85
  86 def collect_file_infos():
  87     for plugin in plugins.values():
  88         for path in plugin.get('cleanup_paths', []):
  89             result = os.stat(path)
  90             plugin.setdefault('file_infos', {})[path] = (result.st_size, result.st_mtime)
  91
  92 def fmt_bytes(b):
  93     b = float(b)
  94     base = 1024
  95     if b >= base * base * base * base:
  96         return '%.2fTB' % (b / base / base / base / base)
  97     elif b >= base * base * base:
  98         return '%.2fGB' % (b / base / base / base)
  99     elif b >= base * base:
 100         return '%.2fMB' % (b / base / base)
 101     elif b >= base:
 102         return '%.2fkB' % (b / base)
 103     else:
 104         return '%.0fB' % b
 105
 106 def get_free_space():
 107     # FIXME: Take eventual root reserved space into account
 108     for l in os.popen('df -P -B1 ' + omd_root).readlines():
 109         if l[0] == '/':
 110             vol, size_bytes, used_bytes, free_bytes, used_perc, mp = l.split()
 111             return int(free_bytes)
 112
 113 def above_threshold(b):
 114     return b >= min_free_bytes
 115
 116 def delete_file(path, reason):
 117     try:
 118         log('Deleting file (%s): %s' % (reason, path))
 119         os.unlink(path)
 120         return True
 121     except Exception, e:
 122         error('Error while deleting %s: %s' % (path, e))
 123     return False
 124
 125 # Loop all files to check wether or not files are older than
 126 # max_age. Simply remove all of them.
 127 def cleanup_aged():
 128     if max_file_age is None:
 129         verbose('Not cleaning up too old files (max_file_age not configured)')
 130         return
 131     max_age = time.time() - max_file_age
 132
 133     for plugin in plugins.values():
 134         for path, (size, mtime) in plugin.get('file_infos', {}).items():
 135             if mtime < max_age:
 136                 if delete_file(path, 'too old'):
 137                     del plugin['file_infos'][path]
 138             else:
 139                 verbose('Not deleting %s' % path)
 140
 141 def oldest_candidate(file_infos):
 142     if file_infos:
 143         # Sort by modification time
 144         sorted_infos = sorted(file_infos.items(), key = lambda i: i[1][1])
 145         oldest = sorted_infos[0]
 146         if oldest[1][1] < time.time() - min_file_age:
 147             return oldest[0]
 148
 149 def cleanup():
 150     if min_file_age is None:
 151         terminate('Not cleaning up oldest files of plugins (min_file_age not configured). terminating.')
 152
 153     # the scheduling of the cleanup job is supposed to be equal for
 154     # all sites. To ensure that not only one single site is always
 155     # cleaning up, we add a a random wait before cleanup.
 156     sleep_sec = float(random.randint(0, 10000)) / 1000
 157     verbose('Sleeping for %0.3f seconds' % sleep_sec)
 158     time.sleep(sleep_sec)
 159
 160     # Loop all cleanup plugins to find the oldest candidate per plugin
 161     # which is older than min_age and delete this file.
 162     for plugin_name, plugin in plugins.items():
 163         oldest = oldest_candidate(plugin.get('file_infos', {}))
 164         if oldest:
 165             delete_file(oldest, plugin_name + ': my oldest')
 166
 167 def main():
 168     load_plugins()
 169     collect_file_infos()
 170
 171     # get used diskspace of the sites volume
 172     bytes_free = get_free_space()
 173     verbose('Free space: %s' % fmt_bytes(bytes_free))
 174
 175     cleanup_aged()
 176
 177     bytes_free = get_free_space()
 178     verbose('Free space (after max_age cleanup): %s' % fmt_bytes(bytes_free))
 179
 180     # check diskspace against configuration
 181     if not opt_force and above_threshold(bytes_free):
 182         # -> ok: exit
 183         verbose('Free space is above threshold of %s. Nothing to be done.' % fmt_bytes(min_free_bytes))
 184         return
 185
 186     # free diskspace is below threshold, start cleanup
 187     cleanup()
 188
 189 # #############################################################################
 190
 191 read_config()
 192
 193 if min_free_bytes is None:
 194     verbose('minimal free bytes (min_free_bytes) not configured. terminating.')
 195     sys.exit(0)
 196
 197 try:
 198     main()
 199 except SystemExit:
 200     raise
 201 except:
 202     terminate('Unexpected exception: %s' % traceback.format_exc())