imgcreate/fs.py

   1 #
   2 # fs.py : Filesystem related utilities and classes
   3 #
   4 # Copyright 2007, Red Hat  Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; version 2 of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import os.path
  21 import sys
  22 import errno
  23 import stat
  24 import subprocess
  25 import random
  26 import string
  27 import logging
  28 import tempfile
  29 import time
  30
  31 from imgcreate.errors import *
  32
  33 def makedirs(dirname):
  34     """A version of os.makedirs() that doesn't throw an
  35     exception if the leaf directory already exists.
  36     """
  37     try:
  38         os.makedirs(dirname)
  39     except OSError, e:
  40         if e.errno != errno.EEXIST:
  41             raise
  42
  43 def mksquashfs(in_img, out_img, compress_type):
  44 # Allow gzip to work for older versions of mksquashfs
  45     if compress_type == "gzip":
  46         args = ["/sbin/mksquashfs", in_img, out_img]
  47     else:
  48         args = ["/sbin/mksquashfs", in_img, out_img, "-comp", compress_type]
  49
  50     if not sys.stdout.isatty():
  51         args.append("-no-progress")
  52
  53     ret = subprocess.call(args)
  54     if ret != 0:
  55         raise SquashfsError("'%s' exited with error (%d)" %
  56                             (string.join(args, " "), ret))
  57
  58 def resize2fs(fs, size = None, minimal = False):
  59     if minimal and size is not None:
  60         raise ResizeError("Can't specify both minimal and a size for resize!")
  61     if not minimal and size is None:
  62         raise ResizeError("Must specify either a size or minimal for resize!")
  63
  64     e2fsck(fs)
  65     (fd, saved_image) = tempfile.mkstemp("", "resize-image-", "/tmp")
  66     os.close(fd)
  67     subprocess.call(["/sbin/e2image", "-r", fs, saved_image])
  68
  69     args = ["/sbin/resize2fs", fs]
  70     if minimal:
  71         args.append("-M")
  72     else:
  73         args.append("%sK" %(size / 1024,))
  74     ret = subprocess.call(args)
  75     if ret != 0:
  76         raise ResizeError("resize2fs returned an error (%d)!  image to debug at %s" %(ret, saved_image))
  77
  78     if e2fsck(fs) != 0:
  79         raise ResizeError("fsck after resize returned an error!  image to debug at %s" %(saved_image,))
  80     os.unlink(saved_image)
  81     return 0
  82
  83 def e2fsck(fs):
  84     logging.debug("Checking filesystem %s" % fs)
  85     rc = subprocess.call(["/sbin/e2fsck", "-f", "-y", fs])
  86     return rc
  87
  88 class BindChrootMount:
  89     """Represents a bind mount of a directory into a chroot."""
  90     def __init__(self, src, chroot, dest = None):
  91         self.src = src
  92         self.root = chroot
  93
  94         if not dest:
  95             dest = src
  96         self.dest = self.root + "/" + dest
  97
  98         self.mounted = False
  99
 100     def mount(self):
 101         if self.mounted:
 102             return
 103
 104         makedirs(self.dest)
 105         rc = subprocess.call(["/bin/mount", "--bind", self.src, self.dest])
 106         if rc != 0:
 107             raise MountError("Bind-mounting '%s' to '%s' failed" %
 108                              (self.src, self.dest))
 109         self.mounted = True
 110
 111     def unmount(self):
 112         if not self.mounted:
 113             return
 114
 115         rc = subprocess.call(["/bin/umount", self.dest])
 116         if rc != 0:
 117             logging.debug("Unable to unmount %s normally, using lazy unmount" % self.dest)
 118             rc = subprocess.call(["/bin/umount", "-l", self.dest])
 119             if rc != 0:
 120                 raise MountError("Unable to unmount fs at %s" % self.dest)
 121             else:
 122                 logging.debug("lazy umount succeeded on %s" % self.dest)
 123                 print >> sys.stdout, "lazy umount succeeded on %s" % self.dest
 124
 125         self.mounted = False
 126
 127 class LoopbackMount:
 128     """LoopbackMount  compatibility layer for old API"""
 129     def __init__(self, lofile, mountdir, fstype = None):
 130         self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
 131         self.losetup = False
 132
 133     def cleanup(self):
 134         self.diskmount.cleanup()
 135
 136     def unmount(self):
 137         self.diskmount.unmount()
 138
 139     def lounsetup(self):
 140         if self.losetup:
 141             rc = subprocess.call(["/sbin/losetup", "-d", self.loopdev])
 142             self.losetup = False
 143             self.loopdev = None
 144
 145     def loopsetup(self):
 146         if self.losetup:
 147             return
 148
 149         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 150                                        stdout=subprocess.PIPE)
 151         losetupOutput = losetupProc.communicate()[0]
 152
 153         if losetupProc.returncode:
 154             raise MountError("Failed to allocate loop device for '%s'" %
 155                              self.lofile)
 156
 157         self.loopdev = losetupOutput.split()[0]
 158
 159         rc = subprocess.call(["/sbin/losetup", self.loopdev, self.lofile])
 160         if rc != 0:
 161             raise MountError("Failed to allocate loop device for '%s'" %
 162                              self.lofile)
 163
 164         self.losetup = True
 165
 166     def mount(self):
 167         self.diskmount.mount()
 168
 169 class SparseLoopbackMount(LoopbackMount):
 170     """SparseLoopbackMount  compatibility layer for old API"""
 171     def __init__(self, lofile, mountdir, size, fstype = None):
 172         self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
 173
 174     def expand(self, create = False, size = None):
 175         self.diskmount.disk.expand(create, size)
 176
 177     def truncate(self, size = None):
 178         self.diskmount.disk.truncate(size)
 179
 180     def create(self):
 181         self.diskmount.disk.create()
 182
 183 class SparseExtLoopbackMount(SparseLoopbackMount):
 184     """SparseExtLoopbackMount  compatibility layer for old API"""
 185     def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
 186         self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True)
 187
 188
 189     def __format_filesystem(self):
 190         self.diskmount.__format_filesystem()
 191
 192     def create(self):
 193         self.diskmount.disk.create()
 194
 195     def resize(self, size = None):
 196         return self.diskmount.__resize_filesystem(size)
 197
 198     def mount(self):
 199         self.diskmount.mount()
 200
 201     def __fsck(self):
 202         self.extdiskmount.__fsck()
 203
 204     def __get_size_from_filesystem(self):
 205         return self.diskmount.__get_size_from_filesystem()
 206
 207     def __resize_to_minimal(self):
 208         return self.diskmount.__resize_to_minimal()
 209
 210     def resparse(self, size = None):
 211         return self.diskmount.resparse(size)
 212
 213 class Disk:
 214     """Generic base object for a disk
 215
 216     The 'create' method must make the disk visible as a block device - eg
 217     by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
 218     method must undo the 'create' operation.
 219     """
 220     def __init__(self, size, device = None):
 221         self._device = device
 222         self._size = size
 223
 224     def create(self):
 225         pass
 226
 227     def cleanup(self):
 228         pass
 229
 230     def get_device(self):
 231         return self._device
 232     def set_device(self, path):
 233         self._device = path
 234     device = property(get_device, set_device)
 235
 236     def get_size(self):
 237         return self._size
 238     size = property(get_size)
 239
 240
 241 class RawDisk(Disk):
 242     """A Disk backed by a block device.
 243     Note that create() is a no-op.
 244     """
 245     def __init__(self, size, device):
 246         Disk.__init__(self, size, device)
 247
 248     def fixed(self):
 249         return True
 250
 251     def exists(self):
 252         return True
 253
 254 class LoopbackDisk(Disk):
 255     """A Disk backed by a file via the loop module."""
 256     def __init__(self, lofile, size):
 257         Disk.__init__(self, size)
 258         self.lofile = lofile
 259
 260     def fixed(self):
 261         return False
 262
 263     def exists(self):
 264         return os.path.exists(self.lofile)
 265
 266     def create(self):
 267         if self.device is not None:
 268             return
 269
 270         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 271                                        stdout=subprocess.PIPE)
 272         losetupOutput = losetupProc.communicate()[0]
 273
 274         if losetupProc.returncode:
 275             raise MountError("Failed to allocate loop device for '%s'" %
 276                              self.lofile)
 277
 278         device = losetupOutput.split()[0]
 279
 280         logging.debug("Losetup add %s mapping to %s"  % (device, self.lofile))
 281         rc = subprocess.call(["/sbin/losetup", device, self.lofile])
 282         if rc != 0:
 283             raise MountError("Failed to allocate loop device for '%s'" %
 284                              self.lofile)
 285         self.device = device
 286
 287     def cleanup(self):
 288         if self.device is None:
 289             return
 290         logging.debug("Losetup remove %s" % self.device)
 291         rc = subprocess.call(["/sbin/losetup", "-d", self.device])
 292         self.device = None
 293
 294
 295
 296 class SparseLoopbackDisk(LoopbackDisk):
 297     """A Disk backed by a sparse file via the loop module."""
 298     def __init__(self, lofile, size):
 299         LoopbackDisk.__init__(self, lofile, size)
 300
 301     def expand(self, create = False, size = None):
 302         flags = os.O_WRONLY
 303         if create:
 304             flags |= os.O_CREAT
 305             makedirs(os.path.dirname(self.lofile))
 306
 307         if size is None:
 308             size = self.size
 309
 310         logging.debug("Extending sparse file %s to %d" % (self.lofile, size))
 311         fd = os.open(self.lofile, flags)
 312
 313         if size <= 0:
 314             size = 1
 315         os.lseek(fd, size-1, 0)
 316         os.write(fd, '\x00')
 317         os.close(fd)
 318
 319     def truncate(self, size = None):
 320         if size is None:
 321             size = self.size
 322
 323         logging.debug("Truncating sparse file %s to %d" % (self.lofile, size))
 324         fd = os.open(self.lofile, os.O_WRONLY)
 325         os.ftruncate(fd, size)
 326         os.close(fd)
 327
 328     def create(self):
 329         self.expand(create = True)
 330         LoopbackDisk.create(self)
 331
 332 class Mount:
 333     """A generic base class to deal with mounting things."""
 334     def __init__(self, mountdir):
 335         self.mountdir = mountdir
 336
 337     def cleanup(self):
 338         self.unmount()
 339
 340     def mount(self):
 341         pass
 342
 343     def unmount(self):
 344         pass
 345
 346 class DiskMount(Mount):
 347     """A Mount object that handles mounting of a Disk."""
 348     def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
 349         Mount.__init__(self, mountdir)
 350
 351         self.disk = disk
 352         self.fstype = fstype
 353         self.rmmountdir = rmmountdir
 354
 355         self.mounted = False
 356         self.rmdir   = False
 357
 358     def cleanup(self):
 359         Mount.cleanup(self)
 360         self.disk.cleanup()
 361
 362     def unmount(self):
 363         if self.mounted:
 364             logging.debug("Unmounting directory %s" % self.mountdir)
 365             rc = subprocess.call(["/bin/umount", self.mountdir])
 366             if rc == 0:
 367                 self.mounted = False
 368             else:
 369                 logging.debug("Unmounting directory %s failed, using lazy umount" % self.mountdir)
 370                 print >> sys.stdout, "Unmounting directory %s failed, using lazy umount" %self.mountdir
 371                 rc = subprocess.call(["/bin/umount", "-l", self.mountdir])
 372                 if rc != 0:
 373                     raise MountError("Unable to unmount filesystem at %s" % self.mountdir)
 374                 else:
 375                     logging.debug("lazy umount succeeded on %s" % self.mountdir)
 376                     print >> sys.stdout, "lazy umount succeeded on %s" % self.mountdir
 377                     self.mounted = False
 378
 379         if self.rmdir and not self.mounted:
 380             try:
 381                 os.rmdir(self.mountdir)
 382             except OSError, e:
 383                 pass
 384             self.rmdir = False
 385
 386
 387     def __create(self):
 388         self.disk.create()
 389
 390
 391     def mount(self):
 392         if self.mounted:
 393             return
 394
 395         if not os.path.isdir(self.mountdir):
 396             logging.debug("Creating mount point %s" % self.mountdir)
 397             os.makedirs(self.mountdir)
 398             self.rmdir = self.rmmountdir
 399
 400         self.__create()
 401
 402         logging.debug("Mounting %s at %s" % (self.disk.device, self.mountdir))
 403         args = [ "/bin/mount", self.disk.device, self.mountdir ]
 404         if self.fstype:
 405             args.extend(["-t", self.fstype])
 406
 407         rc = subprocess.call(args)
 408         if rc != 0:
 409             raise MountError("Failed to mount '%s' to '%s'" %
 410                              (self.disk.device, self.mountdir))
 411
 412         self.mounted = True
 413
 414 class ExtDiskMount(DiskMount):
 415     """A DiskMount object that is able to format/resize ext[23] filesystems."""
 416     def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True):
 417         DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
 418         self.blocksize = blocksize
 419         self.fslabel = "_" + fslabel
 420
 421     def __format_filesystem(self):
 422         logging.debug("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
 423         rc = subprocess.call(["/sbin/mkfs." + self.fstype,
 424                               "-F", "-L", self.fslabel,
 425                               "-m", "1", "-b", str(self.blocksize),
 426                               self.disk.device])
 427         #                      str(self.disk.size / self.blocksize)])
 428         if rc != 0:
 429             raise MountError("Error creating %s filesystem" % (self.fstype,))
 430         logging.debug("Tuning filesystem on %s" % self.disk.device)
 431         subprocess.call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
 432                          "-ouser_xattr,acl", self.disk.device])
 433
 434     def __resize_filesystem(self, size = None):
 435         current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
 436
 437         if size is None:
 438             size = self.disk.size
 439
 440         if size == current_size:
 441             return
 442
 443         if size > current_size:
 444             self.disk.expand(size)
 445
 446         resize2fs(self.disk.lofile, size)
 447         return size
 448
 449     def __create(self):
 450         resize = False
 451         if not self.disk.fixed() and self.disk.exists():
 452             resize = True
 453
 454         self.disk.create()
 455
 456         if resize:
 457             self.__resize_filesystem()
 458         else:
 459             self.__format_filesystem()
 460
 461     def mount(self):
 462         self.__create()
 463         DiskMount.mount(self)
 464
 465     def __fsck(self):
 466         return e2fsck(self.disk.lofile)
 467         return rc
 468
 469     def __get_size_from_filesystem(self):
 470         def parse_field(output, field):
 471             for line in output.split("\n"):
 472                 if line.startswith(field + ":"):
 473                     return line[len(field) + 1:].strip()
 474
 475             raise KeyError("Failed to find field '%s' in output" % field)
 476
 477         dev_null = os.open("/dev/null", os.O_WRONLY)
 478         try:
 479             out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
 480                                    stdout = subprocess.PIPE,
 481                                    stderr = dev_null).communicate()[0]
 482         finally:
 483             os.close(dev_null)
 484
 485         return int(parse_field(out, "Block count")) * self.blocksize
 486
 487     def __resize_to_minimal(self):
 488         resize2fs(self.disk.lofile, minimal = True)
 489         return self.__get_size_from_filesystem()
 490
 491     def resparse(self, size = None):
 492         self.cleanup()
 493         minsize = self.__resize_to_minimal()
 494         self.disk.truncate(minsize)
 495         self.__resize_filesystem(size)
 496         return minsize
 497
 498 class DeviceMapperSnapshot(object):
 499     def __init__(self, imgloop, cowloop):
 500         self.imgloop = imgloop
 501         self.cowloop = cowloop
 502
 503         self.__created = False
 504         self.__name = None
 505
 506     def get_path(self):
 507         if self.__name is None:
 508             return None
 509         return os.path.join("/dev/mapper", self.__name)
 510     path = property(get_path)
 511
 512     def create(self):
 513         if self.__created:
 514             return
 515
 516         self.imgloop.create()
 517         self.cowloop.create()
 518
 519         self.__name = "imgcreate-%d-%d" % (os.getpid(),
 520                                            random.randint(0, 2**16))
 521
 522         size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
 523
 524         table = "0 %d snapshot %s %s p 8" % (size / 512,
 525                                              self.imgloop.device,
 526                                              self.cowloop.device)
 527
 528         args = ["/sbin/dmsetup", "create", self.__name, "--table", table]
 529         if subprocess.call(args) != 0:
 530             self.cowloop.cleanup()
 531             self.imgloop.cleanup()
 532             raise SnapshotError("Could not create snapshot device using: " +
 533                                 string.join(args, " "))
 534
 535         self.__created = True
 536
 537     def remove(self, ignore_errors = False):
 538         if not self.__created:
 539             return
 540
 541         # sleep to try to avoid any dm shenanigans
 542         time.sleep(2)
 543         rc = subprocess.call(["/sbin/dmsetup", "remove", self.__name])
 544         if not ignore_errors and rc != 0:
 545             raise SnapshotError("Could not remove snapshot device")
 546
 547         self.__name = None
 548         self.__created = False
 549
 550         self.cowloop.cleanup()
 551         self.imgloop.cleanup()
 552
 553     def get_cow_used(self):
 554         if not self.__created:
 555             return 0
 556
 557         dev_null = os.open("/dev/null", os.O_WRONLY)
 558         try:
 559             out = subprocess.Popen(["/sbin/dmsetup", "status", self.__name],
 560                                    stdout = subprocess.PIPE,
 561                                    stderr = dev_null).communicate()[0]
 562         finally:
 563             os.close(dev_null)
 564
 565         #
 566         # dmsetup status on a snapshot returns e.g.
 567         #   "0 8388608 snapshot 416/1048576"
 568         # or, more generally:
 569         #   "A B snapshot C/D"
 570         # where C is the number of 512 byte sectors in use
 571         #
 572         try:
 573             return int((out.split()[3]).split('/')[0]) * 512
 574         except ValueError:
 575             raise SnapshotError("Failed to parse dmsetup status: " + out)
 576
 577 def create_image_minimizer(path, image, compress_type, target_size = None):
 578     """
 579     Builds a copy-on-write image which can be used to
 580     create a device-mapper snapshot of an image where
 581     the image's filesystem is as small as possible
 582
 583     The steps taken are:
 584       1) Create a sparse COW
 585       2) Loopback mount the image and the COW
 586       3) Create a device-mapper snapshot of the image
 587          using the COW
 588       4) Resize the filesystem to the minimal size
 589       5) Determine the amount of space used in the COW
 590       6) Restroy the device-mapper snapshot
 591       7) Truncate the COW, removing unused space
 592       8) Create a squashfs of the COW
 593     """
 594     imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
 595
 596     cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
 597                                  64L * 1024L * 1024L)
 598
 599     snapshot = DeviceMapperSnapshot(imgloop, cowloop)
 600
 601     try:
 602         snapshot.create()
 603
 604         if target_size is not None:
 605             resize2fs(snapshot.path, target_size)
 606         else:
 607             resize2fs(snapshot.path, minimal = True)
 608
 609         cow_used = snapshot.get_cow_used()
 610     finally:
 611         snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
 612
 613     cowloop.truncate(cow_used)
 614
 615     mksquashfs(cowloop.lofile, path, compress_type)
 616
 617     os.unlink(cowloop.lofile)
 618