imgcreate/fs.py

   1 #
   2 # fs.py : Filesystem related utilities and classes
   3 #
   4 # Copyright 2007, Red Hat  Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; version 2 of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import os.path
  21 import sys
  22 import errno
  23 import stat
  24 import subprocess
  25 import random
  26 import string
  27 import logging
  28 import tempfile
  29 import time
  30 from util import call
  31
  32 from imgcreate.errors import *
  33
  34 def makedirs(dirname):
  35     """A version of os.makedirs() that doesn't throw an
  36     exception if the leaf directory already exists.
  37     """
  38     try:
  39         os.makedirs(dirname)
  40     except OSError, e:
  41         if e.errno != errno.EEXIST:
  42             raise
  43
  44 def mksquashfs(in_img, out_img, compress_type):
  45 # Allow gzip to work for older versions of mksquashfs
  46     if compress_type == "gzip":
  47         args = ["/sbin/mksquashfs", in_img, out_img]
  48     else:
  49         args = ["/sbin/mksquashfs", in_img, out_img, "-comp", compress_type]
  50
  51     if not sys.stdout.isatty():
  52         args.append("-no-progress")
  53
  54     ret = call(args)
  55     if ret != 0:
  56         raise SquashfsError("'%s' exited with error (%d)" %
  57                             (string.join(args, " "), ret))
  58
  59 def resize2fs(fs, size = None, minimal = False):
  60     if minimal and size is not None:
  61         raise ResizeError("Can't specify both minimal and a size for resize!")
  62     if not minimal and size is None:
  63         raise ResizeError("Must specify either a size or minimal for resize!")
  64
  65     e2fsck(fs)
  66     (fd, saved_image) = tempfile.mkstemp("", "resize-image-", "/tmp")
  67     os.close(fd)
  68     call(["/sbin/e2image", "-r", fs, saved_image])
  69
  70     args = ["/sbin/resize2fs", fs]
  71     if minimal:
  72         args.append("-M")
  73     else:
  74         args.append("%sK" %(size / 1024,))
  75     ret = call(args)
  76     if ret != 0:
  77         raise ResizeError("resize2fs returned an error (%d)!  image to debug at %s" %(ret, saved_image))
  78
  79     if e2fsck(fs) != 0:
  80         raise ResizeError("fsck after resize returned an error!  image to debug at %s" %(saved_image,))
  81     os.unlink(saved_image)
  82     return 0
  83
  84 def e2fsck(fs):
  85     logging.info("Checking filesystem %s" % fs)
  86     rc = call(["/sbin/e2fsck", "-f", "-y", fs])
  87     return rc
  88
  89 class BindChrootMount:
  90     """Represents a bind mount of a directory into a chroot."""
  91     def __init__(self, src, chroot, dest = None):
  92         self.src = src
  93         self.root = chroot
  94
  95         if not dest:
  96             dest = src
  97         self.dest = self.root + "/" + dest
  98
  99         self.mounted = False
 100
 101     def mount(self):
 102         if self.mounted:
 103             return
 104
 105         makedirs(self.dest)
 106         rc = call(["/bin/mount", "--bind", self.src, self.dest])
 107         if rc != 0:
 108             raise MountError("Bind-mounting '%s' to '%s' failed" %
 109                              (self.src, self.dest))
 110         self.mounted = True
 111
 112     def unmount(self):
 113         if not self.mounted:
 114             return
 115
 116         rc = call(["/bin/umount", self.dest])
 117         if rc != 0:
 118             logging.info("Unable to unmount %s normally, using lazy unmount" % self.dest)
 119             rc = call(["/bin/umount", "-l", self.dest])
 120             if rc != 0:
 121                 raise MountError("Unable to unmount fs at %s" % self.dest)
 122             else:
 123                 logging.info("lazy umount succeeded on %s" % self.dest)
 124                 print >> sys.stdout, "lazy umount succeeded on %s" % self.dest
 125
 126         self.mounted = False
 127
 128 class LoopbackMount:
 129     """LoopbackMount  compatibility layer for old API"""
 130     def __init__(self, lofile, mountdir, fstype = None):
 131         self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
 132         self.losetup = False
 133
 134     def cleanup(self):
 135         self.diskmount.cleanup()
 136
 137     def unmount(self):
 138         self.diskmount.unmount()
 139
 140     def lounsetup(self):
 141         if self.losetup:
 142             rc = call(["/sbin/losetup", "-d", self.loopdev])
 143             self.losetup = False
 144             self.loopdev = None
 145
 146     def loopsetup(self):
 147         if self.losetup:
 148             return
 149
 150         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 151                                        stdout=subprocess.PIPE)
 152         losetupOutput = losetupProc.communicate()[0]
 153
 154         if losetupProc.returncode:
 155             raise MountError("Failed to allocate loop device for '%s'" %
 156                              self.lofile)
 157
 158         self.loopdev = losetupOutput.split()[0]
 159
 160         rc = call(["/sbin/losetup", self.loopdev, self.lofile])
 161         if rc != 0:
 162             raise MountError("Failed to allocate loop device for '%s'" %
 163                              self.lofile)
 164
 165         self.losetup = True
 166
 167     def mount(self):
 168         self.diskmount.mount()
 169
 170 class SparseLoopbackMount(LoopbackMount):
 171     """SparseLoopbackMount  compatibility layer for old API"""
 172     def __init__(self, lofile, mountdir, size, fstype = None):
 173         self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
 174
 175     def expand(self, create = False, size = None):
 176         self.diskmount.disk.expand(create, size)
 177
 178     def truncate(self, size = None):
 179         self.diskmount.disk.truncate(size)
 180
 181     def create(self):
 182         self.diskmount.disk.create()
 183
 184 class SparseExtLoopbackMount(SparseLoopbackMount):
 185     """SparseExtLoopbackMount  compatibility layer for old API"""
 186     def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
 187         self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size), mountdir, fstype, blocksize, fslabel, rmmountdir = True)
 188
 189
 190     def __format_filesystem(self):
 191         self.diskmount.__format_filesystem()
 192
 193     def create(self):
 194         self.diskmount.disk.create()
 195
 196     def resize(self, size = None):
 197         return self.diskmount.__resize_filesystem(size)
 198
 199     def mount(self):
 200         self.diskmount.mount()
 201
 202     def __fsck(self):
 203         self.extdiskmount.__fsck()
 204
 205     def __get_size_from_filesystem(self):
 206         return self.diskmount.__get_size_from_filesystem()
 207
 208     def __resize_to_minimal(self):
 209         return self.diskmount.__resize_to_minimal()
 210
 211     def resparse(self, size = None):
 212         return self.diskmount.resparse(size)
 213
 214 class Disk:
 215     """Generic base object for a disk
 216
 217     The 'create' method must make the disk visible as a block device - eg
 218     by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
 219     method must undo the 'create' operation.
 220     """
 221     def __init__(self, size, device = None):
 222         self._device = device
 223         self._size = size
 224
 225     def create(self):
 226         pass
 227
 228     def cleanup(self):
 229         pass
 230
 231     def get_device(self):
 232         return self._device
 233     def set_device(self, path):
 234         self._device = path
 235     device = property(get_device, set_device)
 236
 237     def get_size(self):
 238         return self._size
 239     size = property(get_size)
 240
 241
 242 class RawDisk(Disk):
 243     """A Disk backed by a block device.
 244     Note that create() is a no-op.
 245     """
 246     def __init__(self, size, device):
 247         Disk.__init__(self, size, device)
 248
 249     def fixed(self):
 250         return True
 251
 252     def exists(self):
 253         return True
 254
 255 class LoopbackDisk(Disk):
 256     """A Disk backed by a file via the loop module."""
 257     def __init__(self, lofile, size):
 258         Disk.__init__(self, size)
 259         self.lofile = lofile
 260
 261     def fixed(self):
 262         return False
 263
 264     def exists(self):
 265         return os.path.exists(self.lofile)
 266
 267     def create(self):
 268         if self.device is not None:
 269             return
 270
 271         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 272                                        stdout=subprocess.PIPE)
 273         losetupOutput = losetupProc.communicate()[0]
 274
 275         if losetupProc.returncode:
 276             raise MountError("Failed to allocate loop device for '%s'" %
 277                              self.lofile)
 278
 279         device = losetupOutput.split()[0]
 280
 281         logging.info("Losetup add %s mapping to %s"  % (device, self.lofile))
 282         rc = call(["/sbin/losetup", device, self.lofile])
 283         if rc != 0:
 284             raise MountError("Failed to allocate loop device for '%s'" %
 285                              self.lofile)
 286         self.device = device
 287
 288     def cleanup(self):
 289         if self.device is None:
 290             return
 291         logging.info("Losetup remove %s" % self.device)
 292         rc = call(["/sbin/losetup", "-d", self.device])
 293         self.device = None
 294
 295
 296
 297 class SparseLoopbackDisk(LoopbackDisk):
 298     """A Disk backed by a sparse file via the loop module."""
 299     def __init__(self, lofile, size):
 300         LoopbackDisk.__init__(self, lofile, size)
 301
 302     def expand(self, create = False, size = None):
 303         flags = os.O_WRONLY
 304         if create:
 305             flags |= os.O_CREAT
 306             makedirs(os.path.dirname(self.lofile))
 307
 308         if size is None:
 309             size = self.size
 310
 311         logging.info("Extending sparse file %s to %d" % (self.lofile, size))
 312         fd = os.open(self.lofile, flags)
 313
 314         if size <= 0:
 315             size = 1
 316         os.lseek(fd, size-1, 0)
 317         os.write(fd, '\x00')
 318         os.close(fd)
 319
 320     def truncate(self, size = None):
 321         if size is None:
 322             size = self.size
 323
 324         logging.info("Truncating sparse file %s to %d" % (self.lofile, size))
 325         fd = os.open(self.lofile, os.O_WRONLY)
 326         os.ftruncate(fd, size)
 327         os.close(fd)
 328
 329     def create(self):
 330         self.expand(create = True)
 331         LoopbackDisk.create(self)
 332
 333 class Mount:
 334     """A generic base class to deal with mounting things."""
 335     def __init__(self, mountdir):
 336         self.mountdir = mountdir
 337
 338     def cleanup(self):
 339         self.unmount()
 340
 341     def mount(self):
 342         pass
 343
 344     def unmount(self):
 345         pass
 346
 347 class DiskMount(Mount):
 348     """A Mount object that handles mounting of a Disk."""
 349     def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
 350         Mount.__init__(self, mountdir)
 351
 352         self.disk = disk
 353         self.fstype = fstype
 354         self.rmmountdir = rmmountdir
 355
 356         self.mounted = False
 357         self.rmdir   = False
 358
 359     def cleanup(self):
 360         Mount.cleanup(self)
 361         self.disk.cleanup()
 362
 363     def unmount(self):
 364         if self.mounted:
 365             logging.info("Unmounting directory %s" % self.mountdir)
 366             rc = call(["/bin/umount", self.mountdir])
 367             if rc == 0:
 368                 self.mounted = False
 369             else:
 370                 logging.warn("Unmounting directory %s failed, using lazy umount" % self.mountdir)
 371                 print >> sys.stdout, "Unmounting directory %s failed, using lazy umount" %self.mountdir
 372                 rc = call(["/bin/umount", "-l", self.mountdir])
 373                 if rc != 0:
 374                     raise MountError("Unable to unmount filesystem at %s" % self.mountdir)
 375                 else:
 376                     logging.info("lazy umount succeeded on %s" % self.mountdir)
 377                     print >> sys.stdout, "lazy umount succeeded on %s" % self.mountdir
 378                     self.mounted = False
 379
 380         if self.rmdir and not self.mounted:
 381             try:
 382                 os.rmdir(self.mountdir)
 383             except OSError, e:
 384                 pass
 385             self.rmdir = False
 386
 387
 388     def __create(self):
 389         self.disk.create()
 390
 391
 392     def mount(self):
 393         if self.mounted:
 394             return
 395
 396         if not os.path.isdir(self.mountdir):
 397             logging.info("Creating mount point %s" % self.mountdir)
 398             os.makedirs(self.mountdir)
 399             self.rmdir = self.rmmountdir
 400
 401         self.__create()
 402
 403         logging.info("Mounting %s at %s" % (self.disk.device, self.mountdir))
 404         args = [ "/bin/mount", self.disk.device, self.mountdir ]
 405         if self.fstype:
 406             args.extend(["-t", self.fstype])
 407
 408         rc = call(args)
 409         if rc != 0:
 410             raise MountError("Failed to mount '%s' to '%s'" %
 411                              (self.disk.device, self.mountdir))
 412
 413         self.mounted = True
 414
 415 class ExtDiskMount(DiskMount):
 416     """A DiskMount object that is able to format/resize ext[23] filesystems."""
 417     def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True):
 418         DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
 419         self.blocksize = blocksize
 420         self.fslabel = "_" + fslabel
 421
 422     def __format_filesystem(self):
 423         logging.info("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
 424         rc = call(["/sbin/mkfs." + self.fstype,
 425                    "-F", "-L", self.fslabel,
 426                    "-m", "1", "-b", str(self.blocksize),
 427                    self.disk.device])
 428         #          str(self.disk.size / self.blocksize)])
 429
 430         if rc != 0:
 431             raise MountError("Error creating %s filesystem" % (self.fstype,))
 432         logging.info("Tuning filesystem on %s" % self.disk.device)
 433         call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
 434               "-ouser_xattr,acl", self.disk.device])
 435
 436     def __resize_filesystem(self, size = None):
 437         current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
 438
 439         if size is None:
 440             size = self.disk.size
 441
 442         if size == current_size:
 443             return
 444
 445         if size > current_size:
 446             self.disk.expand(size)
 447
 448         resize2fs(self.disk.lofile, size)
 449         return size
 450
 451     def __create(self):
 452         resize = False
 453         if not self.disk.fixed() and self.disk.exists():
 454             resize = True
 455
 456         self.disk.create()
 457
 458         if resize:
 459             self.__resize_filesystem()
 460         else:
 461             self.__format_filesystem()
 462
 463     def mount(self):
 464         self.__create()
 465         DiskMount.mount(self)
 466
 467     def __fsck(self):
 468         return e2fsck(self.disk.lofile)
 469         return rc
 470
 471     def __get_size_from_filesystem(self):
 472         def parse_field(output, field):
 473             for line in output.split("\n"):
 474                 if line.startswith(field + ":"):
 475                     return line[len(field) + 1:].strip()
 476
 477             raise KeyError("Failed to find field '%s' in output" % field)
 478
 479         dev_null = os.open("/dev/null", os.O_WRONLY)
 480         try:
 481             out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
 482                                    stdout = subprocess.PIPE,
 483                                    stderr = dev_null).communicate()[0]
 484         finally:
 485             os.close(dev_null)
 486
 487         return int(parse_field(out, "Block count")) * self.blocksize
 488
 489     def __resize_to_minimal(self):
 490         resize2fs(self.disk.lofile, minimal = True)
 491         return self.__get_size_from_filesystem()
 492
 493     def resparse(self, size = None):
 494         self.cleanup()
 495         minsize = self.__resize_to_minimal()
 496         self.disk.truncate(minsize)
 497         self.__resize_filesystem(size)
 498         return minsize
 499
 500 class DeviceMapperSnapshot(object):
 501     def __init__(self, imgloop, cowloop):
 502         self.imgloop = imgloop
 503         self.cowloop = cowloop
 504
 505         self.__created = False
 506         self.__name = None
 507
 508     def get_path(self):
 509         if self.__name is None:
 510             return None
 511         return os.path.join("/dev/mapper", self.__name)
 512     path = property(get_path)
 513
 514     def create(self):
 515         if self.__created:
 516             return
 517
 518         self.imgloop.create()
 519         self.cowloop.create()
 520
 521         self.__name = "imgcreate-%d-%d" % (os.getpid(),
 522                                            random.randint(0, 2**16))
 523
 524         size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
 525
 526         table = "0 %d snapshot %s %s p 8" % (size / 512,
 527                                              self.imgloop.device,
 528                                              self.cowloop.device)
 529
 530         args = ["/sbin/dmsetup", "create", self.__name,
 531                 "--uuid", "LIVECD-%s" % self.__name, "--table", table]
 532         if call(args) != 0:
 533             self.cowloop.cleanup()
 534             self.imgloop.cleanup()
 535             raise SnapshotError("Could not create snapshot device using: " +
 536                                 string.join(args, " "))
 537
 538         self.__created = True
 539
 540     def remove(self, ignore_errors = False):
 541         if not self.__created:
 542             return
 543
 544         # sleep to try to avoid any dm shenanigans
 545         time.sleep(2)
 546         rc = call(["/sbin/dmsetup", "remove", self.__name])
 547         if not ignore_errors and rc != 0:
 548             raise SnapshotError("Could not remove snapshot device")
 549
 550         self.__name = None
 551         self.__created = False
 552
 553         self.cowloop.cleanup()
 554         self.imgloop.cleanup()
 555
 556     def get_cow_used(self):
 557         if not self.__created:
 558             return 0
 559
 560         dev_null = os.open("/dev/null", os.O_WRONLY)
 561         try:
 562             out = subprocess.Popen(["/sbin/dmsetup", "status", self.__name],
 563                                    stdout = subprocess.PIPE,
 564                                    stderr = dev_null).communicate()[0]
 565         finally:
 566             os.close(dev_null)
 567
 568         #
 569         # dmsetup status on a snapshot returns e.g.
 570         #   "0 8388608 snapshot 416/1048576"
 571         # or, more generally:
 572         #   "A B snapshot C/D"
 573         # where C is the number of 512 byte sectors in use
 574         #
 575         try:
 576             return int((out.split()[3]).split('/')[0]) * 512
 577         except ValueError:
 578             raise SnapshotError("Failed to parse dmsetup status: " + out)
 579
 580 def create_image_minimizer(path, image, compress_type, target_size = None):
 581     """
 582     Builds a copy-on-write image which can be used to
 583     create a device-mapper snapshot of an image where
 584     the image's filesystem is as small as possible
 585
 586     The steps taken are:
 587       1) Create a sparse COW
 588       2) Loopback mount the image and the COW
 589       3) Create a device-mapper snapshot of the image
 590          using the COW
 591       4) Resize the filesystem to the minimal size
 592       5) Determine the amount of space used in the COW
 593       6) Restroy the device-mapper snapshot
 594       7) Truncate the COW, removing unused space
 595       8) Create a squashfs of the COW
 596     """
 597     imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
 598
 599     cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
 600                                  64L * 1024L * 1024L)
 601
 602     snapshot = DeviceMapperSnapshot(imgloop, cowloop)
 603
 604     try:
 605         snapshot.create()
 606
 607         if target_size is not None:
 608             resize2fs(snapshot.path, target_size)
 609         else:
 610             resize2fs(snapshot.path, minimal = True)
 611
 612         cow_used = snapshot.get_cow_used()
 613     finally:
 614         snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
 615
 616     cowloop.truncate(cow_used)
 617
 618     mksquashfs(cowloop.lofile, path, compress_type)
 619
 620     os.unlink(cowloop.lofile)
 621