imgcreate/fs.py

   1 #
   2 # fs.py : Filesystem related utilities and classes
   3 #
   4 # Copyright 2007, Red Hat  Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; version 2 of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import os.path
  21 import sys
  22 import errno
  23 import stat
  24 import subprocess
  25 import random
  26 import string
  27 import logging
  28 import tempfile
  29 import time
  30 from util import call
  31
  32 from imgcreate.errors import *
  33
  34 def makedirs(dirname):
  35     """A version of os.makedirs() that doesn't throw an
  36     exception if the leaf directory already exists.
  37     """
  38     try:
  39         os.makedirs(dirname)
  40     except OSError, e:
  41         if e.errno != errno.EEXIST:
  42             raise
  43
  44 def squashfs_compression_type(sqfs_img):
  45     """Check the compression type of a SquashFS image. If the type cannot be
  46     ascertained, return 'undetermined'. The calling code must decide what to
  47     do."""
  48
  49     env = os.environ.copy()
  50     env['LC_ALL'] = 'C'
  51     args = ['/usr/sbin/unsquashfs', '-s', sqfs_img]
  52     try:
  53         p = subprocess.Popen(args, stdout=subprocess.PIPE,
  54                              stderr=subprocess.PIPE, env=env)
  55         out, err = p.communicate()
  56     except OSError, e:
  57         raise SquashfsError(u"Error white stat-ing '%s'\n'%s'" % (args, e))
  58     except:
  59         raise SquashfsError(u"Error while stat-ing '%s'" % args)
  60     else:
  61         if p.returncode != 0:
  62             raise SquashfsError(
  63                 u"Error while stat-ing '%s'\n'%s'\nreturncode: '%s'" %
  64                 (args, err, p.returncode))
  65         else:
  66             compress_type = 'undetermined'
  67             for l in out.splitlines():
  68                 if l.split(None, 1)[0] == 'Compression':
  69                     compress_type = l.split()[1]
  70                     break
  71     return compress_type
  72
  73 def mksquashfs(in_img, out_img, compress_type):
  74 # Allow gzip to work for older versions of mksquashfs
  75     if not compress_type or compress_type == "gzip":
  76         args = ["/sbin/mksquashfs", in_img, out_img]
  77     else:
  78         args = ["/sbin/mksquashfs", in_img, out_img, "-comp", compress_type]
  79
  80     if not sys.stdout.isatty():
  81         args.append("-no-progress")
  82
  83     ret = call(args)
  84     if ret != 0:
  85         raise SquashfsError("'%s' exited with error (%d)" %
  86                             (string.join(args, " "), ret))
  87
  88 def resize2fs(fs, size = None, minimal = False, tmpdir = "/tmp"):
  89     if minimal and size is not None:
  90         raise ResizeError("Can't specify both minimal and a size for resize!")
  91     if not minimal and size is None:
  92         raise ResizeError("Must specify either a size or minimal for resize!")
  93
  94     e2fsck(fs)
  95
  96     logging.info("resizing %s" % (fs,))
  97     args = ["/sbin/resize2fs", fs]
  98     if minimal:
  99         args.append("-M")
 100     else:
 101         args.append("%sK" %(size / 1024,))
 102     ret = call(args)
 103     if ret != 0:
 104         raise ResizeError("resize2fs returned an error (%d)!" % (ret,))
 105
 106     ret = e2fsck(fs)
 107     if ret != 0:
 108         raise ResizeError("fsck after resize returned an error (%d)!" % (ret,))
 109
 110     return 0
 111
 112 def e2fsck(fs):
 113     logging.info("Checking filesystem %s" % fs)
 114     return call(["/sbin/e2fsck", "-f", "-y", fs])
 115
 116 class BindChrootMount:
 117     """Represents a bind mount of a directory into a chroot."""
 118     def __init__(self, src, chroot, dest = None):
 119         self.src = src
 120         self.root = chroot
 121
 122         if not dest:
 123             dest = src
 124         self.dest = self.root + "/" + dest
 125
 126         self.mounted = False
 127
 128     def mount(self):
 129         if self.mounted:
 130             return
 131
 132         makedirs(self.dest)
 133         rc = call(["/bin/mount", "--bind", self.src, self.dest])
 134         if rc != 0:
 135             raise MountError("Bind-mounting '%s' to '%s' failed" %
 136                              (self.src, self.dest))
 137         self.mounted = True
 138
 139     def unmount(self):
 140         if not self.mounted:
 141             return
 142
 143         rc = call(["/bin/umount", self.dest])
 144         if rc != 0:
 145             logging.info("Unable to unmount %s normally, using lazy unmount" % self.dest)
 146             rc = call(["/bin/umount", "-l", self.dest])
 147             if rc != 0:
 148                 raise MountError("Unable to unmount fs at %s" % self.dest)
 149             else:
 150                 logging.info("lazy umount succeeded on %s" % self.dest)
 151                 print >> sys.stdout, "lazy umount succeeded on %s" % self.dest
 152
 153         self.mounted = False
 154
 155 class LoopbackMount:
 156     """LoopbackMount  compatibility layer for old API"""
 157     def __init__(self, lofile, mountdir, fstype = None):
 158         self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
 159         self.losetup = False
 160
 161     def cleanup(self):
 162         self.diskmount.cleanup()
 163
 164     def unmount(self):
 165         self.diskmount.unmount()
 166
 167     def lounsetup(self):
 168         if self.losetup:
 169             rc = call(["/sbin/losetup", "-d", self.loopdev])
 170             self.losetup = False
 171             self.loopdev = None
 172
 173     def loopsetup(self):
 174         if self.losetup:
 175             return
 176
 177         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 178                                        stdout=subprocess.PIPE)
 179         losetupOutput = losetupProc.communicate()[0]
 180
 181         if losetupProc.returncode:
 182             raise MountError("Failed to allocate loop device for '%s'" %
 183                              self.lofile)
 184
 185         self.loopdev = losetupOutput.split()[0]
 186
 187         rc = call(["/sbin/losetup", self.loopdev, self.lofile])
 188         if rc != 0:
 189             raise MountError("Failed to allocate loop device for '%s'" %
 190                              self.lofile)
 191
 192         self.losetup = True
 193
 194     def mount(self):
 195         self.diskmount.mount()
 196
 197 class SparseLoopbackMount(LoopbackMount):
 198     """SparseLoopbackMount  compatibility layer for old API"""
 199     def __init__(self, lofile, mountdir, size, fstype = None):
 200         self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
 201
 202     def expand(self, create = False, size = None):
 203         self.diskmount.disk.expand(create, size)
 204
 205     def truncate(self, size = None):
 206         self.diskmount.disk.truncate(size)
 207
 208     def create(self):
 209         self.diskmount.disk.create()
 210
 211 class SparseExtLoopbackMount(SparseLoopbackMount):
 212     """SparseExtLoopbackMount  compatibility layer for old API"""
 213     def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
 214         self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size),
 215                                       mountdir, fstype, blocksize, fslabel,
 216                                       rmmountdir = True, tmpdir = "/tmp")
 217
 218
 219     def __format_filesystem(self):
 220         self.diskmount.__format_filesystem()
 221
 222     def create(self):
 223         self.diskmount.disk.create()
 224
 225     def resize(self, size = None):
 226         return self.diskmount.__resize_filesystem(size)
 227
 228     def mount(self):
 229         self.diskmount.mount()
 230
 231     def __fsck(self):
 232         self.extdiskmount.__fsck()
 233
 234     def __get_size_from_filesystem(self):
 235         return self.diskmount.__get_size_from_filesystem()
 236
 237     def __resize_to_minimal(self):
 238         return self.diskmount.__resize_to_minimal()
 239
 240     def resparse(self, size = None):
 241         return self.diskmount.resparse(size)
 242
 243 class Disk:
 244     """Generic base object for a disk
 245
 246     The 'create' method must make the disk visible as a block device - eg
 247     by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
 248     method must undo the 'create' operation.
 249     """
 250     def __init__(self, size, device = None):
 251         self._device = device
 252         self._size = size
 253
 254     def create(self):
 255         pass
 256
 257     def cleanup(self):
 258         pass
 259
 260     def get_device(self):
 261         return self._device
 262     def set_device(self, path):
 263         self._device = path
 264     device = property(get_device, set_device)
 265
 266     def get_size(self):
 267         return self._size
 268     size = property(get_size)
 269
 270
 271 class RawDisk(Disk):
 272     """A Disk backed by a block device.
 273     Note that create() is a no-op.
 274     """
 275     def __init__(self, size, device):
 276         Disk.__init__(self, size, device)
 277
 278     def fixed(self):
 279         return True
 280
 281     def exists(self):
 282         return True
 283
 284 class LoopbackDisk(Disk):
 285     """A Disk backed by a file via the loop module."""
 286     def __init__(self, lofile, size):
 287         Disk.__init__(self, size)
 288         self.lofile = lofile
 289
 290     def fixed(self):
 291         return False
 292
 293     def exists(self):
 294         return os.path.exists(self.lofile)
 295
 296     def create(self):
 297         if self.device is not None:
 298             return
 299
 300         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 301                                        stdout=subprocess.PIPE)
 302         losetupOutput = losetupProc.communicate()[0]
 303
 304         if losetupProc.returncode:
 305             raise MountError("Failed to allocate loop device for '%s'" %
 306                              self.lofile)
 307
 308         device = losetupOutput.split()[0]
 309
 310         logging.info("Losetup add %s mapping to %s"  % (device, self.lofile))
 311         rc = call(["/sbin/losetup", device, self.lofile])
 312         if rc != 0:
 313             raise MountError("Failed to allocate loop device for '%s'" %
 314                              self.lofile)
 315         self.device = device
 316
 317     def cleanup(self):
 318         if self.device is None:
 319             return
 320         logging.info("Losetup remove %s" % self.device)
 321         rc = call(["/sbin/losetup", "-d", self.device])
 322         self.device = None
 323
 324
 325
 326 class SparseLoopbackDisk(LoopbackDisk):
 327     """A Disk backed by a sparse file via the loop module."""
 328     def __init__(self, lofile, size):
 329         LoopbackDisk.__init__(self, lofile, size)
 330
 331     def expand(self, create = False, size = None):
 332         flags = os.O_WRONLY
 333         if create:
 334             flags |= os.O_CREAT
 335             makedirs(os.path.dirname(self.lofile))
 336
 337         if size is None:
 338             size = self.size
 339
 340         logging.info("Extending sparse file %s to %d" % (self.lofile, size))
 341         fd = os.open(self.lofile, flags)
 342
 343         if size <= 0:
 344             size = 1
 345         os.lseek(fd, size-1, 0)
 346         os.write(fd, '\x00')
 347         os.close(fd)
 348
 349     def truncate(self, size = None):
 350         if size is None:
 351             size = self.size
 352
 353         logging.info("Truncating sparse file %s to %d" % (self.lofile, size))
 354         fd = os.open(self.lofile, os.O_WRONLY)
 355         os.ftruncate(fd, size)
 356         os.close(fd)
 357
 358     def create(self):
 359         self.expand(create = True)
 360         LoopbackDisk.create(self)
 361
 362 class Mount:
 363     """A generic base class to deal with mounting things."""
 364     def __init__(self, mountdir):
 365         self.mountdir = mountdir
 366
 367     def cleanup(self):
 368         self.unmount()
 369
 370     def mount(self):
 371         pass
 372
 373     def unmount(self):
 374         pass
 375
 376 class DiskMount(Mount):
 377     """A Mount object that handles mounting of a Disk."""
 378     def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
 379         Mount.__init__(self, mountdir)
 380
 381         self.disk = disk
 382         self.fstype = fstype
 383         self.rmmountdir = rmmountdir
 384
 385         self.mounted = False
 386         self.rmdir   = False
 387
 388     def cleanup(self):
 389         Mount.cleanup(self)
 390         self.disk.cleanup()
 391
 392     def unmount(self):
 393         if self.mounted:
 394             logging.info("Unmounting directory %s" % self.mountdir)
 395             rc = call(["/bin/umount", self.mountdir])
 396             if rc == 0:
 397                 self.mounted = False
 398             else:
 399                 logging.warn("Unmounting directory %s failed, using lazy umount" % self.mountdir)
 400                 print >> sys.stdout, "Unmounting directory %s failed, using lazy umount" %self.mountdir
 401                 rc = call(["/bin/umount", "-l", self.mountdir])
 402                 if rc != 0:
 403                     raise MountError("Unable to unmount filesystem at %s" % self.mountdir)
 404                 else:
 405                     logging.info("lazy umount succeeded on %s" % self.mountdir)
 406                     print >> sys.stdout, "lazy umount succeeded on %s" % self.mountdir
 407                     self.mounted = False
 408
 409         if self.rmdir and not self.mounted:
 410             try:
 411                 os.rmdir(self.mountdir)
 412             except OSError, e:
 413                 pass
 414             self.rmdir = False
 415
 416
 417     def __create(self):
 418         self.disk.create()
 419
 420
 421     def mount(self):
 422         if self.mounted:
 423             return
 424
 425         if not os.path.isdir(self.mountdir):
 426             logging.info("Creating mount point %s" % self.mountdir)
 427             os.makedirs(self.mountdir)
 428             self.rmdir = self.rmmountdir
 429
 430         self.__create()
 431
 432         logging.info("Mounting %s at %s" % (self.disk.device, self.mountdir))
 433         args = [ "/bin/mount", self.disk.device, self.mountdir ]
 434         if self.fstype:
 435             args.extend(["-t", self.fstype])
 436         if self.fstype == "squashfs":
 437             args.extend(["-o", "ro"])
 438         rc = call(args)
 439         if rc != 0:
 440             raise MountError("Failed to mount '%s' to '%s'" %
 441                              (self.disk.device, self.mountdir))
 442
 443         self.mounted = True
 444
 445 class ExtDiskMount(DiskMount):
 446     """A DiskMount object that is able to format/resize ext[23] filesystems."""
 447     def __init__(self, disk, mountdir, fstype, blocksize, fslabel,
 448                  rmmountdir=True, tmpdir="/tmp"):
 449         DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
 450         self.blocksize = blocksize
 451         self.fslabel = "_" + fslabel
 452         self.tmpdir = tmpdir
 453
 454     def __format_filesystem(self):
 455         logging.info("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
 456         rc = call(["/sbin/mkfs." + self.fstype,
 457                    "-F", "-L", self.fslabel,
 458                    "-m", "1", "-b", str(self.blocksize),
 459                    self.disk.device])
 460         #          str(self.disk.size / self.blocksize)])
 461
 462         if rc != 0:
 463             raise MountError("Error creating %s filesystem" % (self.fstype,))
 464         logging.info("Tuning filesystem on %s" % self.disk.device)
 465         call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
 466               "-ouser_xattr,acl", self.disk.device])
 467
 468     def __resize_filesystem(self, size = None):
 469         current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
 470
 471         if size is None:
 472             size = self.disk.size
 473
 474         if size == current_size:
 475             return
 476
 477         if size > current_size:
 478             self.disk.expand(size)
 479
 480         resize2fs(self.disk.lofile, size, tmpdir = self.tmpdir)
 481         return size
 482
 483     def __create(self):
 484         resize = False
 485         if not self.disk.fixed() and self.disk.exists():
 486             resize = True
 487
 488         self.disk.create()
 489
 490         if resize:
 491             self.__resize_filesystem()
 492         else:
 493             self.__format_filesystem()
 494
 495     def mount(self):
 496         self.__create()
 497         DiskMount.mount(self)
 498
 499     def __fsck(self):
 500         return e2fsck(self.disk.lofile)
 501         return rc
 502
 503     def __get_size_from_filesystem(self):
 504         def parse_field(output, field):
 505             for line in output.split("\n"):
 506                 if line.startswith(field + ":"):
 507                     return line[len(field) + 1:].strip()
 508
 509             raise KeyError("Failed to find field '%s' in output" % field)
 510
 511         dev_null = os.open("/dev/null", os.O_WRONLY)
 512         try:
 513             out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
 514                                    stdout = subprocess.PIPE,
 515                                    stderr = dev_null).communicate()[0]
 516         finally:
 517             os.close(dev_null)
 518
 519         return int(parse_field(out, "Block count")) * self.blocksize
 520
 521     def __resize_to_minimal(self):
 522         resize2fs(self.disk.lofile, minimal = True, tmpdir = self.tmpdir)
 523         return self.__get_size_from_filesystem()
 524
 525     def resparse(self, size = None):
 526         self.cleanup()
 527         minsize = self.__resize_to_minimal()
 528         self.disk.truncate(minsize)
 529         self.__resize_filesystem(size)
 530         return minsize
 531
 532 class DeviceMapperSnapshot(object):
 533     def __init__(self, imgloop, cowloop):
 534         self.imgloop = imgloop
 535         self.cowloop = cowloop
 536
 537         self.__created = False
 538         self.__name = None
 539
 540     def get_path(self):
 541         if self.__name is None:
 542             return None
 543         return os.path.join("/dev/mapper", self.__name)
 544     path = property(get_path)
 545
 546     def create(self):
 547         if self.__created:
 548             return
 549
 550         self.imgloop.create()
 551         self.cowloop.create()
 552
 553         self.__name = "imgcreate-%d-%d" % (os.getpid(),
 554                                            random.randint(0, 2**16))
 555
 556         size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
 557
 558         table = "0 %d snapshot %s %s p 8" % (size / 512,
 559                                              self.imgloop.device,
 560                                              self.cowloop.device)
 561
 562         args = ["/sbin/dmsetup", "create", self.__name, "-vv", "--verifyudev",
 563                 "--uuid", "LIVECD-%s" % self.__name, "--table", table]
 564         if call(args) != 0:
 565             self.cowloop.cleanup()
 566             self.imgloop.cleanup()
 567             raise SnapshotError("Could not create snapshot device using: " +
 568                                 string.join(args, " "))
 569
 570         self.__created = True
 571
 572     def remove(self, ignore_errors = False):
 573         if not self.__created:
 574             return
 575
 576         # sleep to try to avoid any dm shenanigans
 577         time.sleep(2)
 578         rc = call(["/sbin/dmsetup", "remove", self.__name])
 579         if not ignore_errors and rc != 0:
 580             raise SnapshotError("Could not remove snapshot device")
 581
 582         self.__name = None
 583         self.__created = False
 584
 585         self.cowloop.cleanup()
 586         self.imgloop.cleanup()
 587
 588     def get_cow_used(self):
 589         if not self.__created:
 590             return 0
 591
 592         dev_null = os.open("/dev/null", os.O_WRONLY)
 593         try:
 594             out = subprocess.Popen(["/sbin/dmsetup", "status", self.__name],
 595                                    stdout = subprocess.PIPE,
 596                                    stderr = dev_null).communicate()[0]
 597         finally:
 598             os.close(dev_null)
 599
 600         #
 601         # dmsetup status on a snapshot returns e.g.
 602         #   "0 8388608 snapshot 416/1048576"
 603         # or, more generally:
 604         #   "A B snapshot C/D"
 605         # where C is the number of 512 byte sectors in use
 606         #
 607         try:
 608             return int((out.split()[3]).split('/')[0]) * 512
 609         except ValueError:
 610             raise SnapshotError("Failed to parse dmsetup status: " + out)
 611
 612 def create_image_minimizer(path, image, compress_type, target_size = None,
 613                            tmpdir = "/tmp"):
 614     """
 615     Builds a copy-on-write image which can be used to
 616     create a device-mapper snapshot of an image where
 617     the image's filesystem is as small as possible
 618
 619     The steps taken are:
 620       1) Create a sparse COW
 621       2) Loopback mount the image and the COW
 622       3) Create a device-mapper snapshot of the image
 623          using the COW
 624       4) Resize the filesystem to the minimal size
 625       5) Determine the amount of space used in the COW
 626       6) Restroy the device-mapper snapshot
 627       7) Truncate the COW, removing unused space
 628       8) Create a squashfs of the COW
 629     """
 630     imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
 631
 632     cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
 633                                  64L * 1024L * 1024L)
 634
 635     snapshot = DeviceMapperSnapshot(imgloop, cowloop)
 636
 637     try:
 638         snapshot.create()
 639
 640         if target_size is not None:
 641             resize2fs(snapshot.path, target_size, tmpdir = tmpdir)
 642         else:
 643             resize2fs(snapshot.path, minimal = True, tmpdir = tmpdir)
 644
 645         cow_used = snapshot.get_cow_used()
 646     finally:
 647         snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
 648
 649     cowloop.truncate(cow_used)
 650
 651     mksquashfs(cowloop.lofile, path, compress_type)
 652
 653     os.unlink(cowloop.lofile)
 654