imgcreate/fs.py

   1 #
   2 # fs.py : Filesystem related utilities and classes
   3 #
   4 # Copyright 2007, Red Hat  Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; version 2 of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import os.path
  21 import sys
  22 import errno
  23 import stat
  24 import subprocess
  25 import random
  26 import string
  27 import logging
  28 import tempfile
  29 import time
  30 from util import call
  31
  32 from imgcreate.errors import *
  33
  34 def makedirs(dirname):
  35     """A version of os.makedirs() that doesn't throw an
  36     exception if the leaf directory already exists.
  37     """
  38     try:
  39         os.makedirs(dirname)
  40     except OSError, e:
  41         if e.errno != errno.EEXIST:
  42             raise
  43
  44 def squashfs_compression_type(sqfs_img):
  45     """Check the compression type of a SquashFS image. If the type cannot be
  46     ascertained, return 'undetermined'. The calling code must decide what to
  47     do."""
  48
  49     env = os.environ.copy()
  50     env['LC_ALL'] = 'C'
  51     args = ['/usr/sbin/unsquashfs', '-s', sqfs_img]
  52     try:
  53         p = subprocess.Popen(args, stdout=subprocess.PIPE,
  54                              stderr=subprocess.PIPE, env=env)
  55         out, err = p.communicate()
  56     except OSError, e:
  57         raise SquashfsError(u"Error white stat-ing '%s'\n'%s'" % (args, e))
  58     except:
  59         raise SquashfsError(u"Error while stat-ing '%s'" % args)
  60     else:
  61         if p.returncode != 0:
  62             raise SquashfsError(
  63                 u"Error while stat-ing '%s'\n'%s'\nreturncode: '%s'" %
  64                 (args, err, p.returncode))
  65         else:
  66             compress_type = 'undetermined'
  67             for l in out.splitlines():
  68                 if l.split(None, 1)[0] == 'Compression':
  69                     compress_type = l.split()[1]
  70                     break
  71     return compress_type
  72
  73 def mksquashfs(in_img, out_img, compress_type):
  74 # Allow gzip to work for older versions of mksquashfs
  75     if not compress_type or compress_type == "gzip":
  76         args = ["/sbin/mksquashfs", in_img, out_img]
  77     else:
  78         args = ["/sbin/mksquashfs", in_img, out_img, "-comp", compress_type]
  79
  80     if not sys.stdout.isatty():
  81         args.append("-no-progress")
  82
  83     ret = call(args)
  84     if ret != 0:
  85         raise SquashfsError("'%s' exited with error (%d)" %
  86                             (string.join(args, " "), ret))
  87
  88 def resize2fs(fs, size = None, minimal = False, tmpdir = "/tmp"):
  89     if minimal and size is not None:
  90         raise ResizeError("Can't specify both minimal and a size for resize!")
  91     if not minimal and size is None:
  92         raise ResizeError("Must specify either a size or minimal for resize!")
  93
  94     e2fsck(fs)
  95
  96     logging.info("resizing %s" % (fs,))
  97     args = ["/sbin/resize2fs", fs]
  98     if minimal:
  99         args.append("-M")
 100     else:
 101         args.append("%sK" %(size / 1024,))
 102     ret = call(args)
 103     if ret != 0:
 104         raise ResizeError("resize2fs returned an error (%d)!" % (ret,))
 105
 106     ret = e2fsck(fs)
 107     if ret != 0:
 108         raise ResizeError("fsck after resize returned an error (%d)!" % (ret,))
 109
 110     return 0
 111
 112 def e2fsck(fs):
 113     logging.info("Checking filesystem %s" % fs)
 114     return call(["/sbin/e2fsck", "-f", "-y", fs])
 115
 116 class BindChrootMount:
 117     """Represents a bind mount of a directory into a chroot."""
 118     def __init__(self, src, chroot, dest = None):
 119         self.src = src
 120         self.root = chroot
 121
 122         if not dest:
 123             dest = src
 124         self.dest = self.root + "/" + dest
 125
 126         self.mounted = False
 127
 128     def mount(self):
 129         if self.mounted:
 130             return
 131
 132         makedirs(self.dest)
 133         rc = call(["/bin/mount", "--bind", self.src, self.dest])
 134         if rc != 0:
 135             raise MountError("Bind-mounting '%s' to '%s' failed" %
 136                              (self.src, self.dest))
 137         self.mounted = True
 138
 139     def unmount(self):
 140         if not self.mounted:
 141             return
 142
 143         rc = call(["/bin/umount", self.dest])
 144         if rc != 0:
 145             logging.info("Unable to unmount %s normally, using lazy unmount" % self.dest)
 146             rc = call(["/bin/umount", "-l", self.dest])
 147             if rc != 0:
 148                 raise MountError("Unable to unmount fs at %s" % self.dest)
 149             else:
 150                 logging.info("lazy umount succeeded on %s" % self.dest)
 151                 print >> sys.stdout, "lazy umount succeeded on %s" % self.dest
 152
 153         self.mounted = False
 154
 155 class LoopbackMount:
 156     """LoopbackMount  compatibility layer for old API"""
 157     def __init__(self, lofile, mountdir, fstype = None):
 158         self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
 159         self.losetup = False
 160
 161     def cleanup(self):
 162         self.diskmount.cleanup()
 163
 164     def unmount(self):
 165         self.diskmount.unmount()
 166
 167     def lounsetup(self):
 168         if self.losetup:
 169             rc = call(["/sbin/losetup", "-d", self.loopdev])
 170             self.losetup = False
 171             self.loopdev = None
 172
 173     def loopsetup(self):
 174         if self.losetup:
 175             return
 176
 177         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 178                                        stdout=subprocess.PIPE)
 179         losetupOutput = losetupProc.communicate()[0]
 180
 181         if losetupProc.returncode:
 182             raise MountError("Failed to allocate loop device for '%s'" %
 183                              self.lofile)
 184
 185         self.loopdev = losetupOutput.split()[0]
 186
 187         rc = call(["/sbin/losetup", self.loopdev, self.lofile])
 188         if rc != 0:
 189             raise MountError("Failed to allocate loop device for '%s'" %
 190                              self.lofile)
 191
 192         self.losetup = True
 193
 194     def mount(self):
 195         self.diskmount.mount()
 196
 197 class SparseLoopbackMount(LoopbackMount):
 198     """SparseLoopbackMount  compatibility layer for old API"""
 199     def __init__(self, lofile, mountdir, size, fstype = None):
 200         self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
 201
 202     def expand(self, create = False, size = None):
 203         self.diskmount.disk.expand(create, size)
 204
 205     def truncate(self, size = None):
 206         self.diskmount.disk.truncate(size)
 207
 208     def create(self):
 209         self.diskmount.disk.create()
 210
 211 class SparseExtLoopbackMount(SparseLoopbackMount):
 212     """SparseExtLoopbackMount  compatibility layer for old API"""
 213     def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
 214         self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size),
 215                                       mountdir, fstype, blocksize, fslabel,
 216                                       rmmountdir = True, tmpdir = "/tmp")
 217
 218
 219     def __format_filesystem(self):
 220         self.diskmount.__format_filesystem()
 221
 222     def create(self):
 223         self.diskmount.disk.create()
 224
 225     def resize(self, size = None):
 226         return self.diskmount.__resize_filesystem(size)
 227
 228     def mount(self):
 229         self.diskmount.mount()
 230
 231     def __fsck(self):
 232         self.extdiskmount.__fsck()
 233
 234     def __get_size_from_filesystem(self):
 235         return self.diskmount.__get_size_from_filesystem()
 236
 237     def __resize_to_minimal(self):
 238         return self.diskmount.__resize_to_minimal()
 239
 240     def resparse(self, size = None):
 241         return self.diskmount.resparse(size)
 242
 243 class Disk:
 244     """Generic base object for a disk
 245
 246     The 'create' method must make the disk visible as a block device - eg
 247     by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
 248     method must undo the 'create' operation.
 249     """
 250     def __init__(self, size, device = None):
 251         self._device = device
 252         self._size = size
 253
 254     def create(self):
 255         pass
 256
 257     def cleanup(self):
 258         pass
 259
 260     def get_device(self):
 261         return self._device
 262     def set_device(self, path):
 263         self._device = path
 264     device = property(get_device, set_device)
 265
 266     def get_size(self):
 267         return self._size
 268     size = property(get_size)
 269
 270
 271 class RawDisk(Disk):
 272     """A Disk backed by a block device.
 273     Note that create() is a no-op.
 274     """
 275     def __init__(self, size, device):
 276         Disk.__init__(self, size, device)
 277
 278     def fixed(self):
 279         return True
 280
 281     def exists(self):
 282         return True
 283
 284 class LoopbackDisk(Disk):
 285     """A Disk backed by a file via the loop module."""
 286     def __init__(self, lofile, size):
 287         Disk.__init__(self, size)
 288         self.lofile = lofile
 289
 290     def fixed(self):
 291         return False
 292
 293     def exists(self):
 294         return os.path.exists(self.lofile)
 295
 296     def create(self):
 297         if self.device is not None:
 298             return
 299
 300         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 301                                        stdout=subprocess.PIPE)
 302         losetupOutput = losetupProc.communicate()[0]
 303
 304         if losetupProc.returncode:
 305             raise MountError("Failed to allocate loop device for '%s'" %
 306                              self.lofile)
 307
 308         device = losetupOutput.split()[0]
 309
 310         logging.info("Losetup add %s mapping to %s"  % (device, self.lofile))
 311         rc = call(["/sbin/losetup", device, self.lofile])
 312         if rc != 0:
 313             raise MountError("Failed to allocate loop device for '%s'" %
 314                              self.lofile)
 315         self.device = device
 316
 317     def cleanup(self):
 318         if self.device is None:
 319             return
 320         logging.info("Losetup remove %s" % self.device)
 321         rc = call(["/sbin/losetup", "-d", self.device])
 322         self.device = None
 323
 324
 325
 326 class SparseLoopbackDisk(LoopbackDisk):
 327     """A Disk backed by a sparse file via the loop module."""
 328     def __init__(self, lofile, size):
 329         LoopbackDisk.__init__(self, lofile, size)
 330
 331     def expand(self, create = False, size = None):
 332         flags = os.O_WRONLY
 333         if create:
 334             flags |= os.O_CREAT
 335             makedirs(os.path.dirname(self.lofile))
 336
 337         if size is None:
 338             size = self.size
 339
 340         logging.info("Extending sparse file %s to %d" % (self.lofile, size))
 341         fd = os.open(self.lofile, flags)
 342
 343         if size <= 0:
 344             size = 1
 345         os.lseek(fd, size-1, 0)
 346         os.write(fd, '\x00')
 347         os.close(fd)
 348
 349     def truncate(self, size = None):
 350         if size is None:
 351             size = self.size
 352
 353         logging.info("Truncating sparse file %s to %d" % (self.lofile, size))
 354         fd = os.open(self.lofile, os.O_WRONLY)
 355         os.ftruncate(fd, size)
 356         os.close(fd)
 357
 358     def create(self):
 359         self.expand(create = True)
 360         LoopbackDisk.create(self)
 361
 362 class Mount:
 363     """A generic base class to deal with mounting things."""
 364     def __init__(self, mountdir):
 365         self.mountdir = mountdir
 366
 367     def cleanup(self):
 368         self.unmount()
 369
 370     def mount(self):
 371         pass
 372
 373     def unmount(self):
 374         pass
 375
 376 class DiskMount(Mount):
 377     """A Mount object that handles mounting of a Disk."""
 378     def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
 379         Mount.__init__(self, mountdir)
 380
 381         self.disk = disk
 382         self.fstype = fstype
 383         self.rmmountdir = rmmountdir
 384
 385         self.mounted = False
 386         self.rmdir   = False
 387
 388     def cleanup(self):
 389         Mount.cleanup(self)
 390         self.disk.cleanup()
 391
 392     def unmount(self):
 393         if self.mounted:
 394             logging.info("Unmounting directory %s" % self.mountdir)
 395             rc = call(["/bin/umount", self.mountdir])
 396             if rc == 0:
 397                 self.mounted = False
 398             else:
 399                 logging.warn("Unmounting directory %s failed, using lazy umount" % self.mountdir)
 400                 print >> sys.stdout, "Unmounting directory %s failed, using lazy umount" %self.mountdir
 401                 rc = call(["/bin/umount", "-l", self.mountdir])
 402                 if rc != 0:
 403                     raise MountError("Unable to unmount filesystem at %s" % self.mountdir)
 404                 else:
 405                     logging.info("lazy umount succeeded on %s" % self.mountdir)
 406                     print >> sys.stdout, "lazy umount succeeded on %s" % self.mountdir
 407                     self.mounted = False
 408
 409         if self.rmdir and not self.mounted:
 410             try:
 411                 os.rmdir(self.mountdir)
 412             except OSError, e:
 413                 pass
 414             self.rmdir = False
 415
 416
 417     def __create(self):
 418         self.disk.create()
 419
 420
 421     def mount(self):
 422         if self.mounted:
 423             return
 424
 425         if not os.path.isdir(self.mountdir):
 426             logging.info("Creating mount point %s" % self.mountdir)
 427             os.makedirs(self.mountdir)
 428             self.rmdir = self.rmmountdir
 429
 430         self.__create()
 431
 432         logging.info("Mounting %s at %s" % (self.disk.device, self.mountdir))
 433         args = [ "/bin/mount", self.disk.device, self.mountdir ]
 434         if self.fstype:
 435             args.extend(["-t", self.fstype])
 436         if self.fstype == "squashfs":
 437             args.extend(["-o", "ro"])
 438         rc = call(args)
 439         if rc != 0:
 440             raise MountError("Failed to mount '%s' to '%s'" %
 441                              (self.disk.device, self.mountdir))
 442
 443         self.mounted = True
 444
 445 class ExtDiskMount(DiskMount):
 446     """A DiskMount object that is able to format/resize ext[23] filesystems."""
 447     def __init__(self, disk, mountdir, fstype, blocksize, fslabel,
 448                  rmmountdir=True, tmpdir="/tmp"):
 449         DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
 450         self.blocksize = blocksize
 451         self.fslabel = "_" + fslabel
 452         self.tmpdir = tmpdir
 453
 454     def __format_filesystem(self):
 455         logging.info("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
 456         args = [ "/sbin/mkfs." + self.fstype ]
 457         if self.fstype.startswith("ext"):
 458             args = args + [ "-F", "-L", self.fslabel, "-m", "1", "-b", str(self.blocksize) ]
 459         elif self.fstype == "xfs":
 460             args = args + [ "-L", self.fslabel[0:10], "-b", "size=%s" % str(self.blocksize) ]
 461         elif self.fstype == "btrfs":
 462             args = args + [ "-L", self.fslabel ]
 463         args = args + [self.disk.device]
 464         print args
 465         rc = call(args)
 466
 467         if rc != 0:
 468             raise MountError("Error creating %s filesystem" % (self.fstype,))
 469         logging.info("Tuning filesystem on %s" % self.disk.device)
 470         call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
 471               "-ouser_xattr,acl", self.disk.device])
 472
 473     def __resize_filesystem(self, size = None):
 474         current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
 475
 476         if size is None:
 477             size = self.disk.size
 478
 479         if size == current_size:
 480             return
 481
 482         if size > current_size:
 483             self.disk.expand(size)
 484
 485         resize2fs(self.disk.lofile, size, tmpdir = self.tmpdir)
 486         return size
 487
 488     def __create(self):
 489         resize = False
 490         if not self.disk.fixed() and self.disk.exists():
 491             resize = True
 492
 493         self.disk.create()
 494
 495         if resize:
 496             self.__resize_filesystem()
 497         else:
 498             self.__format_filesystem()
 499
 500     def mount(self):
 501         self.__create()
 502         DiskMount.mount(self)
 503
 504     def __fsck(self):
 505         return e2fsck(self.disk.lofile)
 506         return rc
 507
 508     def __get_size_from_filesystem(self):
 509         def parse_field(output, field):
 510             for line in output.split("\n"):
 511                 if line.startswith(field + ":"):
 512                     return line[len(field) + 1:].strip()
 513
 514             raise KeyError("Failed to find field '%s' in output" % field)
 515
 516         dev_null = os.open("/dev/null", os.O_WRONLY)
 517         try:
 518             out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
 519                                    stdout = subprocess.PIPE,
 520                                    stderr = dev_null).communicate()[0]
 521         finally:
 522             os.close(dev_null)
 523
 524         return int(parse_field(out, "Block count")) * self.blocksize
 525
 526     def __resize_to_minimal(self):
 527         resize2fs(self.disk.lofile, minimal = True, tmpdir = self.tmpdir)
 528         return self.__get_size_from_filesystem()
 529
 530     def resparse(self, size = None):
 531         self.cleanup()
 532         minsize = self.__resize_to_minimal()
 533         self.disk.truncate(minsize)
 534         self.__resize_filesystem(size)
 535         return minsize
 536
 537 class DeviceMapperSnapshot(object):
 538     def __init__(self, imgloop, cowloop):
 539         self.imgloop = imgloop
 540         self.cowloop = cowloop
 541
 542         self.__created = False
 543         self.__name = None
 544
 545     def get_path(self):
 546         if self.__name is None:
 547             return None
 548         return os.path.join("/dev/mapper", self.__name)
 549     path = property(get_path)
 550
 551     def create(self):
 552         if self.__created:
 553             return
 554
 555         self.imgloop.create()
 556         self.cowloop.create()
 557
 558         self.__name = "imgcreate-%d-%d" % (os.getpid(),
 559                                            random.randint(0, 2**16))
 560
 561         size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
 562
 563         table = "0 %d snapshot %s %s p 8" % (size / 512,
 564                                              self.imgloop.device,
 565                                              self.cowloop.device)
 566
 567         args = ["/sbin/dmsetup", "create", self.__name, "-vv", "--verifyudev",
 568                 "--uuid", "LIVECD-%s" % self.__name, "--table", table]
 569         if call(args) != 0:
 570             self.cowloop.cleanup()
 571             self.imgloop.cleanup()
 572             raise SnapshotError("Could not create snapshot device using: " +
 573                                 string.join(args, " "))
 574
 575         self.__created = True
 576
 577     def remove(self, ignore_errors = False):
 578         if not self.__created:
 579             return
 580
 581         # sleep to try to avoid any dm shenanigans
 582         time.sleep(2)
 583         rc = call(["/sbin/dmsetup", "remove", self.__name])
 584         if not ignore_errors and rc != 0:
 585             raise SnapshotError("Could not remove snapshot device")
 586
 587         self.__name = None
 588         self.__created = False
 589
 590         self.cowloop.cleanup()
 591         self.imgloop.cleanup()
 592
 593     def get_cow_used(self):
 594         if not self.__created:
 595             return 0
 596
 597         dev_null = os.open("/dev/null", os.O_WRONLY)
 598         try:
 599             out = subprocess.Popen(["/sbin/dmsetup", "status", self.__name],
 600                                    stdout = subprocess.PIPE,
 601                                    stderr = dev_null).communicate()[0]
 602         finally:
 603             os.close(dev_null)
 604
 605         #
 606         # dmsetup status on a snapshot returns e.g.
 607         #   "0 8388608 snapshot 416/1048576"
 608         # or, more generally:
 609         #   "A B snapshot C/D"
 610         # where C is the number of 512 byte sectors in use
 611         #
 612         try:
 613             return int((out.split()[3]).split('/')[0]) * 512
 614         except ValueError:
 615             raise SnapshotError("Failed to parse dmsetup status: " + out)
 616
 617 def create_image_minimizer(path, image, compress_type, target_size = None,
 618                            tmpdir = "/tmp"):
 619     """
 620     Builds a copy-on-write image which can be used to
 621     create a device-mapper snapshot of an image where
 622     the image's filesystem is as small as possible
 623
 624     The steps taken are:
 625       1) Create a sparse COW
 626       2) Loopback mount the image and the COW
 627       3) Create a device-mapper snapshot of the image
 628          using the COW
 629       4) Resize the filesystem to the minimal size
 630       5) Determine the amount of space used in the COW
 631       6) Restroy the device-mapper snapshot
 632       7) Truncate the COW, removing unused space
 633       8) Create a squashfs of the COW
 634     """
 635     imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
 636
 637     cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
 638                                  64L * 1024L * 1024L)
 639
 640     snapshot = DeviceMapperSnapshot(imgloop, cowloop)
 641
 642     try:
 643         snapshot.create()
 644
 645         if target_size is not None:
 646             resize2fs(snapshot.path, target_size, tmpdir = tmpdir)
 647         else:
 648             resize2fs(snapshot.path, minimal = True, tmpdir = tmpdir)
 649
 650         cow_used = snapshot.get_cow_used()
 651     finally:
 652         snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
 653
 654     cowloop.truncate(cow_used)
 655
 656     mksquashfs(cowloop.lofile, path, compress_type)
 657
 658     os.unlink(cowloop.lofile)
 659