imgcreate/fs.py

   1 #
   2 # fs.py : Filesystem related utilities and classes
   3 #
   4 # Copyright 2007, Red Hat  Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; version 2 of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import os.path
  21 import sys
  22 import errno
  23 import stat
  24 import subprocess
  25 import random
  26 import string
  27 import logging
  28 import tempfile
  29 import time
  30 from util import call
  31
  32 from imgcreate.errors import *
  33
  34 def makedirs(dirname):
  35     """A version of os.makedirs() that doesn't throw an
  36     exception if the leaf directory already exists.
  37     """
  38     try:
  39         os.makedirs(dirname)
  40     except OSError, e:
  41         if e.errno != errno.EEXIST:
  42             raise
  43
  44 def squashfs_compression_type(sqfs_img):
  45     """Check the compression type of a SquashFS image. If the type cannot be
  46     ascertained, return 'undetermined'. The calling code must decide what to
  47     do."""
  48
  49     env = os.environ.copy()
  50     env['LC_ALL'] = 'C'
  51     args = ['/usr/sbin/unsquashfs', '-s', sqfs_img]
  52     try:
  53         p = subprocess.Popen(args, stdout=subprocess.PIPE,
  54                              stderr=subprocess.PIPE, env=env)
  55         out, err = p.communicate()
  56     except OSError, e:
  57         raise SquashfsError(u"Error white stat-ing '%s'\n'%s'" % (args, e))
  58     except:
  59         raise SquashfsError(u"Error while stat-ing '%s'" % args)
  60     else:
  61         if p.returncode != 0:
  62             raise SquashfsError(
  63                 u"Error while stat-ing '%s'\n'%s'\nreturncode: '%s'" %
  64                 (args, err, p.returncode))
  65         else:
  66             compress_type = 'undetermined'
  67             for l in out.splitlines():
  68                 if l.split(None, 1)[0] == 'Compression':
  69                     compress_type = l.split()[1]
  70                     break
  71     return compress_type
  72
  73 def mksquashfs(in_img, out_img, compress_type):
  74 # Allow gzip to work for older versions of mksquashfs
  75     if compress_type == "gzip":
  76         args = ["/sbin/mksquashfs", in_img, out_img]
  77     else:
  78         args = ["/sbin/mksquashfs", in_img, out_img, "-comp", compress_type]
  79
  80     if not sys.stdout.isatty():
  81         args.append("-no-progress")
  82
  83     ret = call(args)
  84     if ret != 0:
  85         raise SquashfsError("'%s' exited with error (%d)" %
  86                             (string.join(args, " "), ret))
  87
  88 def resize2fs(fs, size = None, minimal = False, tmpdir = "/tmp"):
  89     if minimal and size is not None:
  90         raise ResizeError("Can't specify both minimal and a size for resize!")
  91     if not minimal and size is None:
  92         raise ResizeError("Must specify either a size or minimal for resize!")
  93
  94     e2fsck(fs)
  95
  96     logging.info("resizing %s" % (fs,))
  97     args = ["/sbin/resize2fs", fs]
  98     if minimal:
  99         args.append("-M")
 100     else:
 101         args.append("%sK" %(size / 1024,))
 102     ret = call(args)
 103     if ret != 0:
 104         raise ResizeError("resize2fs returned an error (%d)!" % (ret,))
 105
 106     ret = e2fsck(fs)
 107     if ret != 0:
 108         raise ResizeError("fsck after resize returned an error (%d)!", (ret,))
 109
 110     return 0
 111
 112 def e2fsck(fs):
 113     logging.info("Checking filesystem %s" % fs)
 114     return call(["/sbin/e2fsck", "-f", "-y", fs])
 115
 116 class BindChrootMount:
 117     """Represents a bind mount of a directory into a chroot."""
 118     def __init__(self, src, chroot, dest = None):
 119         self.src = src
 120         self.root = chroot
 121
 122         if not dest:
 123             dest = src
 124         self.dest = self.root + "/" + dest
 125
 126         self.mounted = False
 127
 128     def mount(self):
 129         if self.mounted:
 130             return
 131
 132         makedirs(self.dest)
 133         rc = call(["/bin/mount", "--bind", self.src, self.dest])
 134         if rc != 0:
 135             raise MountError("Bind-mounting '%s' to '%s' failed" %
 136                              (self.src, self.dest))
 137         self.mounted = True
 138
 139     def unmount(self):
 140         if not self.mounted:
 141             return
 142
 143         rc = call(["/bin/umount", self.dest])
 144         if rc != 0:
 145             logging.info("Unable to unmount %s normally, using lazy unmount" % self.dest)
 146             rc = call(["/bin/umount", "-l", self.dest])
 147             if rc != 0:
 148                 raise MountError("Unable to unmount fs at %s" % self.dest)
 149             else:
 150                 logging.info("lazy umount succeeded on %s" % self.dest)
 151                 print >> sys.stdout, "lazy umount succeeded on %s" % self.dest
 152
 153         self.mounted = False
 154
 155 class LoopbackMount:
 156     """LoopbackMount  compatibility layer for old API"""
 157     def __init__(self, lofile, mountdir, fstype = None):
 158         self.diskmount = DiskMount(LoopbackDisk(lofile,size = 0),mountdir,fstype,rmmountdir = True)
 159         self.losetup = False
 160
 161     def cleanup(self):
 162         self.diskmount.cleanup()
 163
 164     def unmount(self):
 165         self.diskmount.unmount()
 166
 167     def lounsetup(self):
 168         if self.losetup:
 169             rc = call(["/sbin/losetup", "-d", self.loopdev])
 170             self.losetup = False
 171             self.loopdev = None
 172
 173     def loopsetup(self):
 174         if self.losetup:
 175             return
 176
 177         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 178                                        stdout=subprocess.PIPE)
 179         losetupOutput = losetupProc.communicate()[0]
 180
 181         if losetupProc.returncode:
 182             raise MountError("Failed to allocate loop device for '%s'" %
 183                              self.lofile)
 184
 185         self.loopdev = losetupOutput.split()[0]
 186
 187         rc = call(["/sbin/losetup", self.loopdev, self.lofile])
 188         if rc != 0:
 189             raise MountError("Failed to allocate loop device for '%s'" %
 190                              self.lofile)
 191
 192         self.losetup = True
 193
 194     def mount(self):
 195         self.diskmount.mount()
 196
 197 class SparseLoopbackMount(LoopbackMount):
 198     """SparseLoopbackMount  compatibility layer for old API"""
 199     def __init__(self, lofile, mountdir, size, fstype = None):
 200         self.diskmount = DiskMount(SparseLoopbackDisk(lofile,size),mountdir,fstype,rmmountdir = True)
 201
 202     def expand(self, create = False, size = None):
 203         self.diskmount.disk.expand(create, size)
 204
 205     def truncate(self, size = None):
 206         self.diskmount.disk.truncate(size)
 207
 208     def create(self):
 209         self.diskmount.disk.create()
 210
 211 class SparseExtLoopbackMount(SparseLoopbackMount):
 212     """SparseExtLoopbackMount  compatibility layer for old API"""
 213     def __init__(self, lofile, mountdir, size, fstype, blocksize, fslabel):
 214         self.diskmount = ExtDiskMount(SparseLoopbackDisk(lofile,size),
 215                                       mountdir, fstype, blocksize, fslabel,
 216                                       rmmountdir = True, tmpdir = "/tmp")
 217
 218
 219     def __format_filesystem(self):
 220         self.diskmount.__format_filesystem()
 221
 222     def create(self):
 223         self.diskmount.disk.create()
 224
 225     def resize(self, size = None):
 226         return self.diskmount.__resize_filesystem(size)
 227
 228     def mount(self):
 229         self.diskmount.mount()
 230
 231     def __fsck(self):
 232         self.extdiskmount.__fsck()
 233
 234     def __get_size_from_filesystem(self):
 235         return self.diskmount.__get_size_from_filesystem()
 236
 237     def __resize_to_minimal(self):
 238         return self.diskmount.__resize_to_minimal()
 239
 240     def resparse(self, size = None):
 241         return self.diskmount.resparse(size)
 242
 243 class Disk:
 244     """Generic base object for a disk
 245
 246     The 'create' method must make the disk visible as a block device - eg
 247     by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
 248     method must undo the 'create' operation.
 249     """
 250     def __init__(self, size, device = None):
 251         self._device = device
 252         self._size = size
 253
 254     def create(self):
 255         pass
 256
 257     def cleanup(self):
 258         pass
 259
 260     def get_device(self):
 261         return self._device
 262     def set_device(self, path):
 263         self._device = path
 264     device = property(get_device, set_device)
 265
 266     def get_size(self):
 267         return self._size
 268     size = property(get_size)
 269
 270
 271 class RawDisk(Disk):
 272     """A Disk backed by a block device.
 273     Note that create() is a no-op.
 274     """
 275     def __init__(self, size, device):
 276         Disk.__init__(self, size, device)
 277
 278     def fixed(self):
 279         return True
 280
 281     def exists(self):
 282         return True
 283
 284 class LoopbackDisk(Disk):
 285     """A Disk backed by a file via the loop module."""
 286     def __init__(self, lofile, size):
 287         Disk.__init__(self, size)
 288         self.lofile = lofile
 289
 290     def fixed(self):
 291         return False
 292
 293     def exists(self):
 294         return os.path.exists(self.lofile)
 295
 296     def create(self):
 297         if self.device is not None:
 298             return
 299
 300         losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
 301                                        stdout=subprocess.PIPE)
 302         losetupOutput = losetupProc.communicate()[0]
 303
 304         if losetupProc.returncode:
 305             raise MountError("Failed to allocate loop device for '%s'" %
 306                              self.lofile)
 307
 308         device = losetupOutput.split()[0]
 309
 310         logging.info("Losetup add %s mapping to %s"  % (device, self.lofile))
 311         rc = call(["/sbin/losetup", device, self.lofile])
 312         if rc != 0:
 313             raise MountError("Failed to allocate loop device for '%s'" %
 314                              self.lofile)
 315         self.device = device
 316
 317     def cleanup(self):
 318         if self.device is None:
 319             return
 320         logging.info("Losetup remove %s" % self.device)
 321         rc = call(["/sbin/losetup", "-d", self.device])
 322         self.device = None
 323
 324
 325
 326 class SparseLoopbackDisk(LoopbackDisk):
 327     """A Disk backed by a sparse file via the loop module."""
 328     def __init__(self, lofile, size):
 329         LoopbackDisk.__init__(self, lofile, size)
 330
 331     def expand(self, create = False, size = None):
 332         flags = os.O_WRONLY
 333         if create:
 334             flags |= os.O_CREAT
 335             makedirs(os.path.dirname(self.lofile))
 336
 337         if size is None:
 338             size = self.size
 339
 340         logging.info("Extending sparse file %s to %d" % (self.lofile, size))
 341         fd = os.open(self.lofile, flags)
 342
 343         if size <= 0:
 344             size = 1
 345         os.lseek(fd, size-1, 0)
 346         os.write(fd, '\x00')
 347         os.close(fd)
 348
 349     def truncate(self, size = None):
 350         if size is None:
 351             size = self.size
 352
 353         logging.info("Truncating sparse file %s to %d" % (self.lofile, size))
 354         fd = os.open(self.lofile, os.O_WRONLY)
 355         os.ftruncate(fd, size)
 356         os.close(fd)
 357
 358     def create(self):
 359         self.expand(create = True)
 360         LoopbackDisk.create(self)
 361
 362 class Mount:
 363     """A generic base class to deal with mounting things."""
 364     def __init__(self, mountdir):
 365         self.mountdir = mountdir
 366
 367     def cleanup(self):
 368         self.unmount()
 369
 370     def mount(self):
 371         pass
 372
 373     def unmount(self):
 374         pass
 375
 376 class DiskMount(Mount):
 377     """A Mount object that handles mounting of a Disk."""
 378     def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
 379         Mount.__init__(self, mountdir)
 380
 381         self.disk = disk
 382         self.fstype = fstype
 383         self.rmmountdir = rmmountdir
 384
 385         self.mounted = False
 386         self.rmdir   = False
 387
 388     def cleanup(self):
 389         Mount.cleanup(self)
 390         self.disk.cleanup()
 391
 392     def unmount(self):
 393         if self.mounted:
 394             logging.info("Unmounting directory %s" % self.mountdir)
 395             rc = call(["/bin/umount", self.mountdir])
 396             if rc == 0:
 397                 self.mounted = False
 398             else:
 399                 logging.warn("Unmounting directory %s failed, using lazy umount" % self.mountdir)
 400                 print >> sys.stdout, "Unmounting directory %s failed, using lazy umount" %self.mountdir
 401                 rc = call(["/bin/umount", "-l", self.mountdir])
 402                 if rc != 0:
 403                     raise MountError("Unable to unmount filesystem at %s" % self.mountdir)
 404                 else:
 405                     logging.info("lazy umount succeeded on %s" % self.mountdir)
 406                     print >> sys.stdout, "lazy umount succeeded on %s" % self.mountdir
 407                     self.mounted = False
 408
 409         if self.rmdir and not self.mounted:
 410             try:
 411                 os.rmdir(self.mountdir)
 412             except OSError, e:
 413                 pass
 414             self.rmdir = False
 415
 416
 417     def __create(self):
 418         self.disk.create()
 419
 420
 421     def mount(self):
 422         if self.mounted:
 423             return
 424
 425         if not os.path.isdir(self.mountdir):
 426             logging.info("Creating mount point %s" % self.mountdir)
 427             os.makedirs(self.mountdir)
 428             self.rmdir = self.rmmountdir
 429
 430         self.__create()
 431
 432         logging.info("Mounting %s at %s" % (self.disk.device, self.mountdir))
 433         args = [ "/bin/mount", self.disk.device, self.mountdir ]
 434         if self.fstype:
 435             args.extend(["-t", self.fstype])
 436
 437         rc = call(args)
 438         if rc != 0:
 439             raise MountError("Failed to mount '%s' to '%s'" %
 440                              (self.disk.device, self.mountdir))
 441
 442         self.mounted = True
 443
 444 class ExtDiskMount(DiskMount):
 445     """A DiskMount object that is able to format/resize ext[23] filesystems."""
 446     def __init__(self, disk, mountdir, fstype, blocksize, fslabel,
 447                  rmmountdir=True, tmpdir="/tmp"):
 448         DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
 449         self.blocksize = blocksize
 450         self.fslabel = "_" + fslabel
 451         self.tmpdir = tmpdir
 452
 453     def __format_filesystem(self):
 454         logging.info("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
 455         rc = call(["/sbin/mkfs." + self.fstype,
 456                    "-F", "-L", self.fslabel,
 457                    "-m", "1", "-b", str(self.blocksize),
 458                    self.disk.device])
 459         #          str(self.disk.size / self.blocksize)])
 460
 461         if rc != 0:
 462             raise MountError("Error creating %s filesystem" % (self.fstype,))
 463         logging.info("Tuning filesystem on %s" % self.disk.device)
 464         call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
 465               "-ouser_xattr,acl", self.disk.device])
 466
 467     def __resize_filesystem(self, size = None):
 468         current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
 469
 470         if size is None:
 471             size = self.disk.size
 472
 473         if size == current_size:
 474             return
 475
 476         if size > current_size:
 477             self.disk.expand(size)
 478
 479         resize2fs(self.disk.lofile, size, tmpdir = self.tmpdir)
 480         return size
 481
 482     def __create(self):
 483         resize = False
 484         if not self.disk.fixed() and self.disk.exists():
 485             resize = True
 486
 487         self.disk.create()
 488
 489         if resize:
 490             self.__resize_filesystem()
 491         else:
 492             self.__format_filesystem()
 493
 494     def mount(self):
 495         self.__create()
 496         DiskMount.mount(self)
 497
 498     def __fsck(self):
 499         return e2fsck(self.disk.lofile)
 500         return rc
 501
 502     def __get_size_from_filesystem(self):
 503         def parse_field(output, field):
 504             for line in output.split("\n"):
 505                 if line.startswith(field + ":"):
 506                     return line[len(field) + 1:].strip()
 507
 508             raise KeyError("Failed to find field '%s' in output" % field)
 509
 510         dev_null = os.open("/dev/null", os.O_WRONLY)
 511         try:
 512             out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
 513                                    stdout = subprocess.PIPE,
 514                                    stderr = dev_null).communicate()[0]
 515         finally:
 516             os.close(dev_null)
 517
 518         return int(parse_field(out, "Block count")) * self.blocksize
 519
 520     def __resize_to_minimal(self):
 521         resize2fs(self.disk.lofile, minimal = True, tmpdir = self.tmpdir)
 522         return self.__get_size_from_filesystem()
 523
 524     def resparse(self, size = None):
 525         self.cleanup()
 526         minsize = self.__resize_to_minimal()
 527         self.disk.truncate(minsize)
 528         self.__resize_filesystem(size)
 529         return minsize
 530
 531 class DeviceMapperSnapshot(object):
 532     def __init__(self, imgloop, cowloop):
 533         self.imgloop = imgloop
 534         self.cowloop = cowloop
 535
 536         self.__created = False
 537         self.__name = None
 538
 539     def get_path(self):
 540         if self.__name is None:
 541             return None
 542         return os.path.join("/dev/mapper", self.__name)
 543     path = property(get_path)
 544
 545     def create(self):
 546         if self.__created:
 547             return
 548
 549         self.imgloop.create()
 550         self.cowloop.create()
 551
 552         self.__name = "imgcreate-%d-%d" % (os.getpid(),
 553                                            random.randint(0, 2**16))
 554
 555         size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
 556
 557         table = "0 %d snapshot %s %s p 8" % (size / 512,
 558                                              self.imgloop.device,
 559                                              self.cowloop.device)
 560
 561         args = ["/sbin/dmsetup", "create", self.__name,
 562                 "--uuid", "LIVECD-%s" % self.__name, "--table", table]
 563         if call(args) != 0:
 564             self.cowloop.cleanup()
 565             self.imgloop.cleanup()
 566             raise SnapshotError("Could not create snapshot device using: " +
 567                                 string.join(args, " "))
 568
 569         self.__created = True
 570
 571     def remove(self, ignore_errors = False):
 572         if not self.__created:
 573             return
 574
 575         # sleep to try to avoid any dm shenanigans
 576         time.sleep(2)
 577         rc = call(["/sbin/dmsetup", "remove", self.__name])
 578         if not ignore_errors and rc != 0:
 579             raise SnapshotError("Could not remove snapshot device")
 580
 581         self.__name = None
 582         self.__created = False
 583
 584         self.cowloop.cleanup()
 585         self.imgloop.cleanup()
 586
 587     def get_cow_used(self):
 588         if not self.__created:
 589             return 0
 590
 591         dev_null = os.open("/dev/null", os.O_WRONLY)
 592         try:
 593             out = subprocess.Popen(["/sbin/dmsetup", "status", self.__name],
 594                                    stdout = subprocess.PIPE,
 595                                    stderr = dev_null).communicate()[0]
 596         finally:
 597             os.close(dev_null)
 598
 599         #
 600         # dmsetup status on a snapshot returns e.g.
 601         #   "0 8388608 snapshot 416/1048576"
 602         # or, more generally:
 603         #   "A B snapshot C/D"
 604         # where C is the number of 512 byte sectors in use
 605         #
 606         try:
 607             return int((out.split()[3]).split('/')[0]) * 512
 608         except ValueError:
 609             raise SnapshotError("Failed to parse dmsetup status: " + out)
 610
 611 def create_image_minimizer(path, image, compress_type, target_size = None,
 612                            tmpdir = "/tmp"):
 613     """
 614     Builds a copy-on-write image which can be used to
 615     create a device-mapper snapshot of an image where
 616     the image's filesystem is as small as possible
 617
 618     The steps taken are:
 619       1) Create a sparse COW
 620       2) Loopback mount the image and the COW
 621       3) Create a device-mapper snapshot of the image
 622          using the COW
 623       4) Resize the filesystem to the minimal size
 624       5) Determine the amount of space used in the COW
 625       6) Restroy the device-mapper snapshot
 626       7) Truncate the COW, removing unused space
 627       8) Create a squashfs of the COW
 628     """
 629     imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
 630
 631     cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
 632                                  64L * 1024L * 1024L)
 633
 634     snapshot = DeviceMapperSnapshot(imgloop, cowloop)
 635
 636     try:
 637         snapshot.create()
 638
 639         if target_size is not None:
 640             resize2fs(snapshot.path, target_size, tmpdir = tmpdir)
 641         else:
 642             resize2fs(snapshot.path, minimal = True, tmpdir = tmpdir)
 643
 644         cow_used = snapshot.get_cow_used()
 645     finally:
 646         snapshot.remove(ignore_errors = (not sys.exc_info()[0] is None))
 647
 648     cowloop.truncate(cow_used)
 649
 650     mksquashfs(cowloop.lofile, path, compress_type)
 651
 652     os.unlink(cowloop.lofile)
 653