tests/docker/docker.py

   1 #!/usr/bin/env python2
   2 #
   3 # Docker controlling module
   4 #
   5 # Copyright (c) 2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Fam Zheng <famz@redhat.com>
   9 #
  10 # This work is licensed under the terms of the GNU GPL, version 2
  11 # or (at your option) any later version. See the COPYING file in
  12 # the top-level directory.
  13
  14 import os
  15 import sys
  16 sys.path.append(os.path.join(os.path.dirname(__file__),
  17                              '..', '..', 'scripts'))
  18 import argparse
  19 import subprocess
  20 import json
  21 import hashlib
  22 import atexit
  23 import uuid
  24 import tempfile
  25 import re
  26 import signal
  27 from tarfile import TarFile, TarInfo
  28 from StringIO import StringIO
  29 from shutil import copy, rmtree
  30 from pwd import getpwuid
  31
  32
  33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
  34
  35
  36 DEVNULL = open(os.devnull, 'wb')
  37
  38
  39 def _text_checksum(text):
  40     """Calculate a digest string unique to the text content"""
  41     return hashlib.sha1(text).hexdigest()
  42
  43 def _file_checksum(filename):
  44     return _text_checksum(open(filename, 'rb').read())
  45
  46 def _guess_docker_command():
  47     """ Guess a working docker command or raise exception if not found"""
  48     commands = [["docker"], ["sudo", "-n", "docker"]]
  49     for cmd in commands:
  50         try:
  51             if subprocess.call(cmd + ["images"],
  52                                stdout=DEVNULL, stderr=DEVNULL) == 0:
  53                 return cmd
  54         except OSError:
  55             pass
  56     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
  57     raise Exception("Cannot find working docker command. Tried:\n%s" % \
  58                     commands_txt)
  59
  60 def _copy_with_mkdir(src, root_dir, sub_path='.'):
  61     """Copy src into root_dir, creating sub_path as needed."""
  62     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
  63     try:
  64         os.makedirs(dest_dir)
  65     except OSError:
  66         # we can safely ignore already created directories
  67         pass
  68
  69     dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
  70     copy(src, dest_file)
  71
  72
  73 def _get_so_libs(executable):
  74     """Return a list of libraries associated with an executable.
  75
  76     The paths may be symbolic links which would need to be resolved to
  77     ensure theright data is copied."""
  78
  79     libs = []
  80     ldd_re = re.compile(r"(/.*/)(\S*)")
  81     try:
  82         ldd_output = subprocess.check_output(["ldd", executable])
  83         for line in ldd_output.split("\n"):
  84             search = ldd_re.search(line)
  85             if search and len(search.groups()) == 2:
  86                 so_path = search.groups()[0]
  87                 so_lib = search.groups()[1]
  88                 libs.append("%s/%s" % (so_path, so_lib))
  89     except subprocess.CalledProcessError:
  90         print "%s had no associated libraries (static build?)" % (executable)
  91
  92     return libs
  93
  94 def _copy_binary_with_libs(src, dest_dir):
  95     """Copy a binary executable and all its dependant libraries.
  96
  97     This does rely on the host file-system being fairly multi-arch
  98     aware so the file don't clash with the guests layout."""
  99
 100     _copy_with_mkdir(src, dest_dir, "/usr/bin")
 101
 102     libs = _get_so_libs(src)
 103     if libs:
 104         for l in libs:
 105             so_path = os.path.dirname(l)
 106             _copy_with_mkdir(l , dest_dir, so_path)
 107
 108 class Docker(object):
 109     """ Running Docker commands """
 110     def __init__(self):
 111         self._command = _guess_docker_command()
 112         self._instances = []
 113         atexit.register(self._kill_instances)
 114         signal.signal(signal.SIGTERM, self._kill_instances)
 115         signal.signal(signal.SIGHUP, self._kill_instances)
 116
 117     def _do(self, cmd, quiet=True, **kwargs):
 118         if quiet:
 119             kwargs["stdout"] = DEVNULL
 120         return subprocess.call(self._command + cmd, **kwargs)
 121
 122     def _do_check(self, cmd, quiet=True, **kwargs):
 123         if quiet:
 124             kwargs["stdout"] = DEVNULL
 125         return subprocess.check_call(self._command + cmd, **kwargs)
 126
 127     def _do_kill_instances(self, only_known, only_active=True):
 128         cmd = ["ps", "-q"]
 129         if not only_active:
 130             cmd.append("-a")
 131         for i in self._output(cmd).split():
 132             resp = self._output(["inspect", i])
 133             labels = json.loads(resp)[0]["Config"]["Labels"]
 134             active = json.loads(resp)[0]["State"]["Running"]
 135             if not labels:
 136                 continue
 137             instance_uuid = labels.get("com.qemu.instance.uuid", None)
 138             if not instance_uuid:
 139                 continue
 140             if only_known and instance_uuid not in self._instances:
 141                 continue
 142             print "Terminating", i
 143             if active:
 144                 self._do(["kill", i])
 145             self._do(["rm", i])
 146
 147     def clean(self):
 148         self._do_kill_instances(False, False)
 149         return 0
 150
 151     def _kill_instances(self, *args, **kwargs):
 152         return self._do_kill_instances(True)
 153
 154     def _output(self, cmd, **kwargs):
 155         return subprocess.check_output(self._command + cmd,
 156                                        stderr=subprocess.STDOUT,
 157                                        **kwargs)
 158
 159     def get_image_dockerfile_checksum(self, tag):
 160         resp = self._output(["inspect", tag])
 161         labels = json.loads(resp)[0]["Config"].get("Labels", {})
 162         return labels.get("com.qemu.dockerfile-checksum", "")
 163
 164     def build_image(self, tag, docker_dir, dockerfile,
 165                     quiet=True, user=False, argv=None, extra_files_cksum=[]):
 166         if argv == None:
 167             argv = []
 168
 169         tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
 170         tmp_df.write(dockerfile)
 171
 172         if user:
 173             uid = os.getuid()
 174             uname = getpwuid(uid).pw_name
 175             tmp_df.write("\n")
 176             tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
 177                          (uname, uid, uname))
 178
 179         tmp_df.write("\n")
 180         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
 181                      _text_checksum("\n".join([dockerfile] +
 182                                     extra_files_cksum)))
 183         tmp_df.flush()
 184
 185         self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
 186                        [docker_dir],
 187                        quiet=quiet)
 188
 189     def update_image(self, tag, tarball, quiet=True):
 190         "Update a tagged image using "
 191
 192         self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
 193
 194     def image_matches_dockerfile(self, tag, dockerfile):
 195         try:
 196             checksum = self.get_image_dockerfile_checksum(tag)
 197         except Exception:
 198             return False
 199         return checksum == _text_checksum(dockerfile)
 200
 201     def run(self, cmd, keep, quiet):
 202         label = uuid.uuid1().hex
 203         if not keep:
 204             self._instances.append(label)
 205         ret = self._do_check(["run", "--label",
 206                              "com.qemu.instance.uuid=" + label] + cmd,
 207                              quiet=quiet)
 208         if not keep:
 209             self._instances.remove(label)
 210         return ret
 211
 212     def command(self, cmd, argv, quiet):
 213         return self._do([cmd] + argv, quiet=quiet)
 214
 215 class SubCommand(object):
 216     """A SubCommand template base class"""
 217     name = None # Subcommand name
 218     def shared_args(self, parser):
 219         parser.add_argument("--quiet", action="store_true",
 220                             help="Run quietly unless an error occured")
 221
 222     def args(self, parser):
 223         """Setup argument parser"""
 224         pass
 225     def run(self, args, argv):
 226         """Run command.
 227         args: parsed argument by argument parser.
 228         argv: remaining arguments from sys.argv.
 229         """
 230         pass
 231
 232 class RunCommand(SubCommand):
 233     """Invoke docker run and take care of cleaning up"""
 234     name = "run"
 235     def args(self, parser):
 236         parser.add_argument("--keep", action="store_true",
 237                             help="Don't remove image when command completes")
 238     def run(self, args, argv):
 239         return Docker().run(argv, args.keep, quiet=args.quiet)
 240
 241 class BuildCommand(SubCommand):
 242     """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
 243     name = "build"
 244     def args(self, parser):
 245         parser.add_argument("--include-executable", "-e",
 246                             help="""Specify a binary that will be copied to the
 247                             container together with all its dependent
 248                             libraries""")
 249         parser.add_argument("--extra-files", "-f", nargs='*',
 250                             help="""Specify files that will be copied in the
 251                             Docker image, fulfilling the ADD directive from the
 252                             Dockerfile""")
 253         parser.add_argument("--add-current-user", "-u", dest="user",
 254                             action="store_true",
 255                             help="Add the current user to image's passwd")
 256         parser.add_argument("tag",
 257                             help="Image Tag")
 258         parser.add_argument("dockerfile",
 259                             help="Dockerfile name")
 260
 261     def run(self, args, argv):
 262         dockerfile = open(args.dockerfile, "rb").read()
 263         tag = args.tag
 264
 265         dkr = Docker()
 266         if dkr.image_matches_dockerfile(tag, dockerfile):
 267             if not args.quiet:
 268                 print "Image is up to date."
 269         else:
 270             # Create a docker context directory for the build
 271             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 272
 273             # Is there a .pre file to run in the build context?
 274             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
 275             if os.path.exists(docker_pre):
 276                 stdout = DEVNULL if args.quiet else None
 277                 rc = subprocess.call(os.path.realpath(docker_pre),
 278                                      cwd=docker_dir, stdout=stdout)
 279                 if rc == 3:
 280                     print "Skip"
 281                     return 0
 282                 elif rc != 0:
 283                     print "%s exited with code %d" % (docker_pre, rc)
 284                     return 1
 285
 286             # Copy any extra files into the Docker context. These can be
 287             # included by the use of the ADD directive in the Dockerfile.
 288             cksum = []
 289             if args.include_executable:
 290                 # FIXME: there is no checksum of this executable and the linked
 291                 # libraries, once the image built any change of this executable
 292                 # or any library won't trigger another build.
 293                 _copy_binary_with_libs(args.include_executable, docker_dir)
 294             for filename in args.extra_files or []:
 295                 _copy_with_mkdir(filename, docker_dir)
 296                 cksum += [_file_checksum(filename)]
 297
 298             argv += ["--build-arg=" + k.lower() + "=" + v
 299                         for k, v in os.environ.iteritems()
 300                         if k.lower() in FILTERED_ENV_NAMES]
 301             dkr.build_image(tag, docker_dir, dockerfile,
 302                             quiet=args.quiet, user=args.user, argv=argv,
 303                             extra_files_cksum=cksum)
 304
 305             rmtree(docker_dir)
 306
 307         return 0
 308
 309 class UpdateCommand(SubCommand):
 310     """ Update a docker image with new executables. Arguments: <tag> <executable>"""
 311     name = "update"
 312     def args(self, parser):
 313         parser.add_argument("tag",
 314                             help="Image Tag")
 315         parser.add_argument("executable",
 316                             help="Executable to copy")
 317
 318     def run(self, args, argv):
 319         # Create a temporary tarball with our whole build context and
 320         # dockerfile for the update
 321         tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
 322         tmp_tar = TarFile(fileobj=tmp, mode='w')
 323
 324         # Add the executable to the tarball
 325         bn = os.path.basename(args.executable)
 326         ff = "/usr/bin/%s" % bn
 327         tmp_tar.add(args.executable, arcname=ff)
 328
 329         # Add any associated libraries
 330         libs = _get_so_libs(args.executable)
 331         if libs:
 332             for l in libs:
 333                 tmp_tar.add(os.path.realpath(l), arcname=l)
 334
 335         # Create a Docker buildfile
 336         df = StringIO()
 337         df.write("FROM %s\n" % args.tag)
 338         df.write("ADD . /\n")
 339         df.seek(0)
 340
 341         df_tar = TarInfo(name="Dockerfile")
 342         df_tar.size = len(df.buf)
 343         tmp_tar.addfile(df_tar, fileobj=df)
 344
 345         tmp_tar.close()
 346
 347         # reset the file pointers
 348         tmp.flush()
 349         tmp.seek(0)
 350
 351         # Run the build with our tarball context
 352         dkr = Docker()
 353         dkr.update_image(args.tag, tmp, quiet=args.quiet)
 354
 355         return 0
 356
 357 class CleanCommand(SubCommand):
 358     """Clean up docker instances"""
 359     name = "clean"
 360     def run(self, args, argv):
 361         Docker().clean()
 362         return 0
 363
 364 class ImagesCommand(SubCommand):
 365     """Run "docker images" command"""
 366     name = "images"
 367     def run(self, args, argv):
 368         return Docker().command("images", argv, args.quiet)
 369
 370 def main():
 371     parser = argparse.ArgumentParser(description="A Docker helper",
 372             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
 373     subparsers = parser.add_subparsers(title="subcommands", help=None)
 374     for cls in SubCommand.__subclasses__():
 375         cmd = cls()
 376         subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
 377         cmd.shared_args(subp)
 378         cmd.args(subp)
 379         subp.set_defaults(cmdobj=cmd)
 380     args, argv = parser.parse_known_args()
 381     return args.cmdobj.run(args, argv)
 382
 383 if __name__ == "__main__":
 384     sys.exit(main())