sd/milkymist-memcard: Fix format string
[qemu/armbru.git] / tests / docker / docker.py
blobc9f20d8d093bed4bd96edc8e1f1d83ab21a77d6d
1 #!/usr/bin/env python3
3 # Docker controlling module
5 # Copyright (c) 2016 Red Hat Inc.
7 # Authors:
8 # Fam Zheng <famz@redhat.com>
10 # This work is licensed under the terms of the GNU GPL, version 2
11 # or (at your option) any later version. See the COPYING file in
12 # the top-level directory.
14 import os
15 import sys
16 import subprocess
17 import json
18 import hashlib
19 import atexit
20 import uuid
21 import argparse
22 import enum
23 import tempfile
24 import re
25 import signal
26 from tarfile import TarFile, TarInfo
27 from io import StringIO
28 from shutil import copy, rmtree
29 from pwd import getpwuid
30 from datetime import datetime, timedelta
33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
36 DEVNULL = open(os.devnull, 'wb')
38 class EngineEnum(enum.IntEnum):
39 AUTO = 1
40 DOCKER = 2
41 PODMAN = 3
43 def __str__(self):
44 return self.name.lower()
46 def __repr__(self):
47 return str(self)
49 @staticmethod
50 def argparse(s):
51 try:
52 return EngineEnum[s.upper()]
53 except KeyError:
54 return s
57 USE_ENGINE = EngineEnum.AUTO
59 def _bytes_checksum(bytes):
60 """Calculate a digest string unique to the text content"""
61 return hashlib.sha1(bytes).hexdigest()
63 def _text_checksum(text):
64 """Calculate a digest string unique to the text content"""
65 return _bytes_checksum(text.encode('utf-8'))
67 def _read_dockerfile(path):
68 return open(path, 'rt', encoding='utf-8').read()
70 def _file_checksum(filename):
71 return _bytes_checksum(open(filename, 'rb').read())
74 def _guess_engine_command():
75 """ Guess a working engine command or raise exception if not found"""
76 commands = []
78 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
79 commands += [["podman"]]
80 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
81 commands += [["docker"], ["sudo", "-n", "docker"]]
82 for cmd in commands:
83 try:
84 # docker version will return the client details in stdout
85 # but still report a status of 1 if it can't contact the daemon
86 if subprocess.call(cmd + ["version"],
87 stdout=DEVNULL, stderr=DEVNULL) == 0:
88 return cmd
89 except OSError:
90 pass
91 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
92 raise Exception("Cannot find working engine command. Tried:\n%s" %
93 commands_txt)
96 def _copy_with_mkdir(src, root_dir, sub_path='.'):
97 """Copy src into root_dir, creating sub_path as needed."""
98 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
99 try:
100 os.makedirs(dest_dir)
101 except OSError:
102 # we can safely ignore already created directories
103 pass
105 dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
106 copy(src, dest_file)
109 def _get_so_libs(executable):
110 """Return a list of libraries associated with an executable.
112 The paths may be symbolic links which would need to be resolved to
113 ensure the right data is copied."""
115 libs = []
116 ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
117 try:
118 ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
119 for line in ldd_output.split("\n"):
120 search = ldd_re.search(line)
121 if search:
122 try:
123 libs.append(s.group(1))
124 except IndexError:
125 pass
126 except subprocess.CalledProcessError:
127 print("%s had no associated libraries (static build?)" % (executable))
129 return libs
132 def _copy_binary_with_libs(src, bin_dest, dest_dir):
133 """Maybe copy a binary and all its dependent libraries.
135 If bin_dest isn't set we only copy the support libraries because
136 we don't need qemu in the docker path to run (due to persistent
137 mapping). Indeed users may get confused if we aren't running what
138 is in the image.
140 This does rely on the host file-system being fairly multi-arch
141 aware so the file don't clash with the guests layout.
144 if bin_dest:
145 _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
146 else:
147 print("only copying support libraries for %s" % (src))
149 libs = _get_so_libs(src)
150 if libs:
151 for l in libs:
152 so_path = os.path.dirname(l)
153 real_l = os.path.realpath(l)
154 _copy_with_mkdir(real_l, dest_dir, so_path)
157 def _check_binfmt_misc(executable):
158 """Check binfmt_misc has entry for executable in the right place.
160 The details of setting up binfmt_misc are outside the scope of
161 this script but we should at least fail early with a useful
162 message if it won't work.
164 Returns the configured binfmt path and a valid flag. For
165 persistent configurations we will still want to copy and dependent
166 libraries.
169 binary = os.path.basename(executable)
170 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
172 if not os.path.exists(binfmt_entry):
173 print ("No binfmt_misc entry for %s" % (binary))
174 return None, False
176 with open(binfmt_entry) as x: entry = x.read()
178 if re.search("flags:.*F.*\n", entry):
179 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
180 (binary))
181 return None, True
183 m = re.search("interpreter (\S+)\n", entry)
184 interp = m.group(1)
185 if interp and interp != executable:
186 print("binfmt_misc for %s does not point to %s, using %s" %
187 (binary, executable, interp))
189 return interp, True
192 def _read_qemu_dockerfile(img_name):
193 # special case for Debian linux-user images
194 if img_name.startswith("debian") and img_name.endswith("user"):
195 img_name = "debian-bootstrap"
197 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
198 img_name + ".docker")
199 return _read_dockerfile(df)
202 def _dockerfile_preprocess(df):
203 out = ""
204 for l in df.splitlines():
205 if len(l.strip()) == 0 or l.startswith("#"):
206 continue
207 from_pref = "FROM qemu/"
208 if l.startswith(from_pref):
209 # TODO: Alternatively we could replace this line with "FROM $ID"
210 # where $ID is the image's hex id obtained with
211 # $ docker images $IMAGE --format="{{.Id}}"
212 # but unfortunately that's not supported by RHEL 7.
213 inlining = _read_qemu_dockerfile(l[len(from_pref):])
214 out += _dockerfile_preprocess(inlining)
215 continue
216 out += l + "\n"
217 return out
220 class Docker(object):
221 """ Running Docker commands """
222 def __init__(self):
223 self._command = _guess_engine_command()
225 if "docker" in self._command and "TRAVIS" not in os.environ:
226 os.environ["DOCKER_BUILDKIT"] = "1"
227 self._buildkit = True
228 else:
229 self._buildkit = False
231 self._instance = None
232 atexit.register(self._kill_instances)
233 signal.signal(signal.SIGTERM, self._kill_instances)
234 signal.signal(signal.SIGHUP, self._kill_instances)
236 def _do(self, cmd, quiet=True, **kwargs):
237 if quiet:
238 kwargs["stdout"] = DEVNULL
239 return subprocess.call(self._command + cmd, **kwargs)
241 def _do_check(self, cmd, quiet=True, **kwargs):
242 if quiet:
243 kwargs["stdout"] = DEVNULL
244 return subprocess.check_call(self._command + cmd, **kwargs)
246 def _do_kill_instances(self, only_known, only_active=True):
247 cmd = ["ps", "-q"]
248 if not only_active:
249 cmd.append("-a")
251 filter = "--filter=label=com.qemu.instance.uuid"
252 if only_known:
253 if self._instance:
254 filter += "=%s" % (self._instance)
255 else:
256 # no point trying to kill, we finished
257 return
259 print("filter=%s" % (filter))
260 cmd.append(filter)
261 for i in self._output(cmd).split():
262 self._do(["rm", "-f", i])
264 def clean(self):
265 self._do_kill_instances(False, False)
266 return 0
268 def _kill_instances(self, *args, **kwargs):
269 return self._do_kill_instances(True)
271 def _output(self, cmd, **kwargs):
272 try:
273 return subprocess.check_output(self._command + cmd,
274 stderr=subprocess.STDOUT,
275 encoding='utf-8',
276 **kwargs)
277 except TypeError:
278 # 'encoding' argument was added in 3.6+
279 return subprocess.check_output(self._command + cmd,
280 stderr=subprocess.STDOUT,
281 **kwargs).decode('utf-8')
284 def inspect_tag(self, tag):
285 try:
286 return self._output(["inspect", tag])
287 except subprocess.CalledProcessError:
288 return None
290 def get_image_creation_time(self, info):
291 return json.loads(info)[0]["Created"]
293 def get_image_dockerfile_checksum(self, tag):
294 resp = self.inspect_tag(tag)
295 labels = json.loads(resp)[0]["Config"].get("Labels", {})
296 return labels.get("com.qemu.dockerfile-checksum", "")
298 def build_image(self, tag, docker_dir, dockerfile,
299 quiet=True, user=False, argv=None, registry=None,
300 extra_files_cksum=[]):
301 if argv is None:
302 argv = []
304 # pre-calculate the docker checksum before any
305 # substitutions we make for caching
306 checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
308 if registry is not None:
309 sources = re.findall("FROM qemu\/(.*)", dockerfile)
310 # Fetch any cache layers we can, may fail
311 for s in sources:
312 pull_args = ["pull", "%s/qemu/%s" % (registry, s)]
313 if self._do(pull_args, quiet=quiet) != 0:
314 registry = None
315 break
316 # Make substitutions
317 if registry is not None:
318 dockerfile = dockerfile.replace("FROM qemu/",
319 "FROM %s/qemu/" %
320 (registry))
322 tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
323 encoding='utf-8',
324 dir=docker_dir, suffix=".docker")
325 tmp_df.write(dockerfile)
327 if user:
328 uid = os.getuid()
329 uname = getpwuid(uid).pw_name
330 tmp_df.write("\n")
331 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
332 (uname, uid, uname))
334 tmp_df.write("\n")
335 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum))
336 for f, c in extra_files_cksum:
337 tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
339 tmp_df.flush()
341 build_args = ["build", "-t", tag, "-f", tmp_df.name]
342 if self._buildkit:
343 build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
345 if registry is not None:
346 pull_args = ["pull", "%s/%s" % (registry, tag)]
347 self._do(pull_args, quiet=quiet)
348 cache = "%s/%s" % (registry, tag)
349 build_args += ["--cache-from", cache]
350 build_args += argv
351 build_args += [docker_dir]
353 self._do_check(build_args,
354 quiet=quiet)
356 def update_image(self, tag, tarball, quiet=True):
357 "Update a tagged image using "
359 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
361 def image_matches_dockerfile(self, tag, dockerfile):
362 try:
363 checksum = self.get_image_dockerfile_checksum(tag)
364 except Exception:
365 return False
366 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
368 def run(self, cmd, keep, quiet, as_user=False):
369 label = uuid.uuid4().hex
370 if not keep:
371 self._instance = label
373 if as_user:
374 uid = os.getuid()
375 cmd = [ "-u", str(uid) ] + cmd
376 # podman requires a bit more fiddling
377 if self._command[0] == "podman":
378 cmd.insert(0, '--userns=keep-id')
380 ret = self._do_check(["run", "--label",
381 "com.qemu.instance.uuid=" + label] + cmd,
382 quiet=quiet)
383 if not keep:
384 self._instance = None
385 return ret
387 def command(self, cmd, argv, quiet):
388 return self._do([cmd] + argv, quiet=quiet)
391 class SubCommand(object):
392 """A SubCommand template base class"""
393 name = None # Subcommand name
395 def shared_args(self, parser):
396 parser.add_argument("--quiet", action="store_true",
397 help="Run quietly unless an error occurred")
399 def args(self, parser):
400 """Setup argument parser"""
401 pass
403 def run(self, args, argv):
404 """Run command.
405 args: parsed argument by argument parser.
406 argv: remaining arguments from sys.argv.
408 pass
411 class RunCommand(SubCommand):
412 """Invoke docker run and take care of cleaning up"""
413 name = "run"
415 def args(self, parser):
416 parser.add_argument("--keep", action="store_true",
417 help="Don't remove image when command completes")
418 parser.add_argument("--run-as-current-user", action="store_true",
419 help="Run container using the current user's uid")
421 def run(self, args, argv):
422 return Docker().run(argv, args.keep, quiet=args.quiet,
423 as_user=args.run_as_current_user)
426 class BuildCommand(SubCommand):
427 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
428 name = "build"
430 def args(self, parser):
431 parser.add_argument("--include-executable", "-e",
432 help="""Specify a binary that will be copied to the
433 container together with all its dependent
434 libraries""")
435 parser.add_argument("--extra-files", nargs='*',
436 help="""Specify files that will be copied in the
437 Docker image, fulfilling the ADD directive from the
438 Dockerfile""")
439 parser.add_argument("--add-current-user", "-u", dest="user",
440 action="store_true",
441 help="Add the current user to image's passwd")
442 parser.add_argument("--registry", "-r",
443 help="cache from docker registry")
444 parser.add_argument("-t", dest="tag",
445 help="Image Tag")
446 parser.add_argument("-f", dest="dockerfile",
447 help="Dockerfile name")
449 def run(self, args, argv):
450 dockerfile = _read_dockerfile(args.dockerfile)
451 tag = args.tag
453 dkr = Docker()
454 if "--no-cache" not in argv and \
455 dkr.image_matches_dockerfile(tag, dockerfile):
456 if not args.quiet:
457 print("Image is up to date.")
458 else:
459 # Create a docker context directory for the build
460 docker_dir = tempfile.mkdtemp(prefix="docker_build")
462 # Validate binfmt_misc will work
463 if args.include_executable:
464 qpath, enabled = _check_binfmt_misc(args.include_executable)
465 if not enabled:
466 return 1
468 # Is there a .pre file to run in the build context?
469 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
470 if os.path.exists(docker_pre):
471 stdout = DEVNULL if args.quiet else None
472 rc = subprocess.call(os.path.realpath(docker_pre),
473 cwd=docker_dir, stdout=stdout)
474 if rc == 3:
475 print("Skip")
476 return 0
477 elif rc != 0:
478 print("%s exited with code %d" % (docker_pre, rc))
479 return 1
481 # Copy any extra files into the Docker context. These can be
482 # included by the use of the ADD directive in the Dockerfile.
483 cksum = []
484 if args.include_executable:
485 # FIXME: there is no checksum of this executable and the linked
486 # libraries, once the image built any change of this executable
487 # or any library won't trigger another build.
488 _copy_binary_with_libs(args.include_executable,
489 qpath, docker_dir)
491 for filename in args.extra_files or []:
492 _copy_with_mkdir(filename, docker_dir)
493 cksum += [(filename, _file_checksum(filename))]
495 argv += ["--build-arg=" + k.lower() + "=" + v
496 for k, v in os.environ.items()
497 if k.lower() in FILTERED_ENV_NAMES]
498 dkr.build_image(tag, docker_dir, dockerfile,
499 quiet=args.quiet, user=args.user,
500 argv=argv, registry=args.registry,
501 extra_files_cksum=cksum)
503 rmtree(docker_dir)
505 return 0
508 class UpdateCommand(SubCommand):
509 """ Update a docker image with new executables. Args: <tag> <executable>"""
510 name = "update"
512 def args(self, parser):
513 parser.add_argument("tag",
514 help="Image Tag")
515 parser.add_argument("executable",
516 help="Executable to copy")
518 def run(self, args, argv):
519 # Create a temporary tarball with our whole build context and
520 # dockerfile for the update
521 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
522 tmp_tar = TarFile(fileobj=tmp, mode='w')
524 # Add the executable to the tarball, using the current
525 # configured binfmt_misc path. If we don't get a path then we
526 # only need the support libraries copied
527 ff, enabled = _check_binfmt_misc(args.executable)
529 if not enabled:
530 print("binfmt_misc not enabled, update disabled")
531 return 1
533 if ff:
534 tmp_tar.add(args.executable, arcname=ff)
536 # Add any associated libraries
537 libs = _get_so_libs(args.executable)
538 if libs:
539 for l in libs:
540 tmp_tar.add(os.path.realpath(l), arcname=l)
542 # Create a Docker buildfile
543 df = StringIO()
544 df.write("FROM %s\n" % args.tag)
545 df.write("ADD . /\n")
546 df.seek(0)
548 df_tar = TarInfo(name="Dockerfile")
549 df_tar.size = len(df.buf)
550 tmp_tar.addfile(df_tar, fileobj=df)
552 tmp_tar.close()
554 # reset the file pointers
555 tmp.flush()
556 tmp.seek(0)
558 # Run the build with our tarball context
559 dkr = Docker()
560 dkr.update_image(args.tag, tmp, quiet=args.quiet)
562 return 0
565 class CleanCommand(SubCommand):
566 """Clean up docker instances"""
567 name = "clean"
569 def run(self, args, argv):
570 Docker().clean()
571 return 0
574 class ImagesCommand(SubCommand):
575 """Run "docker images" command"""
576 name = "images"
578 def run(self, args, argv):
579 return Docker().command("images", argv, args.quiet)
582 class ProbeCommand(SubCommand):
583 """Probe if we can run docker automatically"""
584 name = "probe"
586 def run(self, args, argv):
587 try:
588 docker = Docker()
589 if docker._command[0] == "docker":
590 print("docker")
591 elif docker._command[0] == "sudo":
592 print("sudo docker")
593 elif docker._command[0] == "podman":
594 print("podman")
595 except Exception:
596 print("no")
598 return
601 class CcCommand(SubCommand):
602 """Compile sources with cc in images"""
603 name = "cc"
605 def args(self, parser):
606 parser.add_argument("--image", "-i", required=True,
607 help="The docker image in which to run cc")
608 parser.add_argument("--cc", default="cc",
609 help="The compiler executable to call")
610 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
611 help="""Extra paths to (ro) mount into container for
612 reading sources""")
614 def run(self, args, argv):
615 if argv and argv[0] == "--":
616 argv = argv[1:]
617 cwd = os.getcwd()
618 cmd = ["--rm", "-w", cwd,
619 "-v", "%s:%s:rw" % (cwd, cwd)]
620 if args.paths:
621 for p in args.paths:
622 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
623 cmd += [args.image, args.cc]
624 cmd += argv
625 return Docker().run(cmd, False, quiet=args.quiet,
626 as_user=True)
629 class CheckCommand(SubCommand):
630 """Check if we need to re-build a docker image out of a dockerfile.
631 Arguments: <tag> <dockerfile>"""
632 name = "check"
634 def args(self, parser):
635 parser.add_argument("tag",
636 help="Image Tag")
637 parser.add_argument("dockerfile", default=None,
638 help="Dockerfile name", nargs='?')
639 parser.add_argument("--checktype", choices=["checksum", "age"],
640 default="checksum", help="check type")
641 parser.add_argument("--olderthan", default=60, type=int,
642 help="number of minutes")
644 def run(self, args, argv):
645 tag = args.tag
647 try:
648 dkr = Docker()
649 except subprocess.CalledProcessError:
650 print("Docker not set up")
651 return 1
653 info = dkr.inspect_tag(tag)
654 if info is None:
655 print("Image does not exist")
656 return 1
658 if args.checktype == "checksum":
659 if not args.dockerfile:
660 print("Need a dockerfile for tag:%s" % (tag))
661 return 1
663 dockerfile = _read_dockerfile(args.dockerfile)
665 if dkr.image_matches_dockerfile(tag, dockerfile):
666 if not args.quiet:
667 print("Image is up to date")
668 return 0
669 else:
670 print("Image needs updating")
671 return 1
672 elif args.checktype == "age":
673 timestr = dkr.get_image_creation_time(info).split(".")[0]
674 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
675 past = datetime.now() - timedelta(minutes=args.olderthan)
676 if created < past:
677 print ("Image created @ %s more than %d minutes old" %
678 (timestr, args.olderthan))
679 return 1
680 else:
681 if not args.quiet:
682 print ("Image less than %d minutes old" % (args.olderthan))
683 return 0
686 def main():
687 global USE_ENGINE
689 parser = argparse.ArgumentParser(description="A Docker helper",
690 usage="%s <subcommand> ..." %
691 os.path.basename(sys.argv[0]))
692 parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
693 help="specify which container engine to use")
694 subparsers = parser.add_subparsers(title="subcommands", help=None)
695 for cls in SubCommand.__subclasses__():
696 cmd = cls()
697 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
698 cmd.shared_args(subp)
699 cmd.args(subp)
700 subp.set_defaults(cmdobj=cmd)
701 args, argv = parser.parse_known_args()
702 if args.engine:
703 USE_ENGINE = args.engine
704 return args.cmdobj.run(args, argv)
707 if __name__ == "__main__":
708 sys.exit(main())