ci/helper

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) 2021 Red Hat, Inc.
   4 # SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 import argparse
   7 import os
   8 import pathlib
   9 import subprocess
  10 import sys
  11 import textwrap
  12
  13 from pathlib import Path
  14 from tempfile import TemporaryDirectory
  15
  16 import util
  17
  18
  19 def required_deps(*deps):
  20     module2pkg = {
  21         "git": "GitPython"
  22     }
  23
  24     def inner_decorator(func):
  25         def wrapped(*args, **kwargs):
  26             cmd = func.__name__[len('_action_'):]
  27             for dep in deps:
  28                 try:
  29                     import importlib
  30                     importlib.import_module(dep)
  31                 except ImportError:
  32                     pkg = module2pkg[dep]
  33                     msg = f"'{pkg}' not found (required by the '{cmd}' command)"
  34                     print(msg, file=sys.stderr)
  35                     sys.exit(1)
  36             func(*args, **kwargs)
  37         return wrapped
  38     return inner_decorator
  39
  40
  41 class Parser:
  42     def __init__(self):
  43         # Options that are common to all actions that use containers
  44         containerparser = argparse.ArgumentParser(add_help=False)
  45         containerparser.add_argument(
  46             "target",
  47             help="perform action on target OS",
  48         )
  49         containerparser.add_argument(
  50             "--engine",
  51             choices=["auto", "podman", "docker"],
  52             default="auto",
  53             help="container engine to use",
  54         )
  55         containerparser.add_argument(
  56             "--login",
  57             default=os.getlogin(),  # exempt from syntax-check
  58             help="login to use inside the container",
  59         )
  60         containerparser.add_argument(
  61             "--image-prefix",
  62             default="registry.gitlab.com/libvirt/libvirt/ci-",
  63             help="use container images from non-default location",
  64         )
  65         containerparser.add_argument(
  66             "--image-tag",
  67             default="latest",
  68             help="use container images with non-default tags",
  69         )
  70         containerparser.add_argument(
  71             "--lcitool-path",
  72             dest="lcitool",
  73             default="lcitool",
  74             help="path to lcitool (default: $PATH)",
  75         )
  76
  77         # Options that are common to actions communicating with a GitLab
  78         # instance
  79         gitlabparser = argparse.ArgumentParser(add_help=False)
  80         gitlabparser.add_argument(
  81             "--namespace",
  82             default="libvirt/libvirt",
  83             help="GitLab project namespace"
  84         )
  85         gitlabparser.add_argument(
  86             "--gitlab-uri",
  87             default="https://gitlab.com",
  88             help="base GitLab URI"
  89         )
  90
  91         # Main parser
  92         self._parser = argparse.ArgumentParser()
  93         subparsers = self._parser.add_subparsers(
  94             dest="action",
  95             metavar="ACTION",
  96         )
  97         subparsers.required = True
  98
  99         jobparser = subparsers.add_parser(
 100             "run",
 101             help="Run a GitLab CI job or 'shell' in a local environment",
 102             parents=[containerparser],
 103             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 104         )
 105         jobparser.add_argument(
 106             "--job",
 107             choices=["build", "codestyle", "potfile", "rpmbuild",
 108                      "shell", "test", "website"],
 109             default="build",
 110             help="Run a GitLab CI job or 'shell' in a local environment",
 111         )
 112         jobparser.set_defaults(func=Application._action_run)
 113
 114         # list-images action
 115         listimagesparser = subparsers.add_parser(
 116             "list-images",
 117             help="list known container images",
 118             parents=[gitlabparser],
 119             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 120         )
 121         listimagesparser.set_defaults(func=Application._action_list_images)
 122
 123         # check_stale action
 124         check_staleparser = subparsers.add_parser(
 125             "check-stale",
 126             help="check for existence of stale images on the GitLab instance",
 127             parents=[gitlabparser],
 128             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 129         )
 130         check_staleparser.set_defaults(func=Application._action_check_stale)
 131
 132     def parse(self):
 133         return self._parser.parse_args()
 134
 135
 136 class Application:
 137     @property
 138     def repo(self):
 139         if self._repo is None:
 140             from git import Repo
 141
 142             self._repo = Repo(search_parent_directories=True)
 143         return self._repo
 144
 145     def __init__(self):
 146         self._basedir = pathlib.Path(__file__).resolve().parent
 147         self._args = Parser().parse()
 148         self._repo = None
 149
 150     @staticmethod
 151     def _prepare_repo_copy(repo, dest):
 152         return repo.clone(dest, local=True)
 153
 154     def _lcitool_run(self, args):
 155         positional_args = ["container"]
 156         opts = ["--user", self._args.login]
 157         tmpdir = TemporaryDirectory(prefix="scratch",
 158                                     dir=Path(self.repo.working_dir, "ci"))
 159
 160         repo_dest_path = Path(tmpdir.name, "libvirt.git").as_posix()
 161         repo_clone = self._prepare_repo_copy(self.repo, repo_dest_path)
 162         opts.extend(["--workload-dir", repo_clone.working_dir])
 163
 164         if self._args.job == "shell":
 165             positional_args.append("shell")
 166         else:
 167             job2func = {
 168                 "test": "run_test",
 169                 "build": "run_build",
 170                 "codestyle": "run_codestyle",
 171                 "potfile": "run_potfile",
 172                 "rpmbuild": "run_rpmbuild",
 173                 "website": "run_website_build",
 174             }
 175
 176             if self._args.engine != "auto":
 177                 positional_args.extend(["--engine", self._args.engine])
 178
 179             with open(Path(tmpdir.name, "script"), "w") as f:
 180                 script_path = f.name
 181                 contents = textwrap.dedent(f"""\
 182                 #!/bin/sh
 183
 184                 cd datadir
 185                 . ci/jobs.sh
 186
 187                 {job2func[self._args.job]}
 188                 """)
 189
 190                 f.write(contents)
 191
 192             positional_args.append("run")
 193             opts.extend(["--script", script_path])
 194
 195         opts.append(f"{self._args.image_prefix}{self._args.target}:{self._args.image_tag}")
 196         proc = None
 197         try:
 198             proc = subprocess.run([self._args.lcitool] + positional_args + opts)
 199         except KeyboardInterrupt:
 200             sys.exit(1)
 201         finally:
 202             # this will take care of the generated script file above as well
 203             tmpdir.cleanup()
 204         return proc.returncode
 205
 206     def _check_stale_images(self):
 207         namespace = self._args.namespace
 208         gitlab_uri = self._args.gitlab_uri
 209         registry_uri = util.get_registry_uri(namespace, gitlab_uri)
 210
 211         stale_images = util.get_registry_stale_images(registry_uri, self._basedir)
 212         if stale_images:
 213             spacing = "\n" + 4 * " "
 214             stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
 215             stale_details = spacing.join(stale_fmt)
 216             stale_ids = ' '.join([str(id) for id in stale_images.values()])
 217             registry_uri = util.get_registry_uri(namespace, gitlab_uri)
 218
 219             msg = textwrap.dedent(f"""
 220                 The following images are stale and can be purged from the registry:
 221
 222                     STALE_DETAILS
 223
 224                 You can delete the images listed above using this shell snippet:
 225
 226                     $ for image_id in {stale_ids}; do
 227                           curl --request DELETE --header "PRIVATE-TOKEN: <access_token>" \\
 228                                {registry_uri}/$image_id;
 229                       done
 230
 231                 You can generate a personal access token here:
 232
 233                     {gitlab_uri}/-/profile/personal_access_tokens
 234             """)
 235             print(msg.replace("STALE_DETAILS", stale_details))
 236
 237     @required_deps("git")
 238     def _action_run(self):
 239         return self._lcitool_run(self._args.job)
 240
 241     def _action_list_images(self):
 242         registry_uri = util.get_registry_uri(self._args.namespace,
 243                                              self._args.gitlab_uri)
 244         images = util.get_registry_images(registry_uri)
 245
 246         # skip the "ci-" prefix each of our container images' name has
 247         name_prefix = "ci-"
 248         names = [i["name"][len(name_prefix):] for i in images]
 249         names.sort()
 250
 251         native = [name for name in names if "-cross-" not in name]
 252         cross = [name for name in names if "-cross-" in name]
 253
 254         spacing = 4 * " "
 255         print("Available x86 container images:\n")
 256         print(spacing + ("\n" + spacing).join(native))
 257
 258         if cross:
 259             print()
 260             print("Available cross-compiler container images:\n")
 261             print(spacing + ("\n" + spacing).join(cross))
 262
 263     def _action_check_stale(self):
 264         self._check_stale_images()
 265
 266     def run(self):
 267         self._args.func(self)
 268
 269
 270 if __name__ == "__main__":
 271     Application().run()