bin/copy-iuks-to-rsync-server-and-verify

   1 #!/usr/bin/python3
   2
   3 import argparse
   4 import logging
   5 import os
   6 import re
   7 import subprocess
   8 import sys
   9
  10 from typing import List
  11 from pathlib import Path
  12 from urllib.parse import urlparse
  13 from urllib.request import Request, urlopen
  14
  15 from bs4 import BeautifulSoup  # type: ignore
  16
  17 JENKINS_IUKS_BASE_URL = "https://nightly.tails.boum.org/build_IUKs"
  18 RSYNC_SERVER_HOSTNAME = "rsync.lizard"
  19 LOG_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
  20 log = logging.getLogger()
  21
  22
  23 def main():
  24     parser = argparse.ArgumentParser(
  25         description="Copy IUKs from Jenkins to our rsync server \
  26         and verify that they match those built locally")
  27     parser.add_argument("--hashes-file",
  28                         type=str,
  29                         action="store",
  30                         required=True)
  31     parser.add_argument("--jenkins-build-id",
  32                         type=int,
  33                         action="store",
  34                         required=True)
  35     parser.add_argument("--work-dir", type=str, action="store", default=".")
  36     parser.add_argument("-q",
  37                         "--quiet",
  38                         action="store_true",
  39                         help="quiet output")
  40     parser.add_argument("--debug", action="store_true", help="debug output")
  41     parser.add_argument("--skip-sending-hashes-file",
  42                         action="store_true",
  43                         help="Assume the hashes file was uploaded already")
  44     parser.add_argument("--skip-downloading-iuks",
  45                         action="store_true",
  46                         help="Assume the IUKs were already downloaded")
  47     args = parser.parse_args()
  48
  49     if args.debug:
  50         logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
  51     elif args.quiet:
  52         logging.basicConfig(level=logging.WARN, format=LOG_FORMAT)
  53     else:
  54         logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
  55
  56     if not Path(args.hashes_file).exists():
  57         log.error("%s does not exist", args.hashes_file)
  58         sys.exit(1)
  59
  60     if not args.skip_sending_hashes_file:
  61         send_hashes_file(
  62             hashes_file=args.hashes_file,
  63             desthost=RSYNC_SERVER_HOSTNAME,
  64             destdir=args.work_dir,
  65         )
  66
  67     if not args.skip_downloading_iuks:
  68         download_iuks_from_jenkins(
  69             hashes_file=args.hashes_file,
  70             desthost=RSYNC_SERVER_HOSTNAME,
  71             destdir=args.work_dir,
  72             jenkins_iuks_base_url=JENKINS_IUKS_BASE_URL,
  73             jenkins_build_id=args.jenkins_build_id,
  74         )
  75
  76     verify_iuks(
  77         desthost=RSYNC_SERVER_HOSTNAME,
  78         iuks_dir=args.work_dir,
  79         hashes_file=Path(args.work_dir, args.hashes_file).name,
  80     )
  81
  82
  83 def send_hashes_file(hashes_file: str, desthost: str, destdir: str) -> None:
  84     log.info("Sending %(f)s to %(d)s on %(h)s…" % {
  85         "f": hashes_file,
  86         "d": destdir,
  87         "h": desthost,
  88     })
  89     subprocess.run(
  90         ["scp", hashes_file, "%s:%s" % (desthost, destdir)], check=True)
  91
  92
  93 def iuks_listed_in(hashes_file: str) -> List[str]:
  94     with Path(hashes_file).open() as f:
  95         lines = f.readlines()
  96     return [line.split('  ')[-1].rstrip() for line in lines]
  97
  98
  99 def get_jenkins_iuks_urls(jenkins_iuks_base_url: str,
 100                           jenkins_build_id: int) -> List[str]:
 101     urls: List[str] = []
 102     source_version_index_url = jenkins_iuks_base_url + \
 103         "/configurations/axis-SOURCE_VERSION"
 104     for source_version_url in [
 105             source_version_index_url + '/' + link.get('href')
 106             for link in BeautifulSoup(
 107                 urlopen(Request(source_version_index_url)),
 108                 'html.parser').find_all(href=re.compile('^[1-9]'))
 109     ]:
 110         axis_label_index_url = source_version_url + "axis-label_exp/"
 111         log.debug("Looking at %s", axis_label_index_url)
 112         label_urls = [
 113             axis_label_index_url + link.get('href')
 114             for link in BeautifulSoup(urlopen(Request(axis_label_index_url)),
 115                                       'html.parser').find_all(
 116                                           href=re.compile('^[a-z]'))
 117         ]
 118         if len(label_urls) == 0:
 119             log.debug("Found no label URL in %s, ignoring this source version",
 120                       axis_label_index_url)
 121             continue
 122         if len(label_urls) > 1:
 123             log.error("Found too many label URLs in %s: %s",
 124                       axis_label_index_url, label_urls)
 125             sys.exit(1)
 126         label_url = label_urls[0]
 127
 128         artifacts_index_url = label_url + '/builds/' + str(
 129             jenkins_build_id) + '/archive/'
 130         log.debug("Looking at %s", artifacts_index_url)
 131         iuk_urls = [
 132             artifacts_index_url + link.get('href') for link in BeautifulSoup(
 133                 urlopen(Request(artifacts_index_url)), 'html.parser').find_all(
 134                     href=re.compile('[.]iuk$'))
 135         ]
 136         if len(iuk_urls) == 0:
 137             log.debug("Found no IUK URL in %s, ignoring this source version",
 138                       artifacts_index_url)
 139             continue
 140         if len(iuk_urls) > 1:
 141             log.error("Found too many IUK URLs in %s: %s", artifacts_index_url,
 142                       iuk_urls)
 143             sys.exit(1)
 144         else:
 145             iuk_url = iuk_urls[0]
 146         urls.append(iuk_url)
 147     log.debug("Found IUK URLs: %s", urls)
 148     return urls
 149
 150
 151 def download_iuks_from_jenkins(hashes_file: str, desthost: str, destdir: str,
 152                                jenkins_iuks_base_url: str,
 153                                jenkins_build_id: int) -> None:
 154     log.info("Downloading IUKs from Jenkins to %s…", desthost)
 155     expected_iuks = iuks_listed_in(hashes_file)
 156     log.debug("IUKS: %s", ', '.join(expected_iuks))
 157     jenkins_iuks_urls = get_jenkins_iuks_urls(jenkins_iuks_base_url,
 158                                               jenkins_build_id)
 159     jenkins_iuks = [
 160         os.path.basename(urlparse(url).path) for url in jenkins_iuks_urls
 161     ]
 162     if set(expected_iuks) != set(jenkins_iuks):
 163         log.error(
 164             "Jenkins' set of IUKs differs from local one:\n"
 165             " - locally: %s\n"
 166             " - Jenkins: %s\n",
 167             expected_iuks, jenkins_iuks)
 168         sys.exit(1)
 169     for iuk_url in jenkins_iuks_urls:
 170         log.debug("Downloading %s to %s", iuk_url, destdir)
 171         subprocess.run([
 172             "ssh", desthost, "wget", "--quiet", "--no-clobber",
 173             "--directory-prefix=%s" % destdir, iuk_url
 174         ],
 175                        check=True)
 176
 177
 178 def verify_iuks(desthost: str, iuks_dir: str, hashes_file: str) -> None:
 179     log.info("Verifying that IUKs built on Jenkins match those you've built…")
 180     try:
 181         subprocess.run([
 182             "ssh", desthost,
 183             "cd '%(d)s' && sha256sum --check --strict '%(f)s'" % {
 184                 "d": iuks_dir,
 185                 "f": Path(hashes_file).name,
 186             }
 187         ],
 188                        check=True)
 189     except subprocess.CalledProcessError:
 190         print("\nERROR: IUKs built on Jenkins don't match yours\n",
 191               file=sys.stderr)
 192
 193
 194 if __name__ == "__main__":
 195     main()