pkgs/os-specific/bsd/freebsd/update.py

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -i python3 -p git "python3.withPackages (ps: with ps; [ gitpython packaging beautifulsoup4 pandas lxml ])"
   3
   4 import bs4
   5 import git
   6 import io
   7 import json
   8 import os
   9 import packaging.version
  10 import pandas
  11 import re
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import typing
  16 import urllib.request
  17
  18 _QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
  19 _RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$')
  20 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
  21 MIN_VERSION = packaging.version.Version("13.0.0")
  22 MAIN_BRANCH = "main"
  23 TAG_PATTERN = re.compile(
  24     f"^release/({packaging.version.VERSION_PATTERN})$", re.IGNORECASE | re.VERBOSE
  25 )
  26 REMOTE = "origin"
  27 BRANCH_PATTERN = re.compile(
  28     f"^{REMOTE}/((stable|releng)/({packaging.version.VERSION_PATTERN}))$",
  29     re.IGNORECASE | re.VERBOSE,
  30 )
  31
  32
  33 def request_supported_refs() -> list[str]:
  34     # Looks pretty shady but I think this should work with every version of the page in the last 20 years
  35     r = re.compile("^h\d$", re.IGNORECASE)
  36     soup = bs4.BeautifulSoup(
  37         urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
  38     )
  39     header = soup.find(
  40         lambda tag: r.match(tag.name) is not None
  41         and tag.text.lower() == "supported freebsd releases"
  42     )
  43     table = header.find_next("table")
  44     df = pandas.read_html(io.StringIO(table.prettify()))[0]
  45     return list(df["Branch"])
  46
  47
  48 def query_version(repo: git.Repo) -> dict[str, typing.Any]:
  49     # This only works on FreeBSD 13 and later
  50     text = (
  51         subprocess.check_output(
  52             ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"]
  53         )
  54         .decode("utf-8")
  55         .strip()
  56     )
  57     fields = dict()
  58     for line in text.splitlines():
  59         m = _QUERY_VERSION_PATTERN.match(line)
  60         if m is None:
  61             continue
  62         fields[m[1].lower()] = m[2]
  63
  64     parsed = packaging.version.parse(fields["revision"])
  65     fields["major"] = parsed.major
  66     fields["minor"] = parsed.minor
  67
  68     # Extract the patch number from `RELAESE-p<patch>`, which is used
  69     # e.g. in the "releng" branches.
  70     m = _RELEASE_PATCH_PATTERN.match(fields["branch"])
  71     if m is not None:
  72         fields["patch"] = m[1]
  73
  74     return fields
  75
  76
  77 def handle_commit(
  78     repo: git.Repo,
  79     rev: git.objects.commit.Commit,
  80     ref_name: str,
  81     ref_type: str,
  82     supported_refs: list[str],
  83     old_versions: dict[str, typing.Any],
  84 ) -> dict[str, typing.Any]:
  85     if old_versions.get(ref_name, {}).get("rev", None) == rev.hexsha:
  86         print(f"{ref_name}: revision still {rev.hexsha}, skipping")
  87         return old_versions[ref_name]
  88
  89     repo.git.checkout(rev)
  90     print(f"{ref_name}: checked out {rev.hexsha}")
  91
  92     full_hash = (
  93         subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir])
  94         .decode("utf-8")
  95         .strip()
  96     )
  97     print(f"{ref_name}: hash is {full_hash}")
  98
  99     version = query_version(repo)
 100     print(f"{ref_name}: version is {version['version']}")
 101
 102     return {
 103         "rev": rev.hexsha,
 104         "hash": full_hash,
 105         "ref": ref_name,
 106         "refType": ref_type,
 107         "supported": ref_name in supported_refs,
 108         "version": version,
 109     }
 110
 111
 112 def main() -> None:
 113     # Normally uses /run/user/*, which is on a tmpfs and too small
 114     temp_dir = tempfile.TemporaryDirectory(dir="/tmp")
 115     print(f"Selected temporary directory {temp_dir.name}")
 116
 117     if len(sys.argv) >= 2:
 118         orig_repo = git.Repo(sys.argv[1])
 119         print(f"Fetching updates on {orig_repo.git_dir}")
 120         orig_repo.remote("origin").fetch()
 121     else:
 122         print("Cloning source repo")
 123         orig_repo = git.Repo.clone_from(
 124             "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig")
 125         )
 126
 127     supported_refs = request_supported_refs()
 128     print(f"Supported refs are: {' '.join(supported_refs)}")
 129
 130     print("Doing git crimes, do not run `git worktree prune` until after script finishes!")
 131     workdir = os.path.join(temp_dir.name, "work")
 132     git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)
 133
 134     # Have to create object before removing .git otherwise it will complain
 135     repo = git.Repo(workdir)
 136     repo.git.set_persistent_git_options(git_dir=repo.git_dir)
 137     # Remove so that nix hash doesn't see the file
 138     os.remove(os.path.join(workdir, ".git"))
 139
 140     print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")
 141
 142
 143     try:
 144         with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
 145             old_versions = json.load(f)
 146     except FileNotFoundError:
 147         old_versions = dict()
 148
 149     versions = dict()
 150     for tag in repo.tags:
 151         m = TAG_PATTERN.match(tag.name)
 152         if m is None:
 153             continue
 154         version = packaging.version.parse(m[1])
 155         if version < MIN_VERSION:
 156             print(f"Skipping old tag {tag.name} ({version})")
 157             continue
 158
 159         print(f"Trying tag {tag.name} ({version})")
 160
 161         result = handle_commit(
 162             repo, tag.commit, tag.name, "tag", supported_refs, old_versions
 163         )
 164
 165         # Hack in the patch version from parsing the tag, if we didn't
 166         # get one from the "branch" field (from newvers). This is
 167         # probably 0.
 168         versionObj = result["version"]
 169         if "patch" not in versionObj:
 170             versionObj["patch"] = version.micro
 171
 172         versions[tag.name] = result
 173
 174     for branch in repo.remote("origin").refs:
 175         m = BRANCH_PATTERN.match(branch.name)
 176         if m is not None:
 177             fullname = m[1]
 178             version = packaging.version.parse(m[3])
 179             if version < MIN_VERSION:
 180                 print(f"Skipping old branch {fullname} ({version})")
 181                 continue
 182             print(f"Trying branch {fullname} ({version})")
 183         elif branch.name == f"{REMOTE}/{MAIN_BRANCH}":
 184             fullname = MAIN_BRANCH
 185             print(f"Trying development branch {fullname}")
 186         else:
 187             continue
 188
 189         result = handle_commit(
 190             repo, branch.commit, fullname, "branch", supported_refs, old_versions
 191         )
 192         versions[fullname] = result
 193
 194
 195     with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
 196         json.dump(versions, out, sort_keys=True, indent=2)
 197         out.write("\n")
 198
 199 if __name__ == '__main__':
 200     main()