maintainers/scripts/kde/generate-sources.py

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.pyyaml ])"
   3 import base64
   4 import binascii
   5 import json
   6 import pathlib
   7 from typing import Optional
   8 from urllib.parse import urlparse
   9
  10 import bs4
  11 import click
  12 import httpx
  13 import jinja2
  14
  15 import utils
  16
  17
  18 LEAF_TEMPLATE = jinja2.Template('''
  19 { mkKdeDerivation }:
  20 mkKdeDerivation {
  21   pname = "{{ pname }}";
  22 }
  23 '''.strip())
  24
  25 ROOT_TEMPLATE = jinja2.Template('''
  26 {callPackage}: {
  27   {%- for p in packages %}
  28   {{ p }} = callPackage ./{{ p }} { };
  29   {%- endfor %}
  30 }
  31 '''.strip());
  32
  33 def to_sri(hash):
  34     raw = binascii.unhexlify(hash)
  35     b64 = base64.b64encode(raw).decode()
  36     return f"sha256-{b64}"
  37
  38
  39 @click.command
  40 @click.argument(
  41     "set",
  42     type=click.Choice(["frameworks", "gear", "plasma"]),
  43     required=True
  44 )
  45 @click.argument(
  46     "version",
  47     type=str,
  48     required=True
  49 )
  50 @click.option(
  51     "--nixpkgs",
  52     type=click.Path(
  53         exists=True,
  54         file_okay=False,
  55         resolve_path=True,
  56         writable=True,
  57         path_type=pathlib.Path,
  58     ),
  59     default=pathlib.Path(__file__).parent.parent.parent.parent
  60 )
  61 @click.option(
  62     "--sources-url",
  63     type=str,
  64     default=None,
  65 )
  66 def main(set: str, version: str, nixpkgs: pathlib.Path, sources_url: Optional[str]):
  67     root_dir = nixpkgs / "pkgs/kde"
  68     set_dir = root_dir / set
  69     generated_dir = root_dir / "generated"
  70     metadata = utils.KDERepoMetadata.from_json(generated_dir)
  71
  72     if sources_url is None:
  73         set_url = {
  74             "frameworks": "kf",
  75             "gear": "releases",
  76             "plasma": "plasma",
  77         }[set]
  78         sources_url = f"https://kde.org/info/sources/source-{set_url}-{version}/"
  79
  80     sources = httpx.get(sources_url)
  81     sources.raise_for_status()
  82     bs = bs4.BeautifulSoup(sources.text, features="html.parser")
  83
  84     results = {}
  85     for item in bs.select("tr")[1:]:
  86         link = item.select_one("td:nth-child(1) a")
  87         assert link
  88
  89         hash = item.select_one("td:nth-child(3) tt")
  90         assert hash
  91
  92         project_name, version = link.text.rsplit("-", maxsplit=1)
  93         if project_name not in metadata.projects_by_name:
  94             print(f"Warning: unknown tarball: {project_name}")
  95
  96         results[project_name] = {
  97             "version": version,
  98             "url": "mirror://kde" + urlparse(link.attrs["href"]).path,
  99             "hash": to_sri(hash.text)
 100         }
 101
 102         pkg_dir = set_dir / project_name
 103         pkg_file = pkg_dir / "default.nix"
 104         if not pkg_file.exists():
 105             print(f"Generated new package: {set}/{project_name}")
 106             pkg_dir.mkdir(parents=True, exist_ok=True)
 107             with pkg_file.open("w") as fd:
 108                 fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
 109
 110     set_dir.mkdir(parents=True, exist_ok=True)
 111     with (set_dir / "default.nix").open("w") as fd:
 112         fd.write(ROOT_TEMPLATE.render(packages=sorted(results.keys())) + "\n")
 113
 114     sources_dir = generated_dir / "sources"
 115     sources_dir.mkdir(parents=True, exist_ok=True)
 116     with (sources_dir / f"{set}.json").open("w") as fd:
 117         json.dump(results, fd, indent=2)
 118
 119
 120 if __name__ == "__main__":
 121     main()  # type: ignore