pkgs/applications/emulators/retroarch/update_cores.py

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -I nixpkgs=./ -i python3 -p "python3.withPackages (ps: with ps; [ ])" -p git -p nix-prefetch-github -p nix-prefetch-git -p nix-prefetch-scripts
   3
   4 import json
   5 import os
   6 import subprocess
   7 import sys
   8 from concurrent.futures import ThreadPoolExecutor
   9 from pathlib import Path
  10
  11 SCRIPT_PATH = Path(__file__).absolute().parent
  12 HASHES_PATH = SCRIPT_PATH / "hashes.json"
  13 GET_REPO_THREADS = int(os.environ.get("GET_REPO_THREADS", 8))
  14 # To add a new core, add it to the dictionary below. You need to set at least
  15 # `repo`, that is the repository name if the owner of the repository is
  16 # `libretro` itself, otherwise also set `owner`.
  17 # You may set `deep_clone`, `fetch_submodules` or `leave_dot_git` options to
  18 # `True` and they're similar to `fetchgit` options. Also if for some reason you
  19 # need to pin a specific revision, set `rev` to a commit.
  20 # There is also a `fetcher` option that for now only supports `fetchFromGitHub`
  21 # (see `get_repo_hash()`), but it may be extended in the future if there is a
  22 # need to support fetchers from other source hubs.
  23 # To generate the hash file for your new core, you can run
  24 # `<nixpkgs>/pkgs/applications/emulators/retroarch/update_cores.py <core>`. Do
  25 # not forget to add your core to `cores.nix` file with the proper overrides so
  26 # the core can be build.
  27 CORES = {
  28     "2048": {"repo": "libretro-2048"},
  29     "atari800": {"repo": "libretro-atari800"},
  30     "beetle-gba": {"repo": "beetle-gba-libretro"},
  31     "beetle-lynx": {"repo": "beetle-lynx-libretro"},
  32     "beetle-ngp": {"repo": "beetle-ngp-libretro"},
  33     "beetle-pce": {"repo": "beetle-pce-libretro"},
  34     "beetle-pce-fast": {"repo": "beetle-pce-fast-libretro"},
  35     "beetle-pcfx": {"repo": "beetle-pcfx-libretro"},
  36     "beetle-psx": {"repo": "beetle-psx-libretro"},
  37     "beetle-saturn": {"repo": "beetle-saturn-libretro"},
  38     "beetle-supafaust": {"repo": "supafaust"},
  39     "beetle-supergrafx": {"repo": "beetle-supergrafx-libretro"},
  40     "beetle-vb": {"repo": "beetle-vb-libretro"},
  41     "beetle-wswan": {"repo": "beetle-wswan-libretro"},
  42     "blastem": {"repo": "blastem"},
  43     "bluemsx": {"repo": "bluemsx-libretro"},
  44     "bsnes": {"repo": "bsnes-libretro"},
  45     "bsnes-hd": {"repo": "bsnes-hd", "owner": "DerKoun"},
  46     "bsnes-mercury": {"repo": "bsnes-mercury"},
  47     "citra": {"repo": "citra", "fetch_submodules": True},
  48     "desmume": {"repo": "desmume"},
  49     "desmume2015": {"repo": "desmume2015"},
  50     "dolphin": {"repo": "dolphin"},
  51     "dosbox": {"repo": "dosbox-libretro"},
  52     "dosbox-pure": {"repo": "dosbox-pure", "owner": "schellingb"},
  53     # The EasyRPG core is pinned to 0.8 since it depends on version 0.8 of liblcf, which
  54     # was released in April 2023.
  55     # Update the version when a compatible liblcf is available.
  56     # See pkgs/games/easyrpg-player/default.nix for details.
  57     "easyrpg": {"repo": "Player", "owner": "EasyRPG", "fetch_submodules": True, "rev": "0.8"},
  58     "eightyone": {"repo": "81-libretro"},
  59     "fbalpha2012": {"repo": "fbalpha2012"},
  60     "fbneo": {"repo": "fbneo"},
  61     "fceumm": {"repo": "libretro-fceumm"},
  62     "flycast": {"repo": "flycast", "owner": "flyinghead", "fetch_submodules": True},
  63     "fmsx": {"repo": "fmsx-libretro"},
  64     "freeintv": {"repo": "freeintv"},
  65     "fuse": {"repo": "fuse-libretro"},
  66     "gambatte": {"repo": "gambatte-libretro"},
  67     "genesis-plus-gx": {"repo": "Genesis-Plus-GX"},
  68     "gpsp": {"repo": "gpsp"},
  69     "gw": {"repo": "gw-libretro"},
  70     "handy": {"repo": "libretro-handy"},
  71     "hatari": {"repo": "hatari"},
  72     # Setting fetch_submodules=True since libretro/mame constantly gives
  73     # different hashes for its tarballs, see:
  74     # - https://github.com/NixOS/nixpkgs/issues/259488#issuecomment-1751768379
  75     # - https://github.com/NixOS/nixpkgs/pull/303494
  76     "mame": {"repo": "mame", "fetch_submodules": True},
  77     "mame2000": {"repo": "mame2000-libretro"},
  78     "mame2003": {"repo": "mame2003-libretro"},
  79     "mame2003-plus": {"repo": "mame2003-plus-libretro"},
  80     "mame2010": {"repo": "mame2010-libretro"},
  81     "mame2015": {"repo": "mame2015-libretro"},
  82     "mame2016": {"repo": "mame2016-libretro"},
  83     "melonds": {"repo": "melonds"},
  84     "mesen": {"repo": "mesen"},
  85     "mesen-s": {"repo": "mesen-s"},
  86     "meteor": {"repo": "meteor-libretro"},
  87     "mrboom": {"repo": "mrboom-libretro", "owner": "Javanaise", "fetch_submodules": True},
  88     "mgba": {"repo": "mgba"},
  89     "mupen64plus": {"repo": "mupen64plus-libretro-nx"},
  90     "neocd": {"repo": "neocd_libretro"},
  91     "nestopia": {"repo": "nestopia"},
  92     "nxengine": {"repo": "nxengine-libretro"},
  93     "np2kai": {"repo": "NP2kai", "owner": "AZO234", "fetch_submodules": True},
  94     "o2em": {"repo": "libretro-o2em"},
  95     "opera": {"repo": "opera-libretro"},
  96     "parallel-n64": {"repo": "parallel-n64"},
  97     # libretro/lrps2 is a hard-fork of pcsx2 with simplified code to target
  98     # only libretro, while libretro/pcsx2 is supposedly closer to upstream but
  99     # it is a WIP.
 100     # TODO: switch to libretro/pcsx2 when upstream switches to it.
 101     "pcsx2": {"repo": "lrps2"},
 102     "pcsx_rearmed": {"repo": "pcsx_rearmed"},
 103     "picodrive": {"repo": "picodrive", "fetch_submodules": True},
 104     "play": {"repo": "Play-", "owner": "jpd002", "fetch_submodules": True},
 105     "ppsspp": {"repo": "ppsspp", "owner": "hrydgard", "fetch_submodules": True},
 106     "prboom": {"repo": "libretro-prboom"},
 107     "prosystem": {"repo": "prosystem-libretro"},
 108     "puae": {"repo": "libretro-uae"},
 109     "quicknes": {"repo": "QuickNES_Core"},
 110     "sameboy": {"repo": "sameboy"},
 111     "same_cdi": {"repo": "same_cdi"},
 112     # This is the old source code before they upstreamed the source code,
 113     # so now the libretro related code lives in the scummvm/scummvm repository.
 114     # However this broke the old way we were doing builds, so for now point
 115     # to a mirror with the old source code until this issue is fixed.
 116     # TODO: switch to libretro/scummvm since this is more up-to-date
 117     "scummvm": {"repo": "scummvm", "owner": "libretro-mirrors"},
 118     "smsplus-gx": {"repo": "smsplus-gx"},
 119     "snes9x": {"repo": "snes9x", "owner": "snes9xgit"},
 120     "snes9x2002": {"repo": "snes9x2002"},
 121     "snes9x2005": {"repo": "snes9x2005"},
 122     "snes9x2010": {"repo": "snes9x2010"},
 123     "stella": {"repo": "stella", "owner": "stella-emu"},
 124     "stella2014": {"repo": "stella2014-libretro"},
 125     "swanstation": {"repo": "swanstation"},
 126     "tgbdual": {"repo": "tgbdual-libretro"},
 127     "thepowdertoy": {"repo": "ThePowderToy"},
 128     "tic80": {"repo": "tic-80", "fetch_submodules": True},
 129     "vba-m": {"repo": "vbam-libretro"},
 130     "vba-next": {"repo": "vba-next"},
 131     "vecx": {"repo": "libretro-vecx"},
 132     "virtualjaguar": {"repo": "virtualjaguar-libretro"},
 133     "yabause": {"repo": "yabause"},
 134 }
 135
 136
 137 def info(*msg):
 138     print(*msg, file=sys.stderr)
 139
 140
 141 def get_repo_hash_fetchFromGitHub(
 142     repo,
 143     owner="libretro",
 144     deep_clone=False,
 145     fetch_submodules=False,
 146     leave_dot_git=False,
 147     rev=None,
 148 ):
 149     extra_args = []
 150     if deep_clone:
 151         extra_args.append("--deep-clone")
 152     else:
 153         extra_args.append("--no-deep-clone")
 154     if fetch_submodules:
 155         extra_args.append("--fetch-submodules")
 156     else:
 157         extra_args.append("--no-fetch-submodules")
 158     if leave_dot_git:
 159         extra_args.append("--leave-dot-git")
 160     else:
 161         extra_args.append("--no-leave-dot-git")
 162     if rev:
 163         extra_args.append("--rev")
 164         extra_args.append(rev)
 165     try:
 166         result = subprocess.run(
 167             ["nix-prefetch-github", owner, repo, "--meta", *extra_args],
 168             check=True,
 169             capture_output=True,
 170             text=True,
 171         )
 172     except subprocess.CalledProcessError as ex:
 173         info(f"Error while updating {owner}/{repo}:", ex.stderr)
 174         raise ex
 175
 176     j = json.loads(result.stdout)
 177     return {
 178         "fetcher": "fetchFromGitHub",
 179         # Remove False values
 180         "src": {k: v for k, v in j["src"].items() if v},
 181         "version": f"unstable-{j['meta']['commitDate']}",
 182     }
 183
 184
 185 def get_repo_hash(fetcher="fetchFromGitHub", **kwargs):
 186     if fetcher == "fetchFromGitHub":
 187         return get_repo_hash_fetchFromGitHub(**kwargs)
 188     else:
 189         raise ValueError(f"Unsupported fetcher: {fetcher}")
 190
 191
 192 def get_repo_hashes(cores={}):
 193     def get_repo_hash_from_core_def(core_def):
 194         core, repo = core_def
 195         info(f"Getting repo hash for '{core}'...")
 196         result = core, get_repo_hash(**repo)
 197         info(f"Got repo hash for '{core}'!")
 198         return result
 199
 200     with open(HASHES_PATH) as f:
 201         repo_hashes = json.loads(f.read())
 202
 203     info(f"Running with {GET_REPO_THREADS} threads!")
 204     with ThreadPoolExecutor(max_workers=GET_REPO_THREADS) as executor:
 205         new_repo_hashes = executor.map(get_repo_hash_from_core_def, cores.items())
 206
 207     for core, repo in new_repo_hashes:
 208         repo_hashes[core] = repo
 209
 210     return repo_hashes
 211
 212
 213 def main():
 214     # If you don't want to update all cores, pass the name of the cores you
 215     # want to update on the command line. E.g.:
 216     # $ ./update.py citra snes9x
 217     if len(sys.argv) > 1:
 218         cores_to_update = sys.argv[1:]
 219     else:
 220         cores_to_update = CORES.keys()
 221
 222     cores = {core: repo for core, repo in CORES.items() if core in cores_to_update}
 223     repo_hashes = get_repo_hashes(cores)
 224     repo_hashes["!comment"] = "Generated with update_cores.py script, do not edit!"
 225     info(f"Generating '{HASHES_PATH}'...")
 226     with open(HASHES_PATH, "w") as f:
 227         f.write(json.dumps(dict(sorted(repo_hashes.items())), indent=4))
 228         f.write("\n")
 229     info("Finished!")
 230
 231
 232 if __name__ == "__main__":
 233     main()