maintainers/scripts/pluginupdate.py

   1 # python library used to update plugins:
   2 # - pkgs/applications/editors/vim/plugins/update.py
   3 # - pkgs/applications/editors/kakoune/plugins/update.py
   4 # - maintainers/scripts/update-luarocks-packages
   5
   6 # format:
   7 # $ nix run nixpkgs.python3Packages.black -c black update.py
   8 # type-check:
   9 # $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
  10 # linted:
  11 # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
  12
  13 import argparse
  14 import csv
  15 import functools
  16 import http
  17 import json
  18 import os
  19 import subprocess
  20 import logging
  21 import sys
  22 import time
  23 import traceback
  24 import urllib.error
  25 import urllib.parse
  26 import urllib.request
  27 import xml.etree.ElementTree as ET
  28 from datetime import datetime
  29 from functools import wraps
  30 from multiprocessing.dummy import Pool
  31 from pathlib import Path
  32 from typing import Dict, List, Optional, Tuple, Union, Any, Callable
  33 from urllib.parse import urljoin, urlparse
  34 from tempfile import NamedTemporaryFile
  35 from dataclasses import dataclass, asdict
  36
  37 import git
  38
  39 ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry"  # " vim gets confused here
  40 ATOM_LINK = "{http://www.w3.org/2005/Atom}link"  # "
  41 ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated"  # "
  42
  43 LOG_LEVELS = {
  44     logging.getLevelName(level): level for level in [
  45         logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
  46 }
  47
  48 log = logging.getLogger()
  49
  50 def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
  51     """Retry calling the decorated function using an exponential backoff.
  52     http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
  53     original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
  54     (BSD licensed)
  55     :param ExceptionToCheck: the exception on which to retry
  56     :param tries: number of times to try (not retry) before giving up
  57     :param delay: initial delay between retries in seconds
  58     :param backoff: backoff multiplier e.g. value of 2 will double the delay
  59         each retry
  60     """
  61
  62     def deco_retry(f: Callable) -> Callable:
  63         @wraps(f)
  64         def f_retry(*args: Any, **kwargs: Any) -> Any:
  65             mtries, mdelay = tries, delay
  66             while mtries > 1:
  67                 try:
  68                     return f(*args, **kwargs)
  69                 except ExceptionToCheck as e:
  70                     print(f"{str(e)}, Retrying in {mdelay} seconds...")
  71                     time.sleep(mdelay)
  72                     mtries -= 1
  73                     mdelay *= backoff
  74             return f(*args, **kwargs)
  75
  76         return f_retry  # true decorator
  77
  78     return deco_retry
  79
  80 @dataclass
  81 class FetchConfig:
  82     proc: int
  83     github_token: str
  84
  85
  86 def make_request(url: str, token=None) -> urllib.request.Request:
  87     headers = {}
  88     if token is not None:
  89         headers["Authorization"] = f"token {token}"
  90     return urllib.request.Request(url, headers=headers)
  91
  92
  93 # a dictionary of plugins and their new repositories
  94 Redirects = Dict['PluginDesc', 'Repo']
  95
  96 class Repo:
  97     def __init__(
  98         self, uri: str, branch: str
  99     ) -> None:
 100         self.uri = uri
 101         '''Url to the repo'''
 102         self._branch = branch
 103         # Redirect is the new Repo to use
 104         self.redirect: Optional['Repo'] = None
 105         self.token = "dummy_token"
 106
 107     @property
 108     def name(self):
 109         return self.uri.split('/')[-1]
 110
 111     @property
 112     def branch(self):
 113         return self._branch or "HEAD"
 114
 115     def __str__(self) -> str:
 116         return f"{self.uri}"
 117     def __repr__(self) -> str:
 118         return f"Repo({self.name}, {self.uri})"
 119
 120     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
 121     def has_submodules(self) -> bool:
 122         return True
 123
 124     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
 125     def latest_commit(self) -> Tuple[str, datetime]:
 126         log.debug("Latest commit")
 127         loaded = self._prefetch(None)
 128         updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
 129
 130         return loaded['rev'], updated
 131
 132     def _prefetch(self, ref: Optional[str]):
 133         cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri]
 134         if ref is not None:
 135             cmd.append(ref)
 136         log.debug(cmd)
 137         data = subprocess.check_output(cmd)
 138         loaded = json.loads(data)
 139         return loaded
 140
 141     def prefetch(self, ref: Optional[str]) -> str:
 142         print("Prefetching")
 143         loaded = self._prefetch(ref)
 144         return loaded["sha256"]
 145
 146     def as_nix(self, plugin: "Plugin") -> str:
 147         return f'''fetchgit {{
 148       url = "{self.uri}";
 149       rev = "{plugin.commit}";
 150       sha256 = "{plugin.sha256}";
 151     }}'''
 152
 153
 154 class RepoGitHub(Repo):
 155     def __init__(
 156         self, owner: str, repo: str, branch: str
 157     ) -> None:
 158         self.owner = owner
 159         self.repo = repo
 160         self.token = None
 161         '''Url to the repo'''
 162         super().__init__(self.url(""), branch)
 163         log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
 164
 165     @property
 166     def name(self):
 167         return self.repo
 168
 169     def url(self, path: str) -> str:
 170         res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
 171         return res
 172
 173     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
 174     def has_submodules(self) -> bool:
 175         try:
 176             req = make_request(self.url(f"blob/{self.branch}/.gitmodules"), self.token)
 177             urllib.request.urlopen(req, timeout=10).close()
 178         except urllib.error.HTTPError as e:
 179             if e.code == 404:
 180                 return False
 181             else:
 182                 raise
 183         return True
 184
 185     @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
 186     def latest_commit(self) -> Tuple[str, datetime]:
 187         commit_url = self.url(f"commits/{self.branch}.atom")
 188         log.debug("Sending request to %s", commit_url)
 189         commit_req = make_request(commit_url, self.token)
 190         with urllib.request.urlopen(commit_req, timeout=10) as req:
 191             self._check_for_redirect(commit_url, req)
 192             xml = req.read()
 193             root = ET.fromstring(xml)
 194             latest_entry = root.find(ATOM_ENTRY)
 195             assert latest_entry is not None, f"No commits found in repository {self}"
 196             commit_link = latest_entry.find(ATOM_LINK)
 197             assert commit_link is not None, f"No link tag found feed entry {xml}"
 198             url = urlparse(commit_link.get("href"))
 199             updated_tag = latest_entry.find(ATOM_UPDATED)
 200             assert (
 201                 updated_tag is not None and updated_tag.text is not None
 202             ), f"No updated tag found feed entry {xml}"
 203             updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
 204             return Path(str(url.path)).name, updated
 205
 206     def _check_for_redirect(self, url: str, req: http.client.HTTPResponse):
 207         response_url = req.geturl()
 208         if url != response_url:
 209             new_owner, new_name = (
 210                 urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
 211             )
 212
 213             new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
 214             self.redirect = new_repo
 215
 216
 217     def prefetch(self, commit: str) -> str:
 218         if self.has_submodules():
 219             sha256 = super().prefetch(commit)
 220         else:
 221             sha256 = self.prefetch_github(commit)
 222         return sha256
 223
 224     def prefetch_github(self, ref: str) -> str:
 225         cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
 226         log.debug("Running %s", cmd)
 227         data = subprocess.check_output(cmd)
 228         return data.strip().decode("utf-8")
 229
 230     def as_nix(self, plugin: "Plugin") -> str:
 231         if plugin.has_submodules:
 232             submodule_attr = "\n      fetchSubmodules = true;"
 233         else:
 234             submodule_attr = ""
 235
 236         return f'''fetchFromGitHub {{
 237       owner = "{self.owner}";
 238       repo = "{self.repo}";
 239       rev = "{plugin.commit}";
 240       sha256 = "{plugin.sha256}";{submodule_attr}
 241     }}'''
 242
 243
 244 @dataclass(frozen=True)
 245 class PluginDesc:
 246     repo: Repo
 247     branch: str
 248     alias: Optional[str]
 249
 250     @property
 251     def name(self):
 252         if self.alias is None:
 253             return self.repo.name
 254         else:
 255             return self.alias
 256
 257     def __lt__(self, other):
 258         return self.repo.name < other.repo.name
 259
 260     @staticmethod
 261     def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
 262         branch = row["branch"]
 263         repo = make_repo(row['repo'], branch.strip())
 264         repo.token = config.github_token
 265         return PluginDesc(repo, branch.strip(), row["alias"])
 266
 267
 268     @staticmethod
 269     def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
 270         branch = "HEAD"
 271         alias = None
 272         uri = line
 273         if " as " in uri:
 274             uri, alias = uri.split(" as ")
 275             alias = alias.strip()
 276         if "@" in uri:
 277             uri, branch = uri.split("@")
 278         repo = make_repo(uri.strip(), branch.strip())
 279         repo.token = config.github_token
 280         return PluginDesc(repo, branch.strip(), alias)
 281
 282 @dataclass
 283 class Plugin:
 284     name: str
 285     commit: str
 286     has_submodules: bool
 287     sha256: str
 288     date: Optional[datetime] = None
 289
 290     @property
 291     def normalized_name(self) -> str:
 292         return self.name.replace(".", "-")
 293
 294     @property
 295     def version(self) -> str:
 296         assert self.date is not None
 297         return self.date.strftime("%Y-%m-%d")
 298
 299     def as_json(self) -> Dict[str, str]:
 300         copy = self.__dict__.copy()
 301         del copy["date"]
 302         return copy
 303
 304
 305 def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
 306     log.debug("Load plugins from csv %s", input_file)
 307     plugins = []
 308     with open(input_file, newline='') as csvfile:
 309         log.debug("Writing into %s", input_file)
 310         reader = csv.DictReader(csvfile,)
 311         for line in reader:
 312             plugin = PluginDesc.load_from_csv(config, line)
 313             plugins.append(plugin)
 314
 315     return plugins
 316
 317 def run_nix_expr(expr):
 318     with CleanEnvironment():
 319         cmd = ["nix", "eval", "--extra-experimental-features",
 320                 "nix-command", "--impure", "--json", "--expr", expr]
 321         log.debug("Running command %s", " ".join(cmd))
 322         out = subprocess.check_output(cmd)
 323         data = json.loads(out)
 324         return data
 325
 326
 327 class Editor:
 328     """The configuration of the update script."""
 329
 330     def __init__(
 331         self,
 332         name: str,
 333         root: Path,
 334         get_plugins: str,
 335         default_in: Optional[Path] = None,
 336         default_out: Optional[Path] = None,
 337         deprecated: Optional[Path] = None,
 338         cache_file: Optional[str] = None,
 339     ):
 340         log.debug("get_plugins:", get_plugins)
 341         self.name = name
 342         self.root = root
 343         self.get_plugins = get_plugins
 344         self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
 345         self.default_out = default_out or root.joinpath("generated.nix")
 346         self.deprecated = deprecated or root.joinpath("deprecated.json")
 347         self.cache_file = cache_file or f"{name}-plugin-cache.json"
 348         self.nixpkgs_repo = None
 349
 350     def add(self, args):
 351         '''CSV spec'''
 352         log.debug("called the 'add' command")
 353         fetch_config = FetchConfig(args.proc, args.github_token)
 354         editor = self
 355         for plugin_line in args.add_plugins:
 356             log.debug("using plugin_line", plugin_line)
 357             pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
 358             log.debug("loaded as pdesc", pdesc)
 359             append = [ pdesc ]
 360             editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
 361             plugin, _ = prefetch_plugin(pdesc, )
 362             autocommit = not args.no_commit
 363             if autocommit:
 364                 commit(
 365                     editor.nixpkgs_repo,
 366                     "{drv_name}: init at {version}".format(
 367                         drv_name=editor.get_drv_name(plugin.normalized_name),
 368                         version=plugin.version
 369                     ),
 370                     [args.outfile, args.input_file],
 371                 )
 372
 373     # Expects arguments generated by 'update' subparser
 374     def update(self, args ):
 375         '''CSV spec'''
 376         print("the update member function should be overriden in subclasses")
 377
 378     def get_current_plugins(self) -> List[Plugin]:
 379         """To fill the cache"""
 380         data = run_nix_expr(self.get_plugins)
 381         plugins = []
 382         for name, attr in data.items():
 383             p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
 384             plugins.append(p)
 385         return plugins
 386
 387     def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
 388         '''CSV spec'''
 389         return load_plugins_from_csv(config, plugin_file)
 390
 391     def generate_nix(self, _plugins, _outfile: str):
 392         '''Returns nothing for now, writes directly to outfile'''
 393         raise NotImplementedError()
 394
 395     def get_update(self, input_file: str, outfile: str, config: FetchConfig):
 396         cache: Cache = Cache(self.get_current_plugins(), self.cache_file)
 397         _prefetch = functools.partial(prefetch, cache=cache)
 398
 399         def update() -> dict:
 400             plugins = self.load_plugin_spec(config, input_file)
 401
 402             try:
 403                 pool = Pool(processes=config.proc)
 404                 results = pool.map(_prefetch, plugins)
 405             finally:
 406                 cache.store()
 407
 408             plugins, redirects = check_results(results)
 409
 410             self.generate_nix(plugins, outfile)
 411
 412             return redirects
 413
 414         return update
 415
 416
 417     @property
 418     def attr_path(self):
 419         return self.name + "Plugins"
 420
 421     def get_drv_name(self, name: str):
 422         return self.attr_path + "." + name
 423
 424     def rewrite_input(self, *args, **kwargs):
 425         return rewrite_input(*args, **kwargs)
 426
 427     def create_parser(self):
 428         common = argparse.ArgumentParser(
 429             add_help=False,
 430             description=(f"""
 431                 Updates nix derivations for {self.name} plugins.\n
 432                 By default from {self.default_in} to {self.default_out}"""
 433             )
 434         )
 435         common.add_argument(
 436             "--input-names",
 437             "-i",
 438             dest="input_file",
 439             default=self.default_in,
 440             help="A list of plugins in the form owner/repo",
 441         )
 442         common.add_argument(
 443             "--out",
 444             "-o",
 445             dest="outfile",
 446             default=self.default_out,
 447             help="Filename to save generated nix code",
 448         )
 449         common.add_argument(
 450             "--proc",
 451             "-p",
 452             dest="proc",
 453             type=int,
 454             default=30,
 455             help="Number of concurrent processes to spawn. Setting --github-token allows higher values.",
 456         )
 457         common.add_argument(
 458             "--github-token",
 459             "-t",
 460             type=str,
 461             default=os.getenv("GITHUB_API_TOKEN"),
 462             help="""Allows to set --proc to higher values.
 463             Uses GITHUB_API_TOKEN environment variables as the default value.""",
 464         )
 465         common.add_argument(
 466             "--no-commit", "-n", action="store_true", default=False,
 467             help="Whether to autocommit changes"
 468         )
 469         common.add_argument(
 470             "--debug", "-d", choices=LOG_LEVELS.keys(),
 471             default=logging.getLevelName(logging.WARN),
 472             help="Adjust log level"
 473         )
 474
 475         main = argparse.ArgumentParser(
 476             parents=[common],
 477             description=(f"""
 478                 Updates nix derivations for {self.name} plugins.\n
 479                 By default from {self.default_in} to {self.default_out}"""
 480             )
 481         )
 482
 483         subparsers = main.add_subparsers(dest="command", required=False)
 484         padd = subparsers.add_parser(
 485             "add", parents=[],
 486             description="Add new plugin",
 487             add_help=False,
 488         )
 489         padd.set_defaults(func=self.add)
 490         padd.add_argument(
 491             "add_plugins",
 492             default=None,
 493             nargs="+",
 494             help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
 495         )
 496
 497         pupdate = subparsers.add_parser(
 498             "update",
 499             description="Update all or a subset of existing plugins",
 500             add_help=False,
 501         )
 502         pupdate.set_defaults(func=self.update)
 503         return main
 504
 505     def run(self,):
 506         '''
 507         Convenience function
 508         '''
 509         parser = self.create_parser()
 510         args = parser.parse_args()
 511         command = args.command or "update"
 512         log.setLevel(LOG_LEVELS[args.debug])
 513         log.info("Chose to run command: %s", command)
 514
 515         if not args.no_commit:
 516             self.nixpkgs_repo = git.Repo(self.root, search_parent_directories=True)
 517
 518         getattr(self, command)(args)
 519
 520
 521
 522
 523 class CleanEnvironment(object):
 524     def __enter__(self) -> None:
 525         self.old_environ = os.environ.copy()
 526         local_pkgs = str(Path(__file__).parent.parent.parent)
 527         os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
 528         self.empty_config = NamedTemporaryFile()
 529         self.empty_config.write(b"{}")
 530         self.empty_config.flush()
 531         os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
 532
 533     def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
 534         os.environ.update(self.old_environ)
 535         self.empty_config.close()
 536
 537
 538 def prefetch_plugin(
 539     p: PluginDesc,
 540     cache: "Optional[Cache]" = None,
 541 ) -> Tuple[Plugin, Optional[Repo]]:
 542     repo, branch, alias = p.repo, p.branch, p.alias
 543     name = alias or p.repo.name
 544     commit = None
 545     log.info(f"Fetching last commit for plugin {name} from {repo.uri}@{branch}")
 546     commit, date = repo.latest_commit()
 547     cached_plugin = cache[commit] if cache else None
 548     if cached_plugin is not None:
 549         log.debug("Cache hit !")
 550         cached_plugin.name = name
 551         cached_plugin.date = date
 552         return cached_plugin, repo.redirect
 553
 554     has_submodules = repo.has_submodules()
 555     log.debug(f"prefetch {name}")
 556     sha256 = repo.prefetch(commit)
 557
 558     return (
 559         Plugin(name, commit, has_submodules, sha256, date=date),
 560         repo.redirect,
 561     )
 562
 563
 564 def print_download_error(plugin: PluginDesc, ex: Exception):
 565     print(f"{plugin}: {ex}", file=sys.stderr)
 566     ex_traceback = ex.__traceback__
 567     tb_lines = [
 568         line.rstrip("\n")
 569         for line in traceback.format_exception(ex.__class__, ex, ex_traceback)
 570     ]
 571     print("\n".join(tb_lines))
 572
 573 def check_results(
 574     results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]]
 575 ) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
 576     ''' '''
 577     failures: List[Tuple[PluginDesc, Exception]] = []
 578     plugins = []
 579     redirects: Redirects = {}
 580     for (pdesc, result, redirect) in results:
 581         if isinstance(result, Exception):
 582             failures.append((pdesc, result))
 583         else:
 584             new_pdesc = pdesc
 585             if redirect is not None:
 586                 redirects.update({pdesc: redirect})
 587                 new_pdesc = PluginDesc(redirect, pdesc.branch, pdesc.alias)
 588             plugins.append((new_pdesc, result))
 589
 590     print(f"{len(results) - len(failures)} plugins were checked", end="")
 591     if len(failures) == 0:
 592         print()
 593         return plugins, redirects
 594     else:
 595         print(f", {len(failures)} plugin(s) could not be downloaded:\n")
 596
 597         for (plugin, exception) in failures:
 598             print_download_error(plugin, exception)
 599
 600         sys.exit(1)
 601
 602 def make_repo(uri: str, branch) -> Repo:
 603     '''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
 604     # dumb check to see if it's of the form owner/repo (=> github) or https://...
 605     res = urlparse(uri)
 606     if res.netloc in [ "github.com", ""]:
 607         res = res.path.strip('/').split('/')
 608         repo = RepoGitHub(res[0], res[1], branch)
 609     else:
 610         repo = Repo(uri.strip(), branch)
 611     return repo
 612
 613
 614 def get_cache_path(cache_file_name: str) -> Optional[Path]:
 615     xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
 616     if xdg_cache is None:
 617         home = os.environ.get("HOME", None)
 618         if home is None:
 619             return None
 620         xdg_cache = str(Path(home, ".cache"))
 621
 622     return Path(xdg_cache, cache_file_name)
 623
 624
 625 class Cache:
 626     def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None:
 627         self.cache_file = get_cache_path(cache_file_name)
 628
 629         downloads = {}
 630         for plugin in initial_plugins:
 631             downloads[plugin.commit] = plugin
 632         downloads.update(self.load())
 633         self.downloads = downloads
 634
 635     def load(self) -> Dict[str, Plugin]:
 636         if self.cache_file is None or not self.cache_file.exists():
 637             return {}
 638
 639         downloads: Dict[str, Plugin] = {}
 640         with open(self.cache_file) as f:
 641             data = json.load(f)
 642             for attr in data.values():
 643                 p = Plugin(
 644                     attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"]
 645                 )
 646                 downloads[attr["commit"]] = p
 647         return downloads
 648
 649     def store(self) -> None:
 650         if self.cache_file is None:
 651             return
 652
 653         os.makedirs(self.cache_file.parent, exist_ok=True)
 654         with open(self.cache_file, "w+") as f:
 655             data = {}
 656             for name, attr in self.downloads.items():
 657                 data[name] = attr.as_json()
 658             json.dump(data, f, indent=4, sort_keys=True)
 659
 660     def __getitem__(self, key: str) -> Optional[Plugin]:
 661         return self.downloads.get(key, None)
 662
 663     def __setitem__(self, key: str, value: Plugin) -> None:
 664         self.downloads[key] = value
 665
 666
 667 def prefetch(
 668     pluginDesc: PluginDesc, cache: Cache
 669 ) -> Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]:
 670     try:
 671         plugin, redirect = prefetch_plugin(pluginDesc, cache)
 672         cache[plugin.commit] = plugin
 673         return (pluginDesc, plugin, redirect)
 674     except Exception as e:
 675         return (pluginDesc, e, None)
 676
 677
 678
 679 def rewrite_input(
 680     config: FetchConfig,
 681     input_file: Path,
 682     deprecated: Path,
 683     # old pluginDesc and the new
 684     redirects: Redirects = {},
 685     append: List[PluginDesc] = [],
 686 ):
 687     plugins = load_plugins_from_csv(config, input_file,)
 688
 689     plugins.extend(append)
 690
 691     if redirects:
 692
 693         cur_date_iso = datetime.now().strftime("%Y-%m-%d")
 694         with open(deprecated, "r") as f:
 695             deprecations = json.load(f)
 696         for pdesc, new_repo in redirects.items():
 697             new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)
 698             old_plugin, _ = prefetch_plugin(pdesc)
 699             new_plugin, _ = prefetch_plugin(new_pdesc)
 700             if old_plugin.normalized_name != new_plugin.normalized_name:
 701                 deprecations[old_plugin.normalized_name] = {
 702                     "new": new_plugin.normalized_name,
 703                     "date": cur_date_iso,
 704                 }
 705         with open(deprecated, "w") as f:
 706             json.dump(deprecations, f, indent=4, sort_keys=True)
 707             f.write("\n")
 708
 709     with open(input_file, "w") as f:
 710         log.debug("Writing into %s", input_file)
 711         # fields = dataclasses.fields(PluginDesc)
 712         fieldnames = ['repo', 'branch', 'alias']
 713         writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
 714         writer.writeheader()
 715         for plugin in sorted(plugins):
 716             writer.writerow(asdict(plugin))
 717
 718
 719 def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
 720     repo.index.add([str(f.resolve()) for f in files])
 721
 722     if repo.index.diff("HEAD"):
 723         print(f'committing to nixpkgs "{message}"')
 724         repo.index.commit(message)
 725     else:
 726         print("no changes in working tree to commit")
 727
 728
 729
 730 def update_plugins(editor: Editor, args):
 731     """The main entry function of this module. All input arguments are grouped in the `Editor`."""
 732
 733     log.info("Start updating plugins")
 734     fetch_config = FetchConfig(args.proc, args.github_token)
 735     update = editor.get_update(args.input_file, args.outfile, fetch_config)
 736
 737     redirects = update()
 738     editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, redirects)
 739
 740     autocommit = not args.no_commit
 741
 742     if autocommit:
 743         editor.nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
 744         commit(editor.nixpkgs_repo, f"{editor.attr_path}: update", [args.outfile])
 745
 746     if redirects:
 747         update()
 748         if autocommit:
 749             commit(
 750                 editor.nixpkgs_repo,
 751                 f"{editor.attr_path}: resolve github repository redirects",
 752                 [args.outfile, args.input_file, editor.deprecated],
 753             )
 754