pkgs/servers/web-apps/discourse/update.py

   1 #!/usr/bin/env nix-shell
   2 #! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests click click-log packaging ])" bundix bundler nix-update nurl prefetch-yarn-deps
   3 from __future__ import annotations
   4
   5 import click
   6 import click_log
   7 import shutil
   8 import tempfile
   9 import re
  10 import logging
  11 import subprocess
  12 import os
  13 import stat
  14 import json
  15 import requests
  16 import textwrap
  17 from functools import total_ordering
  18 from packaging.version import Version
  19 from pathlib import Path
  20 from typing import Union, Iterable
  21
  22
  23 logger = logging.getLogger(__name__)
  24
  25
  26 @total_ordering
  27 class DiscourseVersion:
  28     """Represents a Discourse style version number and git tag.
  29
  30     This takes either a tag or version string as input and
  31     extrapolates the other. Sorting is implemented to work as expected
  32     in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is
  33     considered lower than 2.0.0.
  34
  35     """
  36
  37     tag: str = ""
  38     version: str = ""
  39     split_version: Iterable[Union[None, int, str]] = []
  40
  41     def __init__(self, version: str):
  42         """Take either a tag or version number, calculate the other."""
  43         if version.startswith('v'):
  44             self.tag = version
  45             self.version = version.lstrip('v')
  46         else:
  47             self.tag = 'v' + version
  48             self.version = version
  49
  50         self._version = Version(self.version)
  51
  52     def __eq__(self, other: DiscourseVersion):
  53         """Versions are equal when their individual parts are."""
  54         return self._version == other._version
  55
  56     def __gt__(self, other: DiscourseVersion):
  57         """Check if this version is greater than the other."""
  58         return self._version > other._version
  59
  60
  61 class DiscourseRepo:
  62     version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$')
  63     _latest_commit_sha = None
  64
  65     def __init__(self, owner: str = 'discourse', repo: str = 'discourse'):
  66         self.owner = owner
  67         self.repo = repo
  68
  69     @property
  70     def versions(self) -> Iterable[str]:
  71         r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json()
  72         tags = [x['ref'].replace('refs/tags/', '') for x in r]
  73
  74         # filter out versions not matching version_regex
  75         versions = filter(self.version_regex.match, tags)
  76         versions = [DiscourseVersion(x) for x in versions]
  77         versions.sort(reverse=True)
  78         return versions
  79
  80     @property
  81     def latest_commit_sha(self) -> str:
  82         if self._latest_commit_sha is None:
  83             r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/commits?per_page=1')
  84             r.raise_for_status()
  85             self._latest_commit_sha = r.json()[0]['sha']
  86
  87         return self._latest_commit_sha
  88
  89     def get_yarn_lock_hash(self, rev: str, path: str):
  90         yarnLockText = self.get_file(path, rev)
  91         with tempfile.NamedTemporaryFile(mode='w') as lockFile:
  92             lockFile.write(yarnLockText)
  93             hash = subprocess.check_output(['prefetch-yarn-deps', lockFile.name]).decode().strip()
  94             return subprocess.check_output(["nix", "hash", "to-sri", "--type", "sha256", hash]).decode().strip()
  95
  96     def get_file(self, filepath, rev):
  97         """Return file contents at a given rev.
  98
  99         :param str filepath: the path to the file, relative to the repo root
 100         :param str rev: the rev to fetch at :return:
 101
 102         """
 103         r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}')
 104         r.raise_for_status()
 105         return r.text
 106
 107
 108 def _call_nix_update(pkg, version):
 109     """Call nix-update from nixpkgs root dir."""
 110     nixpkgs_path = Path(__file__).parent / '../../../../'
 111     return subprocess.check_output(['nix-update', pkg, '--version', version], cwd=nixpkgs_path)
 112
 113
 114 def _nix_eval(expr: str):
 115     nixpkgs_path = Path(__file__).parent / '../../../../'
 116     try:
 117         output = subprocess.check_output(['nix-instantiate', '--strict', '--json', '--eval', '-E', f'(with import {nixpkgs_path} {{}}; {expr})'], text=True)
 118     except subprocess.CalledProcessError:
 119         return None
 120     return json.loads(output)
 121
 122
 123 def _get_current_package_version(pkg: str):
 124     return _nix_eval(f'{pkg}.version')
 125
 126
 127 def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion):
 128     repo = DiscourseRepo()
 129
 130     current_dir = Path(__file__).parent
 131
 132     old = repo.get_file(filepath, old_version.tag)
 133     new = repo.get_file(filepath, new_version.tag)
 134
 135     if old == new:
 136         click.secho(f'{filepath} is unchanged', fg='green')
 137         return
 138
 139     with tempfile.NamedTemporaryFile(mode='w') as o, tempfile.NamedTemporaryFile(mode='w') as n:
 140         o.write(old), n.write(new)
 141         width = shutil.get_terminal_size((80, 20)).columns
 142         diff_proc = subprocess.run(
 143             ['diff', '--color=always', f'--width={width}', '-y', o.name, n.name],
 144             stdout=subprocess.PIPE,
 145             cwd=current_dir,
 146             text=True
 147         )
 148
 149     click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True)
 150     click.echo(diff_proc.stdout + '\n')
 151     return
 152
 153
 154 def _remove_platforms(rubyenv_dir: Path):
 155     for platform in ['arm64-darwin-20', 'x86_64-darwin-18',
 156                      'x86_64-darwin-19', 'x86_64-darwin-20',
 157                      'x86_64-linux', 'aarch64-linux']:
 158         with open(rubyenv_dir / 'Gemfile.lock', 'r') as f:
 159             for line in f:
 160                 if platform in line:
 161                     subprocess.check_output(
 162                         ['bundle', 'lock', '--remove-platform', platform], cwd=rubyenv_dir)
 163                     break
 164
 165
 166 @click_log.simple_verbosity_option(logger)
 167
 168
 169 @click.group()
 170 def cli():
 171     pass
 172
 173
 174 @cli.command()
 175 @click.argument('rev', default='latest')
 176 @click.option('--reverse/--no-reverse', default=False, help='Print diffs from REV to current.')
 177 def print_diffs(rev, reverse):
 178     """Print out diffs for files used as templates for the NixOS module.
 179
 180     The current package version found in the nixpkgs worktree the
 181     script is run from will be used to download the "from" file and
 182     REV used to download the "to" file for the diff, unless the
 183     '--reverse' flag is specified.
 184
 185     REV should be the git rev to find changes in ('vX.Y.Z') or
 186     'latest'; defaults to 'latest'.
 187
 188     """
 189     if rev == 'latest':
 190         repo = DiscourseRepo()
 191         rev = repo.versions[0].tag
 192
 193     old_version = DiscourseVersion(_get_current_package_version('discourse'))
 194     new_version = DiscourseVersion(rev)
 195
 196     if reverse:
 197         old_version, new_version = new_version, old_version
 198
 199     for f in ['config/nginx.sample.conf', 'config/discourse_defaults.conf']:
 200         _diff_file(f, old_version, new_version)
 201
 202
 203 @cli.command()
 204 @click.argument('rev', default='latest')
 205 def update(rev):
 206     """Update gem files and version.
 207
 208     REV: the git rev to update to ('vX.Y.Z[.betaA]') or
 209     'latest'; defaults to 'latest'.
 210
 211     """
 212     repo = DiscourseRepo()
 213
 214     if rev == 'latest':
 215         version = repo.versions[0]
 216     else:
 217         version = DiscourseVersion(rev)
 218
 219     logger.debug(f"Using rev {version.tag}")
 220     logger.debug(f"Using version {version.version}")
 221
 222     rubyenv_dir = Path(__file__).parent / "rubyEnv"
 223
 224     for fn in ['Gemfile.lock', 'Gemfile']:
 225         with open(rubyenv_dir / fn, 'w') as f:
 226             f.write(repo.get_file(fn, version.tag))
 227
 228     # work around https://github.com/nix-community/bundix/issues/8
 229     os.environ["BUNDLE_FORCE_RUBY_PLATFORM"] = "true"
 230     subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir)
 231     _remove_platforms(rubyenv_dir)
 232     subprocess.check_output(['bundix'], cwd=rubyenv_dir)
 233
 234     _call_nix_update('discourse', version.version)
 235
 236     old_yarn_hash = _nix_eval('discourse.assets.yarnOfflineCache.outputHash')
 237     new_yarn_hash = repo.get_yarn_lock_hash(version.tag, "yarn.lock")
 238     click.echo(f"Updating yarn lock hash: {old_yarn_hash} -> {new_yarn_hash}")
 239
 240     with open(Path(__file__).parent / "default.nix", 'r+') as f:
 241         content = f.read()
 242         content = content.replace(old_yarn_hash, new_yarn_hash)
 243         f.seek(0)
 244         f.write(content)
 245         f.truncate()
 246
 247
 248 @cli.command()
 249 @click.argument('rev', default='latest')
 250 def update_mail_receiver(rev):
 251     """Update discourse-mail-receiver.
 252
 253     REV: the git rev to update to ('vX.Y.Z') or 'latest'; defaults to
 254     'latest'.
 255
 256     """
 257     repo = DiscourseRepo(repo="mail-receiver")
 258
 259     if rev == 'latest':
 260         version = repo.versions[0]
 261     else:
 262         version = DiscourseVersion(rev)
 263
 264     _call_nix_update('discourse-mail-receiver', version.version)
 265
 266
 267 @cli.command()
 268 def update_plugins():
 269     """Update plugins to their latest revision."""
 270     plugins = [
 271         {'name': 'discourse-assign'},
 272         {'name': 'discourse-bbcode-color'},
 273         {'name': 'discourse-calendar'},
 274         {'name': 'discourse-canned-replies'},
 275         {'name': 'discourse-chat-integration'},
 276         {'name': 'discourse-checklist'},
 277         {'name': 'discourse-data-explorer'},
 278         {'name': 'discourse-docs'},
 279         {'name': 'discourse-github'},
 280         {'name': 'discourse-ldap-auth', 'owner': 'jonmbake'},
 281         {'name': 'discourse-math'},
 282         {'name': 'discourse-migratepassword', 'owner': 'discoursehosting'},
 283         {'name': 'discourse-openid-connect'},
 284         {'name': 'discourse-prometheus'},
 285         {'name': 'discourse-reactions'},
 286         {'name': 'discourse-saved-searches'},
 287         {'name': 'discourse-solved'},
 288         {'name': 'discourse-spoiler-alert'},
 289         {'name': 'discourse-voting'},
 290         {'name': 'discourse-yearly-review'},
 291     ]
 292
 293     for plugin in plugins:
 294         fetcher = plugin.get('fetcher') or "fetchFromGitHub"
 295         owner = plugin.get('owner') or "discourse"
 296         name = plugin.get('name')
 297         repo_name = plugin.get('repo_name') or name
 298
 299         if fetcher == "fetchFromGitHub":
 300             url = f"https://github.com/{owner}/{repo_name}"
 301         else:
 302             raise NotImplementedError(f"Missing URL pattern for {fetcher}")
 303
 304         repo = DiscourseRepo(owner=owner, repo=repo_name)
 305
 306         # implement the plugin pinning algorithm laid out here:
 307         # https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
 308         # this makes sure we don't upgrade plugins to revisions that
 309         # are incompatible with the packaged Discourse version
 310         try:
 311             compatibility_spec = repo.get_file('.discourse-compatibility', repo.latest_commit_sha)
 312             versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' '))
 313                         for [discourse_version, plugin_rev]
 314                         in [line.lstrip("< ").split(':')
 315                             for line
 316                             in compatibility_spec.splitlines() if line != '']]
 317             discourse_version = DiscourseVersion(_get_current_package_version('discourse'))
 318             versions = list(filter(lambda ver: ver[0] >= discourse_version, versions))
 319             if versions == []:
 320                 rev = repo.latest_commit_sha
 321             else:
 322                 rev = versions[0][1]
 323                 print(rev)
 324         except requests.exceptions.HTTPError:
 325             rev = repo.latest_commit_sha
 326
 327         filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}')
 328         if filename is None:
 329             filename = Path(__file__).parent / 'plugins' / name / 'default.nix'
 330             filename.parent.mkdir()
 331
 332             has_ruby_deps = False
 333             for line in repo.get_file('plugin.rb', rev).splitlines():
 334                 if 'gem ' in line:
 335                     has_ruby_deps = True
 336                     break
 337
 338             with open(filename, 'w') as f:
 339                 f.write(textwrap.dedent(f"""
 340                          {{ lib, mkDiscoursePlugin, fetchFromGitHub }}:
 341
 342                          mkDiscoursePlugin {{
 343                            name = "{name}";"""[1:] + ("""
 344                            bundlerEnvArgs.gemdir = ./.;""" if has_ruby_deps else "") + f"""
 345                            src = {fetcher} {{
 346                              owner = "{owner}";
 347                              repo = "{repo_name}";
 348                              rev = "replace-with-git-rev";
 349                              sha256 = "replace-with-sha256";
 350                            }};
 351                            meta = with lib; {{
 352                              homepage = "";
 353                              maintainers = with maintainers; [ ];
 354                              license = licenses.mit; # change to the correct license!
 355                              description = "";
 356                            }};
 357                          }}"""))
 358
 359             all_plugins_filename = Path(__file__).parent / 'plugins' / 'all-plugins.nix'
 360             with open(all_plugins_filename, 'r+') as f:
 361                 content = f.read()
 362                 pos = -1
 363                 while content[pos] != '}':
 364                     pos -= 1
 365                 content = content[:pos] + f'  {name} = callPackage ./{name} {{}};' + os.linesep + content[pos:]
 366                 f.seek(0)
 367                 f.write(content)
 368                 f.truncate()
 369
 370         else:
 371             filename = filename['file']
 372
 373         prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev')
 374
 375         if prev_commit_sha == rev:
 376             click.echo(f'Plugin {name} is already at the latest revision')
 377             continue
 378
 379         prev_hash = _nix_eval(f'discourse.plugins.{name}.src.outputHash')
 380         new_hash = subprocess.check_output([
 381             "nurl",
 382             "--fetcher", fetcher,
 383             "--hash",
 384             url,
 385             rev,
 386         ], text=True).strip("\n")
 387
 388         click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}")
 389
 390         with open(filename, 'r+') as f:
 391             content = f.read()
 392             content = content.replace(prev_commit_sha, rev)
 393             content = content.replace(prev_hash, new_hash)
 394             f.seek(0)
 395             f.write(content)
 396             f.truncate()
 397
 398         rubyenv_dir = Path(filename).parent
 399         gemfile = rubyenv_dir / "Gemfile"
 400         version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)')
 401         gemfile_text = ''
 402         plugin_file = repo.get_file('plugin.rb', rev)
 403         plugin_file = plugin_file.replace(",\n", ", ") # fix split lines
 404         for line in plugin_file.splitlines():
 405             if 'gem ' in line:
 406                 line = ','.join(filter(lambda x: ":require_name" not in x, line.split(',')))
 407                 gemfile_text = gemfile_text + line + os.linesep
 408
 409                 version_file_match = version_file_regex.match(line)
 410                 if version_file_match is not None:
 411                     filename = version_file_match.groups()[0]
 412                     content = repo.get_file(filename, rev)
 413                     with open(rubyenv_dir / filename, 'w') as f:
 414                         f.write(content)
 415
 416         if len(gemfile_text) > 0:
 417             if os.path.isfile(gemfile):
 418                 os.remove(gemfile)
 419
 420             subprocess.check_output(['bundle', 'init'], cwd=rubyenv_dir)
 421             os.chmod(gemfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
 422
 423             with open(gemfile, 'a') as f:
 424                 f.write(gemfile_text)
 425
 426             subprocess.check_output(['bundle', 'lock', '--add-platform', 'ruby'], cwd=rubyenv_dir)
 427             subprocess.check_output(['bundle', 'lock', '--update'], cwd=rubyenv_dir)
 428             _remove_platforms(rubyenv_dir)
 429             subprocess.check_output(['bundix'], cwd=rubyenv_dir)
 430
 431
 432 if __name__ == '__main__':
 433     cli()