pkgs/servers/web-apps/discourse/update.py

   1 #!/usr/bin/env nix-shell
   2 #! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests click click-log packaging ])" bundix bundler nix-update nurl prefetch-yarn-deps
   3 from __future__ import annotations
   4
   5 import click
   6 import click_log
   7 import shutil
   8 import tempfile
   9 import re
  10 import logging
  11 import subprocess
  12 import os
  13 import stat
  14 import json
  15 import requests
  16 import textwrap
  17 from functools import total_ordering
  18 from packaging.version import Version
  19 from pathlib import Path
  20 from typing import Union, Iterable
  21
  22
  23 logger = logging.getLogger(__name__)
  24
  25
  26 @total_ordering
  27 class DiscourseVersion:
  28     """Represents a Discourse style version number and git tag.
  29
  30     This takes either a tag or version string as input and
  31     extrapolates the other. Sorting is implemented to work as expected
  32     in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is
  33     considered lower than 2.0.0.
  34
  35     """
  36
  37     tag: str = ""
  38     version: str = ""
  39     split_version: Iterable[Union[None, int, str]] = []
  40
  41     def __init__(self, version: str):
  42         """Take either a tag or version number, calculate the other."""
  43         if version.startswith('v'):
  44             self.tag = version
  45             self.version = version.lstrip('v')
  46         else:
  47             self.tag = 'v' + version
  48             self.version = version
  49
  50         self._version = Version(self.version)
  51
  52     def __eq__(self, other: DiscourseVersion):
  53         """Versions are equal when their individual parts are."""
  54         return self._version == other._version
  55
  56     def __gt__(self, other: DiscourseVersion):
  57         """Check if this version is greater than the other."""
  58         return self._version > other._version
  59
  60
  61 class DiscourseRepo:
  62     version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$')
  63     _latest_commit_sha = None
  64
  65     def __init__(self, owner: str = 'discourse', repo: str = 'discourse'):
  66         self.owner = owner
  67         self.repo = repo
  68
  69     @property
  70     def versions(self) -> Iterable[str]:
  71         r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json()
  72         tags = [x['ref'].replace('refs/tags/', '') for x in r]
  73
  74         # filter out versions not matching version_regex
  75         versions = filter(self.version_regex.match, tags)
  76         versions = [DiscourseVersion(x) for x in versions]
  77         versions.sort(reverse=True)
  78         return versions
  79
  80     @property
  81     def latest_commit_sha(self) -> str:
  82         if self._latest_commit_sha is None:
  83             r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/commits?per_page=1')
  84             r.raise_for_status()
  85             self._latest_commit_sha = r.json()[0]['sha']
  86
  87         return self._latest_commit_sha
  88
  89     def get_yarn_lock_hash(self, rev: str, path: str):
  90         yarnLockText = self.get_file(path, rev)
  91         with tempfile.NamedTemporaryFile(mode='w') as lockFile:
  92             lockFile.write(yarnLockText)
  93             hash = subprocess.check_output(['prefetch-yarn-deps', lockFile.name]).decode().strip()
  94             return subprocess.check_output(["nix", "hash", "to-sri", "--type", "sha256", hash]).decode().strip()
  95
  96     def get_file(self, filepath, rev):
  97         """Return file contents at a given rev.
  98
  99         :param str filepath: the path to the file, relative to the repo root
 100         :param str rev: the rev to fetch at :return:
 101
 102         """
 103         r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}')
 104         r.raise_for_status()
 105         return r.text
 106
 107
 108 def _call_nix_update(pkg, version):
 109     """Call nix-update from nixpkgs root dir."""
 110     nixpkgs_path = Path(__file__).parent / '../../../../'
 111     return subprocess.check_output(['nix-update', pkg, '--version', version], cwd=nixpkgs_path)
 112
 113
 114 def _nix_eval(expr: str):
 115     nixpkgs_path = Path(__file__).parent / '../../../../'
 116     try:
 117         output = subprocess.check_output(['nix-instantiate', '--strict', '--json', '--eval', '-E', f'(with import {nixpkgs_path} {{}}; {expr})'], text=True)
 118     except subprocess.CalledProcessError:
 119         return None
 120     return json.loads(output)
 121
 122
 123 def _get_current_package_version(pkg: str):
 124     return _nix_eval(f'{pkg}.version')
 125
 126
 127 def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion):
 128     repo = DiscourseRepo()
 129
 130     current_dir = Path(__file__).parent
 131
 132     old = repo.get_file(filepath, old_version.tag)
 133     new = repo.get_file(filepath, new_version.tag)
 134
 135     if old == new:
 136         click.secho(f'{filepath} is unchanged', fg='green')
 137         return
 138
 139     with tempfile.NamedTemporaryFile(mode='w') as o, tempfile.NamedTemporaryFile(mode='w') as n:
 140         o.write(old), n.write(new)
 141         width = shutil.get_terminal_size((80, 20)).columns
 142         diff_proc = subprocess.run(
 143             ['diff', '--color=always', f'--width={width}', '-y', o.name, n.name],
 144             stdout=subprocess.PIPE,
 145             cwd=current_dir,
 146             text=True
 147         )
 148
 149     click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True)
 150     click.echo(diff_proc.stdout + '\n')
 151     return
 152
 153
 154 def _remove_platforms(rubyenv_dir: Path):
 155     for platform in ['arm64-darwin-20', 'x86_64-darwin-18',
 156                      'x86_64-darwin-19', 'x86_64-darwin-20',
 157                      'x86_64-linux', 'aarch64-linux']:
 158         with open(rubyenv_dir / 'Gemfile.lock', 'r') as f:
 159             for line in f:
 160                 if platform in line:
 161                     subprocess.check_output(
 162                         ['bundle', 'lock', '--remove-platform', platform], cwd=rubyenv_dir)
 163                     break
 164
 165
 166 @click_log.simple_verbosity_option(logger)
 167
 168
 169 @click.group()
 170 def cli():
 171     pass
 172
 173
 174 @cli.command()
 175 @click.argument('rev', default='latest')
 176 @click.option('--reverse/--no-reverse', default=False, help='Print diffs from REV to current.')
 177 def print_diffs(rev, reverse):
 178     """Print out diffs for files used as templates for the NixOS module.
 179
 180     The current package version found in the nixpkgs worktree the
 181     script is run from will be used to download the "from" file and
 182     REV used to download the "to" file for the diff, unless the
 183     '--reverse' flag is specified.
 184
 185     REV should be the git rev to find changes in ('vX.Y.Z') or
 186     'latest'; defaults to 'latest'.
 187
 188     """
 189     if rev == 'latest':
 190         repo = DiscourseRepo()
 191         rev = repo.versions[0].tag
 192
 193     old_version = DiscourseVersion(_get_current_package_version('discourse'))
 194     new_version = DiscourseVersion(rev)
 195
 196     if reverse:
 197         old_version, new_version = new_version, old_version
 198
 199     for f in ['config/nginx.sample.conf', 'config/discourse_defaults.conf']:
 200         _diff_file(f, old_version, new_version)
 201
 202
 203 @cli.command()
 204 @click.argument('rev', default='latest')
 205 def update(rev):
 206     """Update gem files and version.
 207
 208     REV: the git rev to update to ('vX.Y.Z[.betaA]') or
 209     'latest'; defaults to 'latest'.
 210
 211     """
 212     repo = DiscourseRepo()
 213
 214     if rev == 'latest':
 215         version = repo.versions[0]
 216     else:
 217         version = DiscourseVersion(rev)
 218
 219     logger.debug(f"Using rev {version.tag}")
 220     logger.debug(f"Using version {version.version}")
 221
 222     rubyenv_dir = Path(__file__).parent / "rubyEnv"
 223
 224     for fn in ['Gemfile.lock', 'Gemfile']:
 225         with open(rubyenv_dir / fn, 'w') as f:
 226             f.write(repo.get_file(fn, version.tag))
 227
 228     # work around https://github.com/nix-community/bundix/issues/8
 229     os.environ["BUNDLE_FORCE_RUBY_PLATFORM"] = "true"
 230     subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir)
 231     _remove_platforms(rubyenv_dir)
 232     subprocess.check_output(['bundix'], cwd=rubyenv_dir)
 233
 234     _call_nix_update('discourse', version.version)
 235
 236     old_yarn_hash = _nix_eval('discourse.assets.yarnOfflineCache.outputHash')
 237     new_yarn_hash = repo.get_yarn_lock_hash(version.tag, "app/assets/javascripts/yarn-ember5.lock")
 238     click.echo(f"Updating yarn lock hash: {old_yarn_hash} -> {new_yarn_hash}")
 239
 240     old_yarn_dev_hash = _nix_eval('discourse.assets.yarnDevOfflineCache.outputHash')
 241     new_yarn_dev_hash = repo.get_yarn_lock_hash(version.tag, "yarn.lock")
 242     click.echo(f"Updating yarn dev lock hash: {old_yarn_dev_hash} -> {new_yarn_dev_hash}")
 243
 244     with open(Path(__file__).parent / "default.nix", 'r+') as f:
 245         content = f.read()
 246         content = content.replace(old_yarn_hash, new_yarn_hash)
 247         content = content.replace(old_yarn_dev_hash, new_yarn_dev_hash)
 248         f.seek(0)
 249         f.write(content)
 250         f.truncate()
 251
 252
 253 @cli.command()
 254 @click.argument('rev', default='latest')
 255 def update_mail_receiver(rev):
 256     """Update discourse-mail-receiver.
 257
 258     REV: the git rev to update to ('vX.Y.Z') or 'latest'; defaults to
 259     'latest'.
 260
 261     """
 262     repo = DiscourseRepo(repo="mail-receiver")
 263
 264     if rev == 'latest':
 265         version = repo.versions[0]
 266     else:
 267         version = DiscourseVersion(rev)
 268
 269     _call_nix_update('discourse-mail-receiver', version.version)
 270
 271
 272 @cli.command()
 273 def update_plugins():
 274     """Update plugins to their latest revision."""
 275     plugins = [
 276         {'name': 'discourse-assign'},
 277         {'name': 'discourse-bbcode-color'},
 278         {'name': 'discourse-calendar'},
 279         {'name': 'discourse-canned-replies'},
 280         {'name': 'discourse-chat-integration'},
 281         {'name': 'discourse-checklist'},
 282         {'name': 'discourse-data-explorer'},
 283         {'name': 'discourse-docs'},
 284         {'name': 'discourse-github'},
 285         {'name': 'discourse-ldap-auth', 'owner': 'jonmbake'},
 286         {'name': 'discourse-math'},
 287         {'name': 'discourse-migratepassword', 'owner': 'discoursehosting'},
 288         {'name': 'discourse-openid-connect'},
 289         {'name': 'discourse-prometheus'},
 290         {'name': 'discourse-reactions'},
 291         {'name': 'discourse-saved-searches'},
 292         {'name': 'discourse-solved'},
 293         {'name': 'discourse-spoiler-alert'},
 294         {'name': 'discourse-voting'},
 295         {'name': 'discourse-yearly-review'},
 296     ]
 297
 298     for plugin in plugins:
 299         fetcher = plugin.get('fetcher') or "fetchFromGitHub"
 300         owner = plugin.get('owner') or "discourse"
 301         name = plugin.get('name')
 302         repo_name = plugin.get('repo_name') or name
 303
 304         if fetcher == "fetchFromGitHub":
 305             url = f"https://github.com/{owner}/{repo_name}"
 306         else:
 307             raise NotImplementedError(f"Missing URL pattern for {fetcher}")
 308
 309         repo = DiscourseRepo(owner=owner, repo=repo_name)
 310
 311         # implement the plugin pinning algorithm laid out here:
 312         # https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
 313         # this makes sure we don't upgrade plugins to revisions that
 314         # are incompatible with the packaged Discourse version
 315         try:
 316             compatibility_spec = repo.get_file('.discourse-compatibility', repo.latest_commit_sha)
 317             versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' '))
 318                         for [discourse_version, plugin_rev]
 319                         in [line.lstrip("< ").split(':')
 320                             for line
 321                             in compatibility_spec.splitlines() if line != '']]
 322             discourse_version = DiscourseVersion(_get_current_package_version('discourse'))
 323             versions = list(filter(lambda ver: ver[0] >= discourse_version, versions))
 324             if versions == []:
 325                 rev = repo.latest_commit_sha
 326             else:
 327                 rev = versions[0][1]
 328                 print(rev)
 329         except requests.exceptions.HTTPError:
 330             rev = repo.latest_commit_sha
 331
 332         filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}')
 333         if filename is None:
 334             filename = Path(__file__).parent / 'plugins' / name / 'default.nix'
 335             filename.parent.mkdir()
 336
 337             has_ruby_deps = False
 338             for line in repo.get_file('plugin.rb', rev).splitlines():
 339                 if 'gem ' in line:
 340                     has_ruby_deps = True
 341                     break
 342
 343             with open(filename, 'w') as f:
 344                 f.write(textwrap.dedent(f"""
 345                          {{ lib, mkDiscoursePlugin, fetchFromGitHub }}:
 346
 347                          mkDiscoursePlugin {{
 348                            name = "{name}";"""[1:] + ("""
 349                            bundlerEnvArgs.gemdir = ./.;""" if has_ruby_deps else "") + f"""
 350                            src = {fetcher} {{
 351                              owner = "{owner}";
 352                              repo = "{repo_name}";
 353                              rev = "replace-with-git-rev";
 354                              sha256 = "replace-with-sha256";
 355                            }};
 356                            meta = with lib; {{
 357                              homepage = "";
 358                              maintainers = with maintainers; [ ];
 359                              license = licenses.mit; # change to the correct license!
 360                              description = "";
 361                            }};
 362                          }}"""))
 363
 364             all_plugins_filename = Path(__file__).parent / 'plugins' / 'all-plugins.nix'
 365             with open(all_plugins_filename, 'r+') as f:
 366                 content = f.read()
 367                 pos = -1
 368                 while content[pos] != '}':
 369                     pos -= 1
 370                 content = content[:pos] + f'  {name} = callPackage ./{name} {{}};' + os.linesep + content[pos:]
 371                 f.seek(0)
 372                 f.write(content)
 373                 f.truncate()
 374
 375         else:
 376             filename = filename['file']
 377
 378         prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev')
 379
 380         if prev_commit_sha == rev:
 381             click.echo(f'Plugin {name} is already at the latest revision')
 382             continue
 383
 384         prev_hash = _nix_eval(f'discourse.plugins.{name}.src.outputHash')
 385         new_hash = subprocess.check_output([
 386             "nurl",
 387             "--fetcher", fetcher,
 388             "--hash",
 389             url,
 390             rev,
 391         ], text=True).strip("\n")
 392
 393         click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}")
 394
 395         with open(filename, 'r+') as f:
 396             content = f.read()
 397             content = content.replace(prev_commit_sha, rev)
 398             content = content.replace(prev_hash, new_hash)
 399             f.seek(0)
 400             f.write(content)
 401             f.truncate()
 402
 403         rubyenv_dir = Path(filename).parent
 404         gemfile = rubyenv_dir / "Gemfile"
 405         version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)')
 406         gemfile_text = ''
 407         plugin_file = repo.get_file('plugin.rb', rev)
 408         plugin_file = plugin_file.replace(",\n", ", ") # fix split lines
 409         for line in plugin_file.splitlines():
 410             if 'gem ' in line:
 411                 line = ','.join(filter(lambda x: ":require_name" not in x, line.split(',')))
 412                 gemfile_text = gemfile_text + line + os.linesep
 413
 414                 version_file_match = version_file_regex.match(line)
 415                 if version_file_match is not None:
 416                     filename = version_file_match.groups()[0]
 417                     content = repo.get_file(filename, rev)
 418                     with open(rubyenv_dir / filename, 'w') as f:
 419                         f.write(content)
 420
 421         if len(gemfile_text) > 0:
 422             if os.path.isfile(gemfile):
 423                 os.remove(gemfile)
 424
 425             subprocess.check_output(['bundle', 'init'], cwd=rubyenv_dir)
 426             os.chmod(gemfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
 427
 428             with open(gemfile, 'a') as f:
 429                 f.write(gemfile_text)
 430
 431             subprocess.check_output(['bundle', 'lock', '--add-platform', 'ruby'], cwd=rubyenv_dir)
 432             subprocess.check_output(['bundle', 'lock', '--update'], cwd=rubyenv_dir)
 433             _remove_platforms(rubyenv_dir)
 434             subprocess.check_output(['bundix'], cwd=rubyenv_dir)
 435
 436
 437 if __name__ == '__main__':
 438     cli()