pkgs/applications/version-management/gitlab/update.py

   1 #!/usr/bin/env nix-shell
   2 #! nix-shell -I nixpkgs=../../../.. -i python3 -p bundix bundler nix-update nix python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.packaging prefetch-yarn-deps git
   3
   4 import click
   5 import click_log
   6 import re
   7 import logging
   8 import subprocess
   9 import json
  10 import pathlib
  11 import tempfile
  12 from packaging.version import Version
  13 from typing import Iterable
  14
  15 import requests
  16
  17 NIXPKGS_PATH = pathlib.Path(__file__).parent / "../../../../"
  18 GITLAB_DIR = pathlib.Path(__file__).parent
  19
  20 logger = logging.getLogger(__name__)
  21 click_log.basic_config(logger)
  22
  23
  24 class GitLabRepo:
  25     version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?(\-gitlab)?")
  26
  27     def __init__(self, owner: str = "gitlab-org", repo: str = "gitlab"):
  28         self.owner = owner
  29         self.repo = repo
  30
  31     @property
  32     def url(self):
  33         return f"https://gitlab.com/{self.owner}/{self.repo}"
  34
  35     @property
  36     def tags(self) -> Iterable[str]:
  37         """Returns a sorted list of repository tags"""
  38         r = requests.get(self.url + "/refs?sort=updated_desc&ref=master").json()
  39         tags = r.get("Tags", [])
  40
  41         # filter out versions not matching version_regex
  42         versions = list(filter(self.version_regex.match, tags))
  43
  44         # sort, but ignore v, -ee and -gitlab for sorting comparisons
  45         versions.sort(
  46             key=lambda x: Version(
  47                 x.replace("v", "").replace("-ee", "").replace("-gitlab", "")
  48             ),
  49             reverse=True,
  50         )
  51         return versions
  52     def get_git_hash(self, rev: str):
  53         return (
  54             subprocess.check_output(
  55                 [
  56                     "nix-prefetch-url",
  57                     "--unpack",
  58                     f"https://gitlab.com/{self.owner}/{self.repo}/-/archive/{rev}/{self.repo}-{rev}.tar.gz",
  59                 ]
  60             )
  61             .decode("utf-8")
  62             .strip()
  63         )
  64
  65     def get_yarn_hash(self, rev: str):
  66         with tempfile.TemporaryDirectory() as tmp_dir:
  67             with open(tmp_dir + "/yarn.lock", "w") as f:
  68                 f.write(self.get_file("yarn.lock", rev))
  69             return (
  70                 subprocess.check_output(["prefetch-yarn-deps", tmp_dir + "/yarn.lock"])
  71                 .decode("utf-8")
  72                 .strip()
  73             )
  74
  75     @staticmethod
  76     def rev2version(tag: str) -> str:
  77         """
  78         normalize a tag to a version number.
  79         This obviously isn't very smart if we don't pass something that looks like a tag
  80         :param tag: the tag to normalize
  81         :return: a normalized version number
  82         """
  83         # strip v prefix
  84         version = re.sub(r"^v", "", tag)
  85         # strip -ee and -gitlab suffixes
  86         return re.sub(r"-(ee|gitlab)$", "", version)
  87
  88     def get_file(self, filepath, rev):
  89         """
  90         returns file contents at a given rev
  91         :param filepath: the path to the file, relative to the repo root
  92         :param rev: the rev to fetch at
  93         :return:
  94         """
  95         return requests.get(self.url + f"/raw/{rev}/{filepath}").text
  96
  97     def get_data(self, rev):
  98         version = self.rev2version(rev)
  99
 100         passthru = {
 101             v: self.get_file(v, rev).strip()
 102             for v in [
 103                 "GITALY_SERVER_VERSION",
 104                 "GITLAB_PAGES_VERSION",
 105                 "GITLAB_SHELL_VERSION",
 106                 "GITLAB_ELASTICSEARCH_INDEXER_VERSION",
 107             ]
 108         }
 109         passthru["GITLAB_WORKHORSE_VERSION"] = version
 110
 111         return dict(
 112             version=self.rev2version(rev),
 113             repo_hash=self.get_git_hash(rev),
 114             yarn_hash=self.get_yarn_hash(rev),
 115             owner=self.owner,
 116             repo=self.repo,
 117             rev=rev,
 118             passthru=passthru,
 119         )
 120
 121
 122 def _get_data_json():
 123     data_file_path = pathlib.Path(__file__).parent / "data.json"
 124     with open(data_file_path, "r") as f:
 125         return json.load(f)
 126
 127
 128 def _call_nix_update(pkg, version):
 129     """calls nix-update from nixpkgs root dir"""
 130     return subprocess.check_output(
 131         ["nix-update", pkg, "--version", version], cwd=NIXPKGS_PATH
 132     )
 133
 134
 135 @click_log.simple_verbosity_option(logger)
 136 @click.group()
 137 def cli():
 138     pass
 139
 140
 141 @cli.command("update-data")
 142 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
 143 def update_data(rev: str):
 144     """Update data.json"""
 145     logger.info("Updating data.json")
 146
 147     repo = GitLabRepo()
 148     if rev == "latest":
 149         # filter out pre and rc releases
 150         rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
 151
 152     data_file_path = pathlib.Path(__file__).parent / "data.json"
 153
 154     data = repo.get_data(rev)
 155
 156     with open(data_file_path.as_posix(), "w") as f:
 157         json.dump(data, f, indent=2)
 158         f.write("\n")
 159
 160
 161 @cli.command("update-rubyenv")
 162 def update_rubyenv():
 163     """Update rubyEnv"""
 164     logger.info("Updating gitlab")
 165     repo = GitLabRepo()
 166     rubyenv_dir = pathlib.Path(__file__).parent / "rubyEnv"
 167
 168     # load rev from data.json
 169     data = _get_data_json()
 170     rev = data["rev"]
 171     version = data["version"]
 172
 173     for fn in ["Gemfile.lock", "Gemfile"]:
 174         with open(rubyenv_dir / fn, "w") as f:
 175             f.write(repo.get_file(fn, rev))
 176
 177     # patch for openssl 3.x support
 178     subprocess.check_output(
 179         ["sed", "-i", "s:'openssl', '2.*':'openssl', '3.0.2':g", "Gemfile"],
 180         cwd=rubyenv_dir,
 181     )
 182
 183     # Un-vendor sidekiq
 184     #
 185     # The sidekiq dependency was vendored to maintain compatibility with Redis 6.0 (as
 186     # stated in this [comment]) but unfortunately, it seems to cause a crash in the
 187     # application, as noted in this [upstream issue].
 188     #
 189     # We can safely swap out the dependency, as our Redis release in nixpkgs is >= 7.0.
 190     #
 191     # [comment]: https://gitlab.com/gitlab-org/gitlab/-/issues/468435#note_1979750600
 192     # [upstream issue]: https://gitlab.com/gitlab-org/gitlab/-/issues/468435
 193     subprocess.check_output(
 194         ["sed", "-i", "s|gem 'sidekiq', path: 'vendor/gems/sidekiq-7.1.6', require: 'sidekiq'|gem 'sidekiq', '~> 7.1.6'|g", "Gemfile"],
 195         cwd=rubyenv_dir,
 196     )
 197
 198     # Fetch vendored dependencies temporarily in order to build the gemset.nix
 199     subprocess.check_output(["mkdir", "-p", "vendor/gems", "gems"], cwd=rubyenv_dir)
 200     subprocess.check_output(
 201         [
 202             "sh",
 203             "-c",
 204             f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=vendor/gems | tar -xj --strip-components=3",
 205         ],
 206         cwd=f"{rubyenv_dir}/vendor/gems",
 207     )
 208     subprocess.check_output(
 209         [
 210             "sh",
 211             "-c",
 212             f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=gems | tar -xj --strip-components=2",
 213         ],
 214         cwd=f"{rubyenv_dir}/gems",
 215     )
 216
 217     # Undo our gemset.nix patches so that bundix runs through
 218     subprocess.check_output(
 219         ["sed", "-i", "-e", "1d", "-e", "s:\\${src}/::g", "gemset.nix"], cwd=rubyenv_dir
 220     )
 221
 222     subprocess.check_output(["bundle", "lock"], cwd=rubyenv_dir)
 223     subprocess.check_output(["bundix"], cwd=rubyenv_dir)
 224
 225     subprocess.check_output(
 226         [
 227             "sed",
 228             "-i",
 229             "-e",
 230             "1i\\src:",
 231             "-e",
 232             's:path = \\(vendor/[^;]*\\);:path = "${src}/\\1";:g',
 233             "-e",
 234             's:path = \\(gems/[^;]*\\);:path = "${src}/\\1";:g',
 235             "gemset.nix",
 236         ],
 237         cwd=rubyenv_dir,
 238     )
 239     subprocess.check_output(["rm", "-rf", "vendor", "gems"], cwd=rubyenv_dir)
 240
 241
 242 @cli.command("update-gitaly")
 243 def update_gitaly():
 244     """Update gitaly"""
 245     logger.info("Updating gitaly")
 246     data = _get_data_json()
 247     gitaly_server_version = data['passthru']['GITALY_SERVER_VERSION']
 248     repo = GitLabRepo(repo="gitaly")
 249     gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'
 250
 251     makefile = repo.get_file("Makefile", f"v{gitaly_server_version}")
 252     makefile += "\nprint-%:;@echo $($*)\n"
 253
 254     git_version = subprocess.run(["make", "-f", "-", "print-GIT_VERSION"], check=True, input=makefile, text=True, capture_output=True).stdout.strip()
 255
 256     _call_nix_update("gitaly", gitaly_server_version)
 257     _call_nix_update("gitaly.git", git_version)
 258
 259
 260 @cli.command("update-gitlab-pages")
 261 def update_gitlab_pages():
 262     """Update gitlab-pages"""
 263     logger.info("Updating gitlab-pages")
 264     data = _get_data_json()
 265     gitlab_pages_version = data["passthru"]["GITLAB_PAGES_VERSION"]
 266     _call_nix_update("gitlab-pages", gitlab_pages_version)
 267
 268
 269 def get_container_registry_version() -> str:
 270     """Returns the version attribute of gitlab-container-registry"""
 271     return subprocess.check_output(
 272         [
 273             "nix",
 274             "--experimental-features",
 275             "nix-command",
 276             "eval",
 277             "-f",
 278             ".",
 279             "--raw",
 280             "gitlab-container-registry.version",
 281         ],
 282         cwd=NIXPKGS_PATH,
 283     ).decode("utf-8")
 284
 285
 286 @cli.command("update-gitlab-shell")
 287 def update_gitlab_shell():
 288     """Update gitlab-shell"""
 289     logger.info("Updating gitlab-shell")
 290     data = _get_data_json()
 291     gitlab_shell_version = data["passthru"]["GITLAB_SHELL_VERSION"]
 292     _call_nix_update("gitlab-shell", gitlab_shell_version)
 293
 294
 295 @cli.command("update-gitlab-workhorse")
 296 def update_gitlab_workhorse():
 297     """Update gitlab-workhorse"""
 298     logger.info("Updating gitlab-workhorse")
 299     data = _get_data_json()
 300     gitlab_workhorse_version = data["passthru"]["GITLAB_WORKHORSE_VERSION"]
 301     _call_nix_update("gitlab-workhorse", gitlab_workhorse_version)
 302
 303
 304 @cli.command("update-gitlab-container-registry")
 305 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
 306 @click.option(
 307     "--commit", is_flag=True, default=False, help="Commit the changes for you"
 308 )
 309 def update_gitlab_container_registry(rev: str, commit: bool):
 310     """Update gitlab-container-registry"""
 311     logger.info("Updading gitlab-container-registry")
 312     repo = GitLabRepo(repo="container-registry")
 313     old_container_registry_version = get_container_registry_version()
 314
 315     if rev == "latest":
 316         rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
 317
 318     version = repo.rev2version(rev)
 319     _call_nix_update("gitlab-container-registry", version)
 320     if commit:
 321         new_container_registry_version = get_container_registry_version()
 322         commit_container_registry(
 323             old_container_registry_version, new_container_registry_version
 324         )
 325
 326
 327 @cli.command('update-gitlab-elasticsearch-indexer')
 328 def update_gitlab_elasticsearch_indexer():
 329     """Update gitlab-elasticsearch-indexer"""
 330     data = _get_data_json()
 331     gitlab_elasticsearch_indexer_version = data['passthru']['GITLAB_ELASTICSEARCH_INDEXER_VERSION']
 332     _call_nix_update('gitlab-elasticsearch-indexer', gitlab_elasticsearch_indexer_version)
 333
 334
 335 @cli.command("update-all")
 336 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
 337 @click.option(
 338     "--commit", is_flag=True, default=False, help="Commit the changes for you"
 339 )
 340 @click.pass_context
 341 def update_all(ctx, rev: str, commit: bool):
 342     """Update all gitlab components to the latest stable release"""
 343     old_data_json = _get_data_json()
 344     old_container_registry_version = get_container_registry_version()
 345
 346     ctx.invoke(update_data, rev=rev)
 347
 348     new_data_json = _get_data_json()
 349
 350     ctx.invoke(update_rubyenv)
 351     ctx.invoke(update_gitaly)
 352     ctx.invoke(update_gitlab_pages)
 353     ctx.invoke(update_gitlab_shell)
 354     ctx.invoke(update_gitlab_workhorse)
 355     ctx.invoke(update_gitlab_elasticsearch_indexer)
 356     if commit:
 357         commit_gitlab(
 358             old_data_json["version"], new_data_json["version"], new_data_json["rev"]
 359         )
 360
 361     ctx.invoke(update_gitlab_container_registry)
 362     if commit:
 363         new_container_registry_version = get_container_registry_version()
 364         commit_container_registry(
 365             old_container_registry_version, new_container_registry_version
 366         )
 367
 368
 369 def commit_gitlab(old_version: str, new_version: str, new_rev: str) -> None:
 370     """Commits the gitlab changes for you"""
 371     subprocess.run(
 372         [
 373             "git",
 374             "add",
 375             "data.json",
 376             "rubyEnv",
 377             "gitaly",
 378             "gitlab-pages",
 379             "gitlab-shell",
 380             "gitlab-workhorse",
 381             "gitlab-elasticsearch-indexer",
 382         ],
 383         cwd=GITLAB_DIR,
 384     )
 385     subprocess.run(
 386         [
 387             "git",
 388             "commit",
 389             "--message",
 390             f"""gitlab: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/gitlab/-/blob/{new_rev}/CHANGELOG.md""",
 391         ],
 392         cwd=GITLAB_DIR,
 393     )
 394
 395
 396 def commit_container_registry(old_version: str, new_version: str) -> None:
 397     """Commits the gitlab-container-registry changes for you"""
 398     subprocess.run(["git", "add", "gitlab-container-registry"], cwd=GITLAB_DIR)
 399     subprocess.run(
 400         [
 401             "git",
 402             "commit",
 403             "--message",
 404             f"gitlab-container-registry: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/container-registry/-/blob/v{new_version}-gitlab/CHANGELOG.md",
 405         ],
 406         cwd=GITLAB_DIR,
 407     )
 408
 409
 410 if __name__ == "__main__":
 411     cli()