biglybt: 3.5.0.0 -> 3.6.0.0
[NixPkgs.git] / pkgs / applications / version-management / gitlab / update.py
blob195fce350c28a93d58338a3ed108b101db2b6e2a
1 #!/usr/bin/env nix-shell
2 #! nix-shell -I nixpkgs=../../../.. -i python3 -p bundix bundler nix-update nix nix-universal-prefetch python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.packaging prefetch-yarn-deps git
4 import click
5 import click_log
6 import re
7 import logging
8 import subprocess
9 import json
10 import pathlib
11 import tempfile
12 from packaging.version import Version
13 from typing import Iterable
15 import requests
17 NIXPKGS_PATH = pathlib.Path(__file__).parent / "../../../../"
18 GITLAB_DIR = pathlib.Path(__file__).parent
20 logger = logging.getLogger(__name__)
21 click_log.basic_config(logger)
24 class GitLabRepo:
25 version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?(\-gitlab)?")
27 def __init__(self, owner: str = "gitlab-org", repo: str = "gitlab"):
28 self.owner = owner
29 self.repo = repo
31 @property
32 def url(self):
33 return f"https://gitlab.com/{self.owner}/{self.repo}"
35 @property
36 def tags(self) -> Iterable[str]:
37 """Returns a sorted list of repository tags"""
38 r = requests.get(self.url + "/refs?sort=updated_desc&ref=master").json()
39 tags = r.get("Tags", [])
41 # filter out versions not matching version_regex
42 versions = list(filter(self.version_regex.match, tags))
44 # sort, but ignore v, -ee and -gitlab for sorting comparisons
45 versions.sort(
46 key=lambda x: Version(
47 x.replace("v", "").replace("-ee", "").replace("-gitlab", "")
49 reverse=True,
51 return versions
53 def get_git_hash(self, rev: str):
54 return (
55 subprocess.check_output(
57 "nix-universal-prefetch",
58 "fetchFromGitLab",
59 "--owner",
60 self.owner,
61 "--repo",
62 self.repo,
63 "--rev",
64 rev,
67 .decode("utf-8")
68 .strip()
71 def get_yarn_hash(self, rev: str):
72 with tempfile.TemporaryDirectory() as tmp_dir:
73 with open(tmp_dir + "/yarn.lock", "w") as f:
74 f.write(self.get_file("yarn.lock", rev))
75 return (
76 subprocess.check_output(["prefetch-yarn-deps", tmp_dir + "/yarn.lock"])
77 .decode("utf-8")
78 .strip()
81 @staticmethod
82 def rev2version(tag: str) -> str:
83 """
84 normalize a tag to a version number.
85 This obviously isn't very smart if we don't pass something that looks like a tag
86 :param tag: the tag to normalize
87 :return: a normalized version number
88 """
89 # strip v prefix
90 version = re.sub(r"^v", "", tag)
91 # strip -ee and -gitlab suffixes
92 return re.sub(r"-(ee|gitlab)$", "", version)
94 def get_file(self, filepath, rev):
95 """
96 returns file contents at a given rev
97 :param filepath: the path to the file, relative to the repo root
98 :param rev: the rev to fetch at
99 :return:
101 return requests.get(self.url + f"/raw/{rev}/{filepath}").text
103 def get_data(self, rev):
104 version = self.rev2version(rev)
106 passthru = {
107 v: self.get_file(v, rev).strip()
108 for v in [
109 "GITALY_SERVER_VERSION",
110 "GITLAB_PAGES_VERSION",
111 "GITLAB_SHELL_VERSION",
112 "GITLAB_ELASTICSEARCH_INDEXER_VERSION",
115 passthru["GITLAB_WORKHORSE_VERSION"] = version
117 return dict(
118 version=self.rev2version(rev),
119 repo_hash=self.get_git_hash(rev),
120 yarn_hash=self.get_yarn_hash(rev),
121 owner=self.owner,
122 repo=self.repo,
123 rev=rev,
124 passthru=passthru,
128 def _get_data_json():
129 data_file_path = pathlib.Path(__file__).parent / "data.json"
130 with open(data_file_path, "r") as f:
131 return json.load(f)
134 def _call_nix_update(pkg, version):
135 """calls nix-update from nixpkgs root dir"""
136 return subprocess.check_output(
137 ["nix-update", pkg, "--version", version], cwd=NIXPKGS_PATH
141 @click_log.simple_verbosity_option(logger)
142 @click.group()
143 def cli():
144 pass
147 @cli.command("update-data")
148 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
149 def update_data(rev: str):
150 """Update data.json"""
151 logger.info("Updating data.json")
153 repo = GitLabRepo()
154 if rev == "latest":
155 # filter out pre and rc releases
156 rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
158 data_file_path = pathlib.Path(__file__).parent / "data.json"
160 data = repo.get_data(rev)
162 with open(data_file_path.as_posix(), "w") as f:
163 json.dump(data, f, indent=2)
164 f.write("\n")
167 @cli.command("update-rubyenv")
168 def update_rubyenv():
169 """Update rubyEnv"""
170 logger.info("Updating gitlab")
171 repo = GitLabRepo()
172 rubyenv_dir = pathlib.Path(__file__).parent / "rubyEnv"
174 # load rev from data.json
175 data = _get_data_json()
176 rev = data["rev"]
177 version = data["version"]
179 for fn in ["Gemfile.lock", "Gemfile"]:
180 with open(rubyenv_dir / fn, "w") as f:
181 f.write(repo.get_file(fn, rev))
183 # patch for openssl 3.x support
184 subprocess.check_output(
185 ["sed", "-i", "s:'openssl', '2.*':'openssl', '3.0.2':g", "Gemfile"],
186 cwd=rubyenv_dir,
189 # Fetch vendored dependencies temporarily in order to build the gemset.nix
190 subprocess.check_output(["mkdir", "-p", "vendor/gems", "gems"], cwd=rubyenv_dir)
191 subprocess.check_output(
193 "sh",
194 "-c",
195 f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=vendor/gems | tar -xj --strip-components=3",
197 cwd=f"{rubyenv_dir}/vendor/gems",
199 subprocess.check_output(
201 "sh",
202 "-c",
203 f"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=gems | tar -xj --strip-components=3",
205 cwd=f"{rubyenv_dir}/gems",
208 # Undo our gemset.nix patches so that bundix runs through
209 subprocess.check_output(
210 ["sed", "-i", "-e", "1d", "-e", "s:\\${src}/::g", "gemset.nix"], cwd=rubyenv_dir
213 subprocess.check_output(["bundle", "lock"], cwd=rubyenv_dir)
214 subprocess.check_output(["bundix"], cwd=rubyenv_dir)
216 subprocess.check_output(
218 "sed",
219 "-i",
220 "-e",
221 "1i\\src:",
222 "-e",
223 's:path = \\(vendor/[^;]*\\);:path = "${src}/\\1";:g',
224 "-e",
225 's:path = \\(gems/[^;]*\\);:path = "${src}/\\1";:g',
226 "gemset.nix",
228 cwd=rubyenv_dir,
230 subprocess.check_output(["rm", "-rf", "vendor", "gems"], cwd=rubyenv_dir)
233 @cli.command("update-gitaly")
234 def update_gitaly():
235 """Update gitaly"""
236 logger.info("Updating gitaly")
237 data = _get_data_json()
238 gitaly_server_version = data['passthru']['GITALY_SERVER_VERSION']
240 _call_nix_update("gitaly", gitaly_server_version)
243 @cli.command("update-gitlab-pages")
244 def update_gitlab_pages():
245 """Update gitlab-pages"""
246 logger.info("Updating gitlab-pages")
247 data = _get_data_json()
248 gitlab_pages_version = data["passthru"]["GITLAB_PAGES_VERSION"]
249 _call_nix_update("gitlab-pages", gitlab_pages_version)
252 def get_container_registry_version() -> str:
253 """Returns the version attribute of gitlab-container-registry"""
254 return subprocess.check_output(
256 "nix",
257 "--experimental-features",
258 "nix-command",
259 "eval",
260 "-f",
261 ".",
262 "--raw",
263 "gitlab-container-registry.version",
265 cwd=NIXPKGS_PATH,
266 ).decode("utf-8")
269 @cli.command("update-gitlab-shell")
270 def update_gitlab_shell():
271 """Update gitlab-shell"""
272 logger.info("Updating gitlab-shell")
273 data = _get_data_json()
274 gitlab_shell_version = data["passthru"]["GITLAB_SHELL_VERSION"]
275 _call_nix_update("gitlab-shell", gitlab_shell_version)
278 @cli.command("update-gitlab-workhorse")
279 def update_gitlab_workhorse():
280 """Update gitlab-workhorse"""
281 logger.info("Updating gitlab-workhorse")
282 data = _get_data_json()
283 gitlab_workhorse_version = data["passthru"]["GITLAB_WORKHORSE_VERSION"]
284 _call_nix_update("gitlab-workhorse", gitlab_workhorse_version)
287 @cli.command("update-gitlab-container-registry")
288 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
289 @click.option(
290 "--commit", is_flag=True, default=False, help="Commit the changes for you"
292 def update_gitlab_container_registry(rev: str, commit: bool):
293 """Update gitlab-container-registry"""
294 logger.info("Updading gitlab-container-registry")
295 repo = GitLabRepo(repo="container-registry")
296 old_container_registry_version = get_container_registry_version()
298 if rev == "latest":
299 rev = next(filter(lambda x: not ("rc" in x or x.endswith("pre")), repo.tags))
301 version = repo.rev2version(rev)
302 _call_nix_update("gitlab-container-registry", version)
303 if commit:
304 new_container_registry_version = get_container_registry_version()
305 commit_container_registry(
306 old_container_registry_version, new_container_registry_version
310 @cli.command('update-gitlab-elasticsearch-indexer')
311 def update_gitlab_elasticsearch_indexer():
312 """Update gitlab-elasticsearch-indexer"""
313 data = _get_data_json()
314 gitlab_elasticsearch_indexer_version = data['passthru']['GITLAB_ELASTICSEARCH_INDEXER_VERSION']
315 _call_nix_update('gitlab-elasticsearch-indexer', gitlab_elasticsearch_indexer_version)
318 @cli.command("update-all")
319 @click.option("--rev", default="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
320 @click.option(
321 "--commit", is_flag=True, default=False, help="Commit the changes for you"
323 @click.pass_context
324 def update_all(ctx, rev: str, commit: bool):
325 """Update all gitlab components to the latest stable release"""
326 old_data_json = _get_data_json()
327 old_container_registry_version = get_container_registry_version()
329 ctx.invoke(update_data, rev=rev)
331 new_data_json = _get_data_json()
333 ctx.invoke(update_rubyenv)
334 ctx.invoke(update_gitaly)
335 ctx.invoke(update_gitlab_pages)
336 ctx.invoke(update_gitlab_shell)
337 ctx.invoke(update_gitlab_workhorse)
338 ctx.invoke(update_gitlab_elasticsearch_indexer)
339 if commit:
340 commit_gitlab(
341 old_data_json["version"], new_data_json["version"], new_data_json["rev"]
344 ctx.invoke(update_gitlab_container_registry)
345 if commit:
346 new_container_registry_version = get_container_registry_version()
347 commit_container_registry(
348 old_container_registry_version, new_container_registry_version
352 def commit_gitlab(old_version: str, new_version: str, new_rev: str) -> None:
353 """Commits the gitlab changes for you"""
354 subprocess.run(
356 "git",
357 "add",
358 "data.json",
359 "rubyEnv",
360 "gitaly",
361 "gitlab-pages",
362 "gitlab-shell",
363 "gitlab-workhorse",
364 "gitlab-elasticsearch-indexer",
366 cwd=GITLAB_DIR,
368 subprocess.run(
370 "git",
371 "commit",
372 "--message",
373 f"""gitlab: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/gitlab/-/blob/{new_rev}/CHANGELOG.md""",
375 cwd=GITLAB_DIR,
379 def commit_container_registry(old_version: str, new_version: str) -> None:
380 """Commits the gitlab-container-registry changes for you"""
381 subprocess.run(["git", "add", "gitlab-container-registry"], cwd=GITLAB_DIR)
382 subprocess.run(
384 "git",
385 "commit",
386 "--message",
387 f"gitlab-container-registry: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/container-registry/-/blob/v{new_version}-gitlab/CHANGELOG.md",
389 cwd=GITLAB_DIR,
393 if __name__ == "__main__":
394 cli()