1 #!/usr/bin/env nix-shell
2 #! nix-shell -I nixpkgs=../../../.. -i python3 -p bundix bundler nix-update nix nix-universal-prefetch python3 python3Packages.requests python3Packages.click python3Packages.click-log python3Packages.packaging prefetch-yarn-deps git
12 from packaging
.version
import Version
13 from typing
import Iterable
17 NIXPKGS_PATH
= pathlib
.Path(__file__
).parent
/ "../../../../"
18 GITLAB_DIR
= pathlib
.Path(__file__
).parent
20 logger
= logging
.getLogger(__name__
)
21 click_log
.basic_config(logger
)
25 version_regex
= re
.compile(r
"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?(\-gitlab)?")
27 def __init__(self
, owner
: str = "gitlab-org", repo
: str = "gitlab"):
33 return f
"https://gitlab.com/{self.owner}/{self.repo}"
36 def tags(self
) -> Iterable
[str]:
37 """Returns a sorted list of repository tags"""
38 r
= requests
.get(self
.url
+ "/refs?sort=updated_desc&ref=master").json()
39 tags
= r
.get("Tags", [])
41 # filter out versions not matching version_regex
42 versions
= list(filter(self
.version_regex
.match
, tags
))
44 # sort, but ignore v, -ee and -gitlab for sorting comparisons
46 key
=lambda x
: Version(
47 x
.replace("v", "").replace("-ee", "").replace("-gitlab", "")
53 def get_git_hash(self
, rev
: str):
55 subprocess
.check_output(
57 "nix-universal-prefetch",
71 def get_yarn_hash(self
, rev
: str):
72 with tempfile
.TemporaryDirectory() as tmp_dir
:
73 with
open(tmp_dir
+ "/yarn.lock", "w") as f
:
74 f
.write(self
.get_file("yarn.lock", rev
))
76 subprocess
.check_output(["prefetch-yarn-deps", tmp_dir
+ "/yarn.lock"])
82 def rev2version(tag
: str) -> str:
84 normalize a tag to a version number.
85 This obviously isn't very smart if we don't pass something that looks like a tag
86 :param tag: the tag to normalize
87 :return: a normalized version number
90 version
= re
.sub(r
"^v", "", tag
)
91 # strip -ee and -gitlab suffixes
92 return re
.sub(r
"-(ee|gitlab)$", "", version
)
94 def get_file(self
, filepath
, rev
):
96 returns file contents at a given rev
97 :param filepath: the path to the file, relative to the repo root
98 :param rev: the rev to fetch at
101 return requests
.get(self
.url
+ f
"/raw/{rev}/{filepath}").text
103 def get_data(self
, rev
):
104 version
= self
.rev2version(rev
)
107 v
: self
.get_file(v
, rev
).strip()
109 "GITALY_SERVER_VERSION",
110 "GITLAB_PAGES_VERSION",
111 "GITLAB_SHELL_VERSION",
112 "GITLAB_ELASTICSEARCH_INDEXER_VERSION",
115 passthru
["GITLAB_WORKHORSE_VERSION"] = version
118 version
=self
.rev2version(rev
),
119 repo_hash
=self
.get_git_hash(rev
),
120 yarn_hash
=self
.get_yarn_hash(rev
),
128 def _get_data_json():
129 data_file_path
= pathlib
.Path(__file__
).parent
/ "data.json"
130 with
open(data_file_path
, "r") as f
:
134 def _call_nix_update(pkg
, version
):
135 """calls nix-update from nixpkgs root dir"""
136 return subprocess
.check_output(
137 ["nix-update", pkg
, "--version", version
], cwd
=NIXPKGS_PATH
141 @click_log.simple_verbosity_option(logger
)
147 @cli.command("update-data")
148 @click.option("--rev", default
="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
149 def update_data(rev
: str):
150 """Update data.json"""
151 logger
.info("Updating data.json")
155 # filter out pre and rc releases
156 rev
= next(filter(lambda x
: not ("rc" in x
or x
.endswith("pre")), repo
.tags
))
158 data_file_path
= pathlib
.Path(__file__
).parent
/ "data.json"
160 data
= repo
.get_data(rev
)
162 with
open(data_file_path
.as_posix(), "w") as f
:
163 json
.dump(data
, f
, indent
=2)
167 @cli.command("update-rubyenv")
168 def update_rubyenv():
170 logger
.info("Updating gitlab")
172 rubyenv_dir
= pathlib
.Path(__file__
).parent
/ "rubyEnv"
174 # load rev from data.json
175 data
= _get_data_json()
177 version
= data
["version"]
179 for fn
in ["Gemfile.lock", "Gemfile"]:
180 with
open(rubyenv_dir
/ fn
, "w") as f
:
181 f
.write(repo
.get_file(fn
, rev
))
183 # patch for openssl 3.x support
184 subprocess
.check_output(
185 ["sed", "-i", "s:'openssl', '2.*':'openssl', '3.0.2':g", "Gemfile"],
189 # Fetch vendored dependencies temporarily in order to build the gemset.nix
190 subprocess
.check_output(["mkdir", "-p", "vendor/gems", "gems"], cwd
=rubyenv_dir
)
191 subprocess
.check_output(
195 f
"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=vendor/gems | tar -xj --strip-components=3",
197 cwd
=f
"{rubyenv_dir}/vendor/gems",
199 subprocess
.check_output(
203 f
"curl -L https://gitlab.com/gitlab-org/gitlab/-/archive/v{version}-ee/gitlab-v{version}-ee.tar.bz2?path=gems | tar -xj --strip-components=3",
205 cwd
=f
"{rubyenv_dir}/gems",
208 # Undo our gemset.nix patches so that bundix runs through
209 subprocess
.check_output(
210 ["sed", "-i", "-e", "1d", "-e", "s:\\${src}/::g", "gemset.nix"], cwd
=rubyenv_dir
213 subprocess
.check_output(["bundle", "lock"], cwd
=rubyenv_dir
)
214 subprocess
.check_output(["bundix"], cwd
=rubyenv_dir
)
216 subprocess
.check_output(
223 's:path = \\(vendor/[^;]*\\);:path = "${src}/\\1";:g',
225 's:path = \\(gems/[^;]*\\);:path = "${src}/\\1";:g',
230 subprocess
.check_output(["rm", "-rf", "vendor", "gems"], cwd
=rubyenv_dir
)
233 @cli.command("update-gitaly")
236 logger
.info("Updating gitaly")
237 data
= _get_data_json()
238 gitaly_server_version
= data
['passthru']['GITALY_SERVER_VERSION']
240 _call_nix_update("gitaly", gitaly_server_version
)
243 @cli.command("update-gitlab-pages")
244 def update_gitlab_pages():
245 """Update gitlab-pages"""
246 logger
.info("Updating gitlab-pages")
247 data
= _get_data_json()
248 gitlab_pages_version
= data
["passthru"]["GITLAB_PAGES_VERSION"]
249 _call_nix_update("gitlab-pages", gitlab_pages_version
)
252 def get_container_registry_version() -> str:
253 """Returns the version attribute of gitlab-container-registry"""
254 return subprocess
.check_output(
257 "--experimental-features",
263 "gitlab-container-registry.version",
269 @cli.command("update-gitlab-shell")
270 def update_gitlab_shell():
271 """Update gitlab-shell"""
272 logger
.info("Updating gitlab-shell")
273 data
= _get_data_json()
274 gitlab_shell_version
= data
["passthru"]["GITLAB_SHELL_VERSION"]
275 _call_nix_update("gitlab-shell", gitlab_shell_version
)
278 @cli.command("update-gitlab-workhorse")
279 def update_gitlab_workhorse():
280 """Update gitlab-workhorse"""
281 logger
.info("Updating gitlab-workhorse")
282 data
= _get_data_json()
283 gitlab_workhorse_version
= data
["passthru"]["GITLAB_WORKHORSE_VERSION"]
284 _call_nix_update("gitlab-workhorse", gitlab_workhorse_version
)
287 @cli.command("update-gitlab-container-registry")
288 @click.option("--rev", default
="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
290 "--commit", is_flag
=True, default
=False, help="Commit the changes for you"
292 def update_gitlab_container_registry(rev
: str, commit
: bool):
293 """Update gitlab-container-registry"""
294 logger
.info("Updading gitlab-container-registry")
295 repo
= GitLabRepo(repo
="container-registry")
296 old_container_registry_version
= get_container_registry_version()
299 rev
= next(filter(lambda x
: not ("rc" in x
or x
.endswith("pre")), repo
.tags
))
301 version
= repo
.rev2version(rev
)
302 _call_nix_update("gitlab-container-registry", version
)
304 new_container_registry_version
= get_container_registry_version()
305 commit_container_registry(
306 old_container_registry_version
, new_container_registry_version
310 @cli.command('update-gitlab-elasticsearch-indexer')
311 def update_gitlab_elasticsearch_indexer():
312 """Update gitlab-elasticsearch-indexer"""
313 data
= _get_data_json()
314 gitlab_elasticsearch_indexer_version
= data
['passthru']['GITLAB_ELASTICSEARCH_INDEXER_VERSION']
315 _call_nix_update('gitlab-elasticsearch-indexer', gitlab_elasticsearch_indexer_version
)
318 @cli.command("update-all")
319 @click.option("--rev", default
="latest", help="The rev to use (vX.Y.Z-ee), or 'latest'")
321 "--commit", is_flag
=True, default
=False, help="Commit the changes for you"
324 def update_all(ctx
, rev
: str, commit
: bool):
325 """Update all gitlab components to the latest stable release"""
326 old_data_json
= _get_data_json()
327 old_container_registry_version
= get_container_registry_version()
329 ctx
.invoke(update_data
, rev
=rev
)
331 new_data_json
= _get_data_json()
333 ctx
.invoke(update_rubyenv
)
334 ctx
.invoke(update_gitaly
)
335 ctx
.invoke(update_gitlab_pages
)
336 ctx
.invoke(update_gitlab_shell
)
337 ctx
.invoke(update_gitlab_workhorse
)
338 ctx
.invoke(update_gitlab_elasticsearch_indexer
)
341 old_data_json
["version"], new_data_json
["version"], new_data_json
["rev"]
344 ctx
.invoke(update_gitlab_container_registry
)
346 new_container_registry_version
= get_container_registry_version()
347 commit_container_registry(
348 old_container_registry_version
, new_container_registry_version
352 def commit_gitlab(old_version
: str, new_version
: str, new_rev
: str) -> None:
353 """Commits the gitlab changes for you"""
364 "gitlab-elasticsearch-indexer",
373 f
"""gitlab: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/gitlab/-/blob/{new_rev}/CHANGELOG.md""",
379 def commit_container_registry(old_version
: str, new_version
: str) -> None:
380 """Commits the gitlab-container-registry changes for you"""
381 subprocess
.run(["git", "add", "gitlab-container-registry"], cwd
=GITLAB_DIR
)
387 f
"gitlab-container-registry: {old_version} -> {new_version}\n\nhttps://gitlab.com/gitlab-org/container-registry/-/blob/v{new_version}-gitlab/CHANGELOG.md",
393 if __name__
== "__main__":