nixos/tests/gitlab.nix

   1 # This test runs gitlab and performs the following tests:
   2 # - Creating users
   3 # - Pushing commits
   4 #   - over the API
   5 #   - over SSH
   6 # - Creating Merge Requests and merging them
   7 # - Opening and closing issues.
   8 # - Downloading repository archives as tar.gz and tar.bz2
   9 import ./make-test-python.nix ({ pkgs, lib, ... }:
  10
  11 with lib;
  12
  13 let
  14   inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
  15   initialRootPassword = "notproduction";
  16   rootProjectId = "2";
  17
  18   aliceUsername = "alice";
  19   aliceUserId = "2";
  20   alicePassword = "alicepassword";
  21   aliceProjectId = "2";
  22   aliceProjectName = "test-alice";
  23
  24   bobUsername = "bob";
  25   bobUserId = "3";
  26   bobPassword = "bobpassword";
  27   bobProjectId = "3";
  28 in {
  29   name = "gitlab";
  30   meta = with pkgs.lib.maintainers; {
  31     maintainers = [ globin yayayayaka ];
  32   };
  33
  34   nodes = {
  35     gitlab = { ... }: {
  36       imports = [ common/user-account.nix ];
  37
  38       virtualisation.memorySize = if pkgs.stdenv.is64bit then 4096 else 2047;
  39       virtualisation.cores = 4;
  40       virtualisation.useNixStoreImage = true;
  41       virtualisation.writableStore = false;
  42
  43       systemd.services.gitlab.serviceConfig.Restart = mkForce "no";
  44       systemd.services.gitlab-workhorse.serviceConfig.Restart = mkForce "no";
  45       systemd.services.gitaly.serviceConfig.Restart = mkForce "no";
  46       systemd.services.gitlab-sidekiq.serviceConfig.Restart = mkForce "no";
  47
  48       services.nginx = {
  49         enable = true;
  50         recommendedProxySettings = true;
  51         virtualHosts = {
  52           localhost = {
  53             locations."/".proxyPass = "http://unix:/run/gitlab/gitlab-workhorse.socket";
  54           };
  55         };
  56       };
  57
  58       services.openssh.enable = true;
  59
  60       services.dovecot2 = {
  61         enable = true;
  62         enableImap = true;
  63       };
  64
  65       systemd.services.gitlab-backup.environment.BACKUP = "dump";
  66
  67       services.gitlab = {
  68         enable = true;
  69         databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4";
  70         initialRootPasswordFile = pkgs.writeText "rootPassword" initialRootPassword;
  71         smtp.enable = true;
  72         extraConfig = {
  73           incoming_email = {
  74             enabled = true;
  75             mailbox = "inbox";
  76             address = "alice@localhost";
  77             user = "alice";
  78             password = "foobar";
  79             host = "localhost";
  80             port = 143;
  81           };
  82           # https://github.com/NixOS/nixpkgs/issues/132295
  83           # pages = {
  84           #   enabled = true;
  85           #   host = "localhost";
  86           # };
  87         };
  88         secrets = {
  89           secretFile = pkgs.writeText "secret" "Aig5zaic";
  90           otpFile = pkgs.writeText "otpsecret" "Riew9mue";
  91           dbFile = pkgs.writeText "dbsecret" "we2quaeZ";
  92           jwsFile = pkgs.runCommand "oidcKeyBase" {} "${pkgs.openssl}/bin/openssl genrsa 2048 > $out";
  93         };
  94       };
  95     };
  96   };
  97
  98   testScript = { nodes, ... }:
  99     let
 100       auth = pkgs.writeText "auth.json" (builtins.toJSON {
 101         grant_type = "password";
 102         username = "root";
 103         password = initialRootPassword;
 104       });
 105
 106       createUserAlice = pkgs.writeText "create-user-alice.json" (builtins.toJSON rec {
 107         username = aliceUsername;
 108         name = username;
 109         email = "alice@localhost";
 110         password = alicePassword;
 111         skip_confirmation = true;
 112       });
 113
 114       createUserBob = pkgs.writeText "create-user-bob.json" (builtins.toJSON rec {
 115         username = bobUsername;
 116         name = username;
 117         email = "bob@localhost";
 118         password = bobPassword;
 119         skip_confirmation = true;
 120       });
 121
 122       aliceAuth = pkgs.writeText "alice-auth.json" (builtins.toJSON {
 123         grant_type = "password";
 124         username = aliceUsername;
 125         password = alicePassword;
 126       });
 127
 128       bobAuth = pkgs.writeText "bob-auth.json" (builtins.toJSON {
 129         grant_type = "password";
 130         username = bobUsername;
 131         password = bobPassword;
 132       });
 133
 134       aliceAddSSHKey = pkgs.writeText "alice-add-ssh-key.json" (builtins.toJSON {
 135         id = aliceUserId;
 136         title = "snakeoil@nixos";
 137         key = snakeOilPublicKey;
 138       });
 139
 140       createProjectAlice = pkgs.writeText "create-project-alice.json" (builtins.toJSON {
 141         name = aliceProjectName;
 142         visibility = "public";
 143       });
 144
 145       putFile = pkgs.writeText "put-file.json" (builtins.toJSON {
 146         branch = "master";
 147         author_email = "author@example.com";
 148         author_name = "Firstname Lastname";
 149         content = "some content";
 150         commit_message = "create a new file";
 151       });
 152
 153       mergeRequest = pkgs.writeText "merge-request.json" (builtins.toJSON {
 154         id = bobProjectId;
 155         target_project_id = aliceProjectId;
 156         source_branch = "master";
 157         target_branch = "master";
 158         title = "Add some other file";
 159       });
 160
 161       newIssue = pkgs.writeText "new-issue.json" (builtins.toJSON {
 162         title = "useful issue title";
 163       });
 164
 165       closeIssue = pkgs.writeText "close-issue.json" (builtins.toJSON {
 166         issue_iid = 1;
 167         state_event = "close";
 168       });
 169
 170       # Wait for all GitLab services to be fully started.
 171       waitForServices = ''
 172         gitlab.wait_for_unit("gitaly.service")
 173         gitlab.wait_for_unit("gitlab-workhorse.service")
 174         # https://github.com/NixOS/nixpkgs/issues/132295
 175         # gitlab.wait_for_unit("gitlab-pages.service")
 176         gitlab.wait_for_unit("gitlab-mailroom.service")
 177         gitlab.wait_for_unit("gitlab.service")
 178         gitlab.wait_for_unit("gitlab-sidekiq.service")
 179         gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/tmp/sockets/gitlab.socket")
 180         gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in")
 181       '';
 182
 183       # The actual test of GitLab. Only push data to GitLab if
 184       # `doSetup` is is true.
 185       test = doSetup: ''
 186         GIT_SSH_COMMAND = "ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null"
 187
 188         gitlab.succeed(
 189             "curl -isSf http://gitlab | grep -i location | grep http://gitlab/users/sign_in"
 190         )
 191         gitlab.succeed(
 192             "${pkgs.sudo}/bin/sudo -u gitlab -H gitlab-rake gitlab:check 1>&2"
 193         )
 194         gitlab.succeed(
 195             "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${auth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers"
 196         )
 197       '' + optionalString doSetup ''
 198         with subtest("Create user Alice"):
 199             gitlab.succeed(
 200                 """[ "$(curl -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createUserAlice} http://gitlab/api/v4/users)" = "201" ]"""
 201             )
 202             gitlab.succeed(
 203                 "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${aliceAuth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers-alice"
 204             )
 205
 206         with subtest("Create user Bob"):
 207             gitlab.succeed(
 208                 """ [ "$(curl -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createUserBob} http://gitlab/api/v4/users)" = "201" ]"""
 209             )
 210             gitlab.succeed(
 211                 "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${bobAuth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers-bob"
 212             )
 213
 214         with subtest("Setup Git and SSH for Alice"):
 215             gitlab.succeed("git config --global user.name Alice")
 216             gitlab.succeed("git config --global user.email alice@nixos.invalid")
 217             gitlab.succeed("mkdir -m 700 /root/.ssh")
 218             gitlab.succeed("cat ${snakeOilPrivateKey} > /root/.ssh/id_ecdsa")
 219             gitlab.succeed("chmod 600 /root/.ssh/id_ecdsa")
 220             gitlab.succeed(
 221                 """
 222                 [ "$(curl \
 223                     -o /dev/null \
 224                     -w '%{http_code}' \
 225                     -X POST \
 226                     -H 'Content-Type: application/json' \
 227                     -H @/tmp/headers-alice -d @${aliceAddSSHKey} \
 228                     http://gitlab/api/v4/user/keys)" = "201" ]
 229                 """
 230             )
 231
 232         with subtest("Create a new repository"):
 233             # Alice creates a new repository
 234             gitlab.succeed(
 235                 """
 236                 [ "$(curl \
 237                     -o /dev/null \
 238                     -w '%{http_code}' \
 239                     -X POST \
 240                     -H 'Content-Type: application/json' \
 241                     -H @/tmp/headers-alice \
 242                     -d @${createProjectAlice} \
 243                     http://gitlab/api/v4/projects)" = "201" ]
 244                 """
 245             )
 246
 247             # Alice commits an initial commit
 248             gitlab.succeed(
 249                 """
 250                 [ "$(curl \
 251                     -o /dev/null \
 252                     -w '%{http_code}' \
 253                     -X POST \
 254                     -H 'Content-Type: application/json' \
 255                     -H @/tmp/headers-alice \
 256                     -d @${putFile} \
 257                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/files/some-file.txt)" = "201" ]"""
 258             )
 259
 260         with subtest("git clone over HTTP"):
 261             gitlab.succeed(
 262                 """git clone http://gitlab/alice/${aliceProjectName}.git clone-via-http""",
 263                 timeout=15
 264             )
 265
 266         with subtest("Push a commit via SSH"):
 267             gitlab.succeed(
 268                 f"""GIT_SSH_COMMAND="{GIT_SSH_COMMAND}" git clone gitlab@gitlab:alice/${aliceProjectName}.git""",
 269                 timeout=15
 270             )
 271             gitlab.succeed(
 272                 """echo "a commit sent over ssh" > ${aliceProjectName}/ssh.txt"""
 273             )
 274             gitlab.succeed(
 275                 """
 276                 cd ${aliceProjectName} || exit 1
 277                 git add .
 278                 """
 279             )
 280             gitlab.succeed(
 281                 """
 282                 cd ${aliceProjectName} || exit 1
 283                 git commit -m "Add a commit to be sent over ssh"
 284                 """
 285             )
 286             gitlab.succeed(
 287                 f"""
 288                 cd ${aliceProjectName} || exit 1
 289                 GIT_SSH_COMMAND="{GIT_SSH_COMMAND}" git push --set-upstream origin master
 290                 """,
 291                 timeout=15
 292             )
 293
 294         with subtest("Fork a project"):
 295             # Bob forks Alice's project
 296             gitlab.succeed(
 297                 """
 298                 [ "$(curl \
 299                     -o /dev/null \
 300                     -w '%{http_code}' \
 301                     -X POST \
 302                     -H 'Content-Type: application/json' \
 303                     -H @/tmp/headers-bob \
 304                     http://gitlab/api/v4/projects/${aliceProjectId}/fork)" = "201" ]
 305                 """
 306             )
 307
 308             # Bob creates a commit
 309             gitlab.wait_until_succeeds(
 310                 """
 311                 [ "$(curl \
 312                     -o /dev/null \
 313                     -w '%{http_code}' \
 314                     -X POST \
 315                     -H 'Content-Type: application/json' \
 316                     -H @/tmp/headers-bob \
 317                     -d @${putFile} \
 318                     http://gitlab/api/v4/projects/${bobProjectId}/repository/files/some-other-file.txt)" = "201" ]
 319                 """
 320             )
 321
 322         with subtest("Create a Merge Request"):
 323             # Bob opens a merge request against Alice's repository
 324             gitlab.wait_until_succeeds(
 325                 """
 326                 [ "$(curl \
 327                     -o /dev/null \
 328                     -w '%{http_code}' \
 329                     -X POST \
 330                     -H 'Content-Type: application/json' \
 331                     -H @/tmp/headers-bob \
 332                     -d @${mergeRequest} \
 333                     http://gitlab/api/v4/projects/${bobProjectId}/merge_requests)" = "201" ]
 334                 """
 335             )
 336
 337             # Alice merges the MR
 338             gitlab.wait_until_succeeds(
 339                 """
 340                 [ "$(curl \
 341                     -o /dev/null \
 342                     -w '%{http_code}' \
 343                     -X PUT \
 344                     -H 'Content-Type: application/json' \
 345                     -H @/tmp/headers-alice \
 346                     -d @${mergeRequest} \
 347                     http://gitlab/api/v4/projects/${aliceProjectId}/merge_requests/1/merge)" = "200" ]
 348                 """
 349             )
 350
 351         with subtest("Create an Issue"):
 352             # Bob opens an issue on Alice's repository
 353             gitlab.succeed(
 354                 """[ "$(curl \
 355                     -o /dev/null \
 356                     -w '%{http_code}' \
 357                     -X POST \
 358                     -H 'Content-Type: application/json' \
 359                     -H @/tmp/headers-bob \
 360                     -d @${newIssue} \
 361                     http://gitlab/api/v4/projects/${aliceProjectId}/issues)" = "201" ]
 362                 """
 363             )
 364
 365             # Alice closes the issue
 366             gitlab.wait_until_succeeds(
 367                 """
 368                 [ "$(curl \
 369                     -o /dev/null \
 370                     -w '%{http_code}' \
 371                     -X PUT \
 372                     -H 'Content-Type: application/json' \
 373                     -H @/tmp/headers-alice -d @${closeIssue} http://gitlab/api/v4/projects/${aliceProjectId}/issues/1)" = "200" ]
 374                 """
 375             )
 376       '' + ''
 377         with subtest("Download archive.tar.gz"):
 378             gitlab.succeed(
 379                 """
 380                 [ "$(curl \
 381                     -o /dev/null \
 382                     -w '%{http_code}' \
 383                     -H @/tmp/headers-alice \
 384                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.gz)" = "200" ]
 385                 """
 386             )
 387             gitlab.succeed(
 388                 """
 389                 curl \
 390                     -H @/tmp/headers-alice \
 391                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.gz > /tmp/archive.tar.gz
 392                 """
 393             )
 394             gitlab.succeed("test -s /tmp/archive.tar.gz")
 395
 396         with subtest("Download archive.tar.bz2"):
 397             gitlab.succeed(
 398                 """
 399                 [ "$(curl \
 400                     -o /dev/null \
 401                     -w '%{http_code}' \
 402                     -H @/tmp/headers-alice \
 403                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.bz2)" = "200" ]
 404                 """
 405             )
 406             gitlab.succeed(
 407                 """
 408                 curl \
 409                     -H @/tmp/headers-alice \
 410                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.bz2 > /tmp/archive.tar.bz2
 411                 """
 412             )
 413             gitlab.succeed("test -s /tmp/archive.tar.bz2")
 414       '';
 415
 416   in ''
 417       gitlab.start()
 418     ''
 419     + waitForServices
 420     + test true
 421     + ''
 422       gitlab.systemctl("start gitlab-backup.service")
 423       gitlab.wait_for_unit("gitlab-backup.service")
 424       gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/backup/dump_gitlab_backup.tar")
 425       gitlab.systemctl("stop postgresql.service gitlab.target")
 426       gitlab.succeed(
 427           "find ${nodes.gitlab.config.services.gitlab.statePath} -mindepth 1 -maxdepth 1 -not -name backup -execdir rm -r {} +"
 428       )
 429       gitlab.succeed("systemd-tmpfiles --create")
 430       gitlab.succeed("rm -rf ${nodes.gitlab.config.services.postgresql.dataDir}")
 431       gitlab.systemctl("start gitlab-config.service gitaly.service gitlab-postgresql.service")
 432       gitlab.wait_for_file("${nodes.gitlab.config.services.gitlab.statePath}/tmp/sockets/gitaly.socket")
 433       gitlab.succeed(
 434           "sudo -u gitlab -H gitlab-rake gitlab:backup:restore RAILS_ENV=production BACKUP=dump force=yes"
 435       )
 436       gitlab.systemctl("start gitlab.target")
 437     ''
 438     + waitForServices
 439     + test false;
 440 })