nixos/tests/gitlab.nix

   1 # This test runs gitlab and performs the following tests:
   2 # - Creating users
   3 # - Pushing commits
   4 #   - over the API
   5 #   - over SSH
   6 # - Creating Merge Requests and merging them
   7 # - Opening and closing issues.
   8 # - Downloading repository archives as tar.gz and tar.bz2
   9 # Run with
  10 # [nixpkgs]$ nix-build -A nixosTests.gitlab
  11
  12 { pkgs, lib, ... }:
  13
  14 let
  15   inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
  16   initialRootPassword = "notproduction";
  17   rootProjectId = "2";
  18
  19   aliceUsername = "alice";
  20   aliceUserId = "2";
  21   alicePassword = "R5twyCgU0uXC71wT9BBTCqLs6HFZ7h3L";
  22   aliceProjectId = "1";
  23   aliceProjectName = "test-alice";
  24
  25   bobUsername = "bob";
  26   bobUserId = "3";
  27   bobPassword = "XwkkBbl2SiIwabQzgcoaTbhsotijEEtF";
  28   bobProjectId = "2";
  29 in {
  30   name = "gitlab";
  31   meta.maintainers = with lib.maintainers; [ globin yayayayaka ];
  32
  33   nodes = {
  34     gitlab = { ... }: {
  35       imports = [ common/user-account.nix ];
  36
  37       environment.systemPackages = with pkgs; [ git ];
  38
  39       virtualisation.memorySize = 6144;
  40       virtualisation.cores = 4;
  41       virtualisation.useNixStoreImage = true;
  42       virtualisation.writableStore = false;
  43
  44       systemd.services.gitlab.serviceConfig.Restart = lib.mkForce "no";
  45       systemd.services.gitlab-workhorse.serviceConfig.Restart = lib.mkForce "no";
  46       systemd.services.gitaly.serviceConfig.Restart = lib.mkForce "no";
  47       systemd.services.gitlab-sidekiq.serviceConfig.Restart = lib.mkForce "no";
  48
  49       services.nginx = {
  50         enable = true;
  51         recommendedProxySettings = true;
  52         virtualHosts = {
  53           localhost = {
  54             locations."/".proxyPass = "http://unix:/run/gitlab/gitlab-workhorse.socket";
  55           };
  56         };
  57       };
  58
  59       services.openssh.enable = true;
  60
  61       services.dovecot2 = {
  62         enable = true;
  63         enableImap = true;
  64       };
  65
  66       systemd.services.gitlab-backup.environment.BACKUP = "dump";
  67
  68       services.gitlab = {
  69         enable = true;
  70         databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4";
  71         initialRootPasswordFile = pkgs.writeText "rootPassword" initialRootPassword;
  72         smtp.enable = true;
  73         pages = {
  74           enable = true;
  75           settings.pages-domain = "localhost";
  76         };
  77         extraConfig = {
  78           incoming_email = {
  79             enabled = true;
  80             mailbox = "inbox";
  81             address = "alice@localhost";
  82             user = "alice";
  83             password = "foobar";
  84             host = "localhost";
  85             port = 143;
  86           };
  87         };
  88         secrets = {
  89           secretFile = pkgs.writeText "secret" "Aig5zaic";
  90           otpFile = pkgs.writeText "otpsecret" "Riew9mue";
  91           dbFile = pkgs.writeText "dbsecret" "we2quaeZ";
  92           jwsFile = pkgs.runCommand "oidcKeyBase" {} "${pkgs.openssl}/bin/openssl genrsa 2048 > $out";
  93         };
  94
  95         # reduce memory usage
  96         sidekiq.concurrency = 1;
  97         puma.workers = 2;
  98       };
  99     };
 100   };
 101
 102   testScript = { nodes, ... }:
 103     let
 104       auth = pkgs.writeText "auth.json" (builtins.toJSON {
 105         grant_type = "password";
 106         username = "root";
 107         password = initialRootPassword;
 108       });
 109
 110       createUserAlice = pkgs.writeText "create-user-alice.json" (builtins.toJSON rec {
 111         username = aliceUsername;
 112         name = username;
 113         email = "alice@localhost";
 114         password = alicePassword;
 115         skip_confirmation = true;
 116       });
 117
 118       createUserBob = pkgs.writeText "create-user-bob.json" (builtins.toJSON rec {
 119         username = bobUsername;
 120         name = username;
 121         email = "bob@localhost";
 122         password = bobPassword;
 123         skip_confirmation = true;
 124       });
 125
 126       aliceAuth = pkgs.writeText "alice-auth.json" (builtins.toJSON {
 127         grant_type = "password";
 128         username = aliceUsername;
 129         password = alicePassword;
 130       });
 131
 132       bobAuth = pkgs.writeText "bob-auth.json" (builtins.toJSON {
 133         grant_type = "password";
 134         username = bobUsername;
 135         password = bobPassword;
 136       });
 137
 138       aliceAddSSHKey = pkgs.writeText "alice-add-ssh-key.json" (builtins.toJSON {
 139         id = aliceUserId;
 140         title = "snakeoil@nixos";
 141         key = snakeOilPublicKey;
 142       });
 143
 144       createProjectAlice = pkgs.writeText "create-project-alice.json" (builtins.toJSON {
 145         name = aliceProjectName;
 146         visibility = "public";
 147       });
 148
 149       putFile = pkgs.writeText "put-file.json" (builtins.toJSON {
 150         branch = "master";
 151         author_email = "author@example.com";
 152         author_name = "Firstname Lastname";
 153         content = "some content";
 154         commit_message = "create a new file";
 155       });
 156
 157       mergeRequest = pkgs.writeText "merge-request.json" (builtins.toJSON {
 158         id = bobProjectId;
 159         target_project_id = aliceProjectId;
 160         source_branch = "master";
 161         target_branch = "master";
 162         title = "Add some other file";
 163       });
 164
 165       newIssue = pkgs.writeText "new-issue.json" (builtins.toJSON {
 166         title = "useful issue title";
 167       });
 168
 169       closeIssue = pkgs.writeText "close-issue.json" (builtins.toJSON {
 170         issue_iid = 1;
 171         state_event = "close";
 172       });
 173
 174       # Wait for all GitLab services to be fully started.
 175       waitForServices = ''
 176         gitlab.wait_for_unit("gitaly.service")
 177         gitlab.wait_for_unit("gitlab-workhorse.service")
 178         gitlab.wait_for_unit("gitlab-mailroom.service")
 179         gitlab.wait_for_unit("gitlab.service")
 180         gitlab.wait_for_unit("gitlab-pages.service")
 181         gitlab.wait_for_unit("gitlab-sidekiq.service")
 182         gitlab.wait_for_file("${nodes.gitlab.services.gitlab.statePath}/tmp/sockets/gitlab.socket")
 183         gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in")
 184       '';
 185
 186       # The actual test of GitLab. Only push data to GitLab if
 187       # `doSetup` is is true.
 188       test = doSetup: ''
 189         GIT_SSH_COMMAND = "ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/dev/null"
 190
 191         gitlab.succeed(
 192             "curl -isSf http://gitlab | grep -i location | grep http://gitlab/users/sign_in"
 193         )
 194         gitlab.succeed(
 195             "${pkgs.sudo}/bin/sudo -u gitlab -H gitlab-rake gitlab:check 1>&2"
 196         )
 197         gitlab.succeed(
 198             "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${auth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers"
 199         )
 200       '' + lib.optionalString doSetup ''
 201         with subtest("Create user Alice"):
 202             gitlab.succeed(
 203                 """[ "$(curl -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createUserAlice} http://gitlab/api/v4/users)" = "201" ]"""
 204             )
 205             gitlab.succeed(
 206                 "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${aliceAuth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers-alice"
 207             )
 208
 209         with subtest("Create user Bob"):
 210             gitlab.succeed(
 211                 """ [ "$(curl -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -H @/tmp/headers -d @${createUserBob} http://gitlab/api/v4/users)" = "201" ]"""
 212             )
 213             gitlab.succeed(
 214                 "echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @${bobAuth} http://gitlab/oauth/token | ${pkgs.jq}/bin/jq -r '.access_token')\" >/tmp/headers-bob"
 215             )
 216
 217         with subtest("Setup Git and SSH for Alice"):
 218             gitlab.succeed("git config --global user.name Alice")
 219             gitlab.succeed("git config --global user.email alice@nixos.invalid")
 220             gitlab.succeed("mkdir -m 700 /root/.ssh")
 221             gitlab.succeed("cat ${snakeOilPrivateKey} > /root/.ssh/id_ecdsa")
 222             gitlab.succeed("chmod 600 /root/.ssh/id_ecdsa")
 223             gitlab.succeed(
 224                 """
 225                 [ "$(curl \
 226                     -o /dev/null \
 227                     -w '%{http_code}' \
 228                     -X POST \
 229                     -H 'Content-Type: application/json' \
 230                     -H @/tmp/headers-alice -d @${aliceAddSSHKey} \
 231                     http://gitlab/api/v4/user/keys)" = "201" ]
 232                 """
 233             )
 234
 235         with subtest("Create a new repository"):
 236             # Alice creates a new repository
 237             gitlab.succeed(
 238                 """
 239                 [ "$(curl \
 240                     -o /dev/null \
 241                     -w '%{http_code}' \
 242                     -X POST \
 243                     -H 'Content-Type: application/json' \
 244                     -H @/tmp/headers-alice \
 245                     -d @${createProjectAlice} \
 246                     http://gitlab/api/v4/projects)" = "201" ]
 247                 """
 248             )
 249
 250             # Alice commits an initial commit
 251             gitlab.succeed(
 252                 """
 253                 [ "$(curl \
 254                     -o /dev/null \
 255                     -w '%{http_code}' \
 256                     -X POST \
 257                     -H 'Content-Type: application/json' \
 258                     -H @/tmp/headers-alice \
 259                     -d @${putFile} \
 260                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/files/some-file.txt)" = "201" ]"""
 261             )
 262
 263         with subtest("git clone over HTTP"):
 264             gitlab.succeed(
 265                 """git clone http://gitlab/alice/${aliceProjectName}.git clone-via-http""",
 266                 timeout=15
 267             )
 268
 269         with subtest("Push a commit via SSH"):
 270             gitlab.succeed(
 271                 f"""GIT_SSH_COMMAND="{GIT_SSH_COMMAND}" git clone gitlab@gitlab:alice/${aliceProjectName}.git""",
 272                 timeout=15
 273             )
 274             gitlab.succeed(
 275                 """echo "a commit sent over ssh" > ${aliceProjectName}/ssh.txt"""
 276             )
 277             gitlab.succeed(
 278                 """
 279                 cd ${aliceProjectName} || exit 1
 280                 git add .
 281                 """
 282             )
 283             gitlab.succeed(
 284                 """
 285                 cd ${aliceProjectName} || exit 1
 286                 git commit -m "Add a commit to be sent over ssh"
 287                 """
 288             )
 289             gitlab.succeed(
 290                 f"""
 291                 cd ${aliceProjectName} || exit 1
 292                 GIT_SSH_COMMAND="{GIT_SSH_COMMAND}" git push --set-upstream origin master
 293                 """,
 294                 timeout=15
 295             )
 296
 297         with subtest("Fork a project"):
 298             # Bob forks Alice's project
 299             gitlab.succeed(
 300                 """
 301                 [ "$(curl \
 302                     -o /dev/null \
 303                     -w '%{http_code}' \
 304                     -X POST \
 305                     -H 'Content-Type: application/json' \
 306                     -H @/tmp/headers-bob \
 307                     http://gitlab/api/v4/projects/${aliceProjectId}/fork)" = "201" ]
 308                 """
 309             )
 310
 311             # Bob creates a commit
 312             gitlab.wait_until_succeeds(
 313                 """
 314                 [ "$(curl \
 315                     -o /dev/null \
 316                     -w '%{http_code}' \
 317                     -X POST \
 318                     -H 'Content-Type: application/json' \
 319                     -H @/tmp/headers-bob \
 320                     -d @${putFile} \
 321                     http://gitlab/api/v4/projects/${bobProjectId}/repository/files/some-other-file.txt)" = "201" ]
 322                 """
 323             )
 324
 325         with subtest("Create a Merge Request"):
 326             # Bob opens a merge request against Alice's repository
 327             gitlab.wait_until_succeeds(
 328                 """
 329                 [ "$(curl \
 330                     -o /dev/null \
 331                     -w '%{http_code}' \
 332                     -X POST \
 333                     -H 'Content-Type: application/json' \
 334                     -H @/tmp/headers-bob \
 335                     -d @${mergeRequest} \
 336                     http://gitlab/api/v4/projects/${bobProjectId}/merge_requests)" = "201" ]
 337                 """
 338             )
 339
 340             # Alice merges the MR
 341             gitlab.wait_until_succeeds(
 342                 """
 343                 [ "$(curl \
 344                     -o /dev/null \
 345                     -w '%{http_code}' \
 346                     -X PUT \
 347                     -H 'Content-Type: application/json' \
 348                     -H @/tmp/headers-alice \
 349                     -d @${mergeRequest} \
 350                     http://gitlab/api/v4/projects/${aliceProjectId}/merge_requests/1/merge)" = "200" ]
 351                 """
 352             )
 353
 354         with subtest("Create an Issue"):
 355             # Bob opens an issue on Alice's repository
 356             gitlab.succeed(
 357                 """[ "$(curl \
 358                     -o /dev/null \
 359                     -w '%{http_code}' \
 360                     -X POST \
 361                     -H 'Content-Type: application/json' \
 362                     -H @/tmp/headers-bob \
 363                     -d @${newIssue} \
 364                     http://gitlab/api/v4/projects/${aliceProjectId}/issues)" = "201" ]
 365                 """
 366             )
 367
 368             # Alice closes the issue
 369             gitlab.wait_until_succeeds(
 370                 """
 371                 [ "$(curl \
 372                     -o /dev/null \
 373                     -w '%{http_code}' \
 374                     -X PUT \
 375                     -H 'Content-Type: application/json' \
 376                     -H @/tmp/headers-alice -d @${closeIssue} http://gitlab/api/v4/projects/${aliceProjectId}/issues/1)" = "200" ]
 377                 """
 378             )
 379       '' + ''
 380         with subtest("Download archive.tar.gz"):
 381             gitlab.succeed(
 382                 """
 383                 [ "$(curl \
 384                     -o /dev/null \
 385                     -w '%{http_code}' \
 386                     -H @/tmp/headers-alice \
 387                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.gz)" = "200" ]
 388                 """
 389             )
 390             gitlab.succeed(
 391                 """
 392                 curl \
 393                     -H @/tmp/headers-alice \
 394                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.gz > /tmp/archive.tar.gz
 395                 """
 396             )
 397             gitlab.succeed("test -s /tmp/archive.tar.gz")
 398
 399         with subtest("Download archive.tar.bz2"):
 400             gitlab.succeed(
 401                 """
 402                 [ "$(curl \
 403                     -o /dev/null \
 404                     -w '%{http_code}' \
 405                     -H @/tmp/headers-alice \
 406                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.bz2)" = "200" ]
 407                 """
 408             )
 409             gitlab.succeed(
 410                 """
 411                 curl \
 412                     -H @/tmp/headers-alice \
 413                     http://gitlab/api/v4/projects/${aliceProjectId}/repository/archive.tar.bz2 > /tmp/archive.tar.bz2
 414                 """
 415             )
 416             gitlab.succeed("test -s /tmp/archive.tar.bz2")
 417       '';
 418
 419   in ''
 420       gitlab.start()
 421     ''
 422     + waitForServices
 423     + test true
 424     + ''
 425       gitlab.systemctl("start gitlab-backup.service")
 426       gitlab.wait_for_unit("gitlab-backup.service")
 427       gitlab.wait_for_file("${nodes.gitlab.services.gitlab.statePath}/backup/dump_gitlab_backup.tar")
 428       gitlab.systemctl("stop postgresql.service gitlab-config.service gitlab.target")
 429       gitlab.succeed(
 430           "find ${nodes.gitlab.services.gitlab.statePath} -mindepth 1 -maxdepth 1 -not -name backup -execdir rm -r {} +"
 431       )
 432       gitlab.succeed("systemd-tmpfiles --create")
 433       gitlab.succeed("rm -rf ${nodes.gitlab.services.postgresql.dataDir}")
 434       gitlab.systemctl("start gitlab-config.service gitaly.service gitlab-postgresql.service")
 435       gitlab.wait_for_file("${nodes.gitlab.services.gitlab.statePath}/tmp/sockets/gitaly.socket")
 436       gitlab.succeed(
 437           "sudo -u gitlab -H gitlab-rake gitlab:backup:restore RAILS_ENV=production BACKUP=dump force=yes"
 438       )
 439       gitlab.systemctl("start gitlab.target")
 440     ''
 441     + waitForServices
 442     + test false;
 443 }