.gitlab-ci.yml

   1 stages:
   2   - docker_images
   3   - build_unit_test
   4   - pre_deploy
   5   - deploy
   6   - security
   7
   8 variables:
   9   DOCKER_VERSION: '19.03.0'
  10   # We don't use the default CI_REGISTRY_IMAGE variable because the security release process on dev.gitlab.org needs to target the gitlab.com registry.
  11   # https://gitlab.com/gitterHQ/webapp/-/merge_requests/1952#note_386372016
  12   GITTER_CI_REGISTRY_IMAGE: registry.gitlab.com/gitterhq/webapp
  13
  14 ##############
  15 # Conditions #
  16 ##############
  17 .if-merge-request: &if-merge-request
  18   if: '$CI_MERGE_REQUEST_IID'
  19
  20 .if-develop-branch: &if-develop-branch
  21   if: '$CI_COMMIT_BRANCH == "develop"'
  22
  23 .if-master-branch: &if-master-branch
  24   if: '$CI_COMMIT_BRANCH == "master"'
  25
  26 .if-release-branch: &if-release-branch
  27   if: '$CI_COMMIT_BRANCH =~ /^release\/.*$/'
  28
  29 .if-hotfix-branch: &if-hotfix-branch
  30   if: '$CI_COMMIT_BRANCH =~ /^hotfix\/.*$/'
  31
  32 .if-mr-source-renovate-branch: &if-mr-source-renovate-branch
  33   if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^renovate\/.*$/'
  34
  35 .if-tag: &if-tag
  36   if: '$CI_COMMIT_TAG'
  37
  38 .if-not-canonical-project: &if-not-canonical-project
  39   # Allow for use on:
  40   #  - https://gitlab.com/gitterHQ/webapp
  41   #  - https://dev.gitlab.org/gitlab/gitter/webapp
  42   if: '$CI_PROJECT_PATH != "gitterHQ/webapp" && $CI_PROJECT_PATH != "gitlab/gitter/webapp"'
  43
  44 .if-schedule: &if-schedule
  45   if: '$CI_PIPELINE_SOURCE == "schedule"'
  46
  47 workflow:
  48   rules:
  49     # For merge requests, create a pipeline.
  50     - <<: *if-merge-request
  51     # For `develop` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
  52     - <<: *if-develop-branch
  53     # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
  54     - <<: *if-master-branch
  55     # For `release/*` branch, create a pipeline.
  56     - <<: *if-release-branch
  57     # For tags, create a pipeline.
  58     - <<: *if-tag
  59
  60 default:
  61   tags:
  62     - gitlab-org
  63
  64 .use-docker-in-docker:
  65   image: docker:${DOCKER_VERSION}
  66   services:
  67     - docker:${DOCKER_VERSION}-dind
  68   variables:
  69     DOCKER_DRIVER: overlay2
  70     DOCKER_HOST: tcp://docker:2375
  71     DOCKER_TLS_CERTDIR: ''
  72   tags:
  73     # See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
  74     - gitlab-org-docker
  75
  76 #########
  77 # Rules #
  78 #########
  79 .rules:staging:
  80   rules:
  81     # Don't run this job for forks since it should only be run from the canonical project.
  82     - <<: *if-not-canonical-project
  83       when: never
  84     - <<: *if-release-branch
  85     - <<: *if-hotfix-branch
  86
  87 .webapp_variables: &webapp_variables
  88   FF_NETWORK_PER_BUILD: 1
  89   BLUEBIRD_DEBUG: 1
  90   NEO4J_AUTH: none
  91   SYNAPSE_SERVER_NAME: my.matrix.host
  92   SYNAPSE_CONFIG_PATH: /data/gitlab-ci-homeserver.yaml
  93
  94 .webapp_services:
  95   - &webapp_service_mongo
  96     name: $GITTER_CI_REGISTRY_IMAGE/mongo:latest
  97     alias: mongo
  98   - &webapp_service_redis
  99     name: redis:3.0.3
 100     alias: redis
 101   - &webapp_service_elasticsearch
 102     name: $GITTER_CI_REGISTRY_IMAGE/elasticsearch:latest
 103     alias: elasticsearch
 104   - &webapp_service_neo4j
 105     name: neo4j:2.3
 106     alias: neo4j
 107   - &webapp_service_synapse
 108     name: $GITTER_CI_REGISTRY_IMAGE/synapse:latest
 109     alias: synapse
 110
 111 .node_job:
 112   image: $GITTER_CI_REGISTRY_IMAGE:latest
 113   before_script:
 114     - node --version
 115     - npm --version
 116     - npm config set prefer-offline true
 117     - npm config set cache ./npm_cache
 118     # Only copy the cache from the Docker container if the cache from GitLab CI is not in place
 119     - '[[ -d ./npm_cache ]] || mv /npm_cache ./npm_cache'
 120     # Install the npm dependencies from scratch to avoid the `graceful-fs` problem from multiple npm installs.
 121     # (old gulp 3 dependency problem with Node.js v14)
 122     # Otherwise, it will throw `ReferenceError: primordials is not defined`.
 123     # See https://stackoverflow.com/a/58394828/796832
 124     #
 125     # We're trying to make this fast by using npm cache.
 126     - npm ci
 127   # Cache modules in between jobs
 128   # Based on https://docs.gitlab.com/ee/ci/caching/#cache-nodejs-dependencies
 129   cache:
 130     # CI_COMMIT_REF_SLUG is the branch or tag name in lowercase
 131     key: ${CI_COMMIT_REF_SLUG}
 132     paths:
 133       - ./npm_cache
 134
 135 validate:
 136   extends: .node_job
 137   stage: build_unit_test
 138   script:
 139     - "echo 'TODO: Skip validation for now because we have new eslint failures'"
 140     - make validate
 141     # TODO: This should be part of `make validate` -> `gulpfile-linter`
 142     # but the prettier API doesn't easily allow glob checking and I want
 143     # to re-use the scripts that GitLab has, https://gitlab.com/gitlab-org/gitlab-ce/issues/57010
 144     - npm run prettier -- --check "**/*.{js,vue}"
 145
 146 .test_job:
 147   extends: .node_job
 148   variables:
 149     <<: *webapp_variables
 150     NODE_ENV: test-docker
 151   stage: build_unit_test
 152   services:
 153     - *webapp_service_mongo
 154     - *webapp_service_redis
 155     - *webapp_service_elasticsearch
 156     - *webapp_service_neo4j
 157     - *webapp_service_synapse
 158   script:
 159     - make ci-test
 160
 161 test:
 162   extends:
 163     - .test_job
 164   rules:
 165     - when: always
 166   retry: 1
 167
 168 test_e2e:
 169   extends: .test_job
 170   variables:
 171     <<: *webapp_variables
 172     ENABLE_FIXTURE_ENDPOINTS: 1
 173     DISABLE_GITHUB_API: 1
 174     NODE_ENV: test-docker
 175     # "fake" dbus address to prevent errors
 176     # https://github.com/SeleniumHQ/docker-selenium/issues/87
 177     # via https://hub.docker.com/r/cypress/browsers/dockerfile
 178     # #oom-chrome-cypress
 179     DBUS_SESSION_BUS_ADDRESS: /dev/null
 180   tags:
 181     # Run the on Gitter internal runner so our Cypress video doesn't have gaps/freezing,
 182     # see https://github.com/cypress-io/cypress/issues/4722#issuecomment-526313109
 183     # Maybe in the future, Cypress will detect and give a warning when this happens,
 184     # see https://github.com/cypress-io/cypress/issues/5061
 185     - internal
 186   rules:
 187     # Don't run this job for forks since it uses the 'internal' runner.
 188     - <<: *if-not-canonical-project
 189       when: never
 190     - <<: *if-mr-source-renovate-branch
 191     - when: manual
 192       allow_failure: true
 193   script:
 194     # install Chromebrowser
 195     # via https://hub.docker.com/r/cypress/browsers/dockerfile
 196     # We are using Chrome because of OOM issues #oom-chrome-cypress, see https://github.com/cypress-io/cypress/issues/350#issuecomment-633700002
 197     - apt-get install -y wget gnupg
 198     - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 199     - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
 200     - apt-get update -q -y
 201     - apt-get install -y dbus-x11 google-chrome-stable
 202     - rm -rf /var/lib/apt/lists/*
 203     # Cypress dependencies https://docs.cypress.io/guides/guides/continuous-integration.html#Dependencies
 204     - apt-get update -q -y
 205     - apt-get --yes install libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
 206     # Create `output/assets/js/vue-ssr-server-bundle.json`
 207     - npm run task-js
 208     # Start the server and wait for it to come up
 209     - mkdir -p logs
 210     - npm start > logs/server-output.txt 2>&1 & node test/e2e/support/wait-for-server.js http://localhost:5000
 211     # Run the tests
 212     # We are using Chrome because of OOM issues #oom-chrome-cypress, see https://github.com/cypress-io/cypress/issues/350#issuecomment-633700002
 213     - npm run test-e2e-run -- --browser chrome --headless
 214   artifacts:
 215     when: always
 216     paths:
 217       - logs
 218       - test/e2e/videos
 219       - test/e2e/screenshots
 220       - cypress/logs
 221     expire_in: 1 day
 222   retry: 2
 223
 224 package:
 225   extends: .node_job
 226   stage: build_unit_test
 227   script:
 228     # While testing the deploment, you can use cached artifacts instead of packaging every time which is slow
 229     #- (apt-get update && apt-get install unzip && curl -Ls https://gitlab.com/gitterHQ/webapp/-/jobs/60049410/artifacts/download -o previous-artifacts.zip && unzip previous-artifacts.zip) || (make package)
 230     - make package
 231   artifacts:
 232     paths:
 233       - output/app.tar.gz
 234       - output/assets.tar.gz
 235       - output/app/ASSET_TAG
 236       - output/app/GIT_COMMIT
 237       - output/app/VERSION
 238     expire_in: 1 week
 239
 240 mobile-asset-build:
 241   extends: .node_job
 242   stage: build_unit_test
 243   rules:
 244     - <<: *if-master-branch
 245   variables:
 246     NODE_ENV: prod
 247     stats__cube__enabled: 'false'
 248     stats__statsd__enabled: 'false'
 249   script:
 250     - npm run task-js
 251     - npm run build-android-assets
 252     - npm run build-ios-assets
 253   artifacts:
 254     paths:
 255       - output/android/www
 256       - output/ios/www
 257
 258 .distribute_job:
 259   image: $GITTER_CI_REGISTRY_IMAGE/deploy-build-image:latest
 260   stage: pre_deploy
 261   dependencies:
 262     - package
 263   script:
 264     - make upload-to-s3
 265
 266 distribute_beta:
 267   extends: .distribute_job
 268   rules:
 269     # Don't run this job for forks since it should only be run from the canonical project.
 270     - <<: *if-not-canonical-project
 271       when: never
 272     - <<: *if-develop-branch
 273   variables:
 274     DIST_S3_URL: s3://gitter-deployments/gitter-webapp/beta
 275
 276 distribute_beta_staging:
 277   extends: .distribute_job
 278   rules:
 279     # Don't run this job for forks since it should only be run from the canonical project.
 280     - <<: *if-not-canonical-project
 281       when: never
 282     - when: always
 283   variables:
 284     DIST_S3_URL: s3://gitter-deployments/gitter-webapp/beta-staging
 285
 286 distribute_staging:
 287   extends:
 288     - .distribute_job
 289     - .rules:staging
 290   variables:
 291     DIST_S3_URL: s3://gitter-deployments/gitter-webapp/staging
 292
 293 distribute_prod:
 294   extends: .distribute_job
 295   rules:
 296     # Don't run this job for forks since it should only be run from the canonical project.
 297     - <<: *if-not-canonical-project
 298       when: never
 299     - <<: *if-tag
 300   variables:
 301     DIST_S3_URL: s3://gitter-deployments/gitter-webapp/prod
 302
 303 containerize:
 304   extends: .use-docker-in-docker
 305   stage: deploy
 306   dependencies:
 307     - package
 308   allow_failure: true
 309   script:
 310     - ls -la output
 311     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 312     - docker build -t $GITTER_CI_REGISTRY_IMAGE/app:$CI_COMMIT_REF_SLUG -f Dockerfile-app-base .
 313     - docker push $GITTER_CI_REGISTRY_IMAGE/app:$CI_COMMIT_REF_SLUG
 314
 315 deploy-build-image:
 316   extends: .use-docker-in-docker
 317   stage: docker_images
 318   rules:
 319     - <<: *if-schedule
 320   script:
 321     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 322     - docker build -t $GITTER_CI_REGISTRY_IMAGE/deploy-build-image:latest scripts/docker/deploy-build-image/
 323     - docker push $GITTER_CI_REGISTRY_IMAGE/deploy-build-image:latest
 324
 325 .deploy_job:
 326   stage: deploy
 327   image: $GITTER_CI_REGISTRY_IMAGE/deploy-build-image:latest
 328   variables:
 329     GIT_STRATEGY: none
 330   before_script:
 331     # run ssh-agent
 332     - eval $(ssh-agent -s)
 333     # add ssh key stored in SSH_PRIVATE_KEY variable to the agent store
 334     - ssh-add <(echo "$DEPLOY_KEY_ANSIBLE_REPO")
 335     - ssh-add <(echo "$INTERNAL_GITTER_NETWORK_SSH_KEY")
 336     - mkdir -p ~/.ssh
 337     - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
 338     # Make the infra tools available (like Ansible)
 339     - git clone git@gitlab.com:gitlab-com/gl-infra/gitter-infrastructure.git
 340     - ANSIBLE_DIR=$(cd gitter-infrastructure/ansible && pwd) && echo $ANSIBLE_DIR
 341     - mkdir -p /root && echo "$ANSIBLE_VAULT_PASS" > /root/.vault_pass
 342   tags:
 343     - internal # This has to be within the Gitter network
 344
 345 deploy_beta:
 346   extends: .deploy_job
 347   dependencies:
 348     - distribute_beta
 349   rules:
 350     # Don't run this job for forks since it uses the 'internal' runner.
 351     - <<: *if-not-canonical-project
 352       when: never
 353     - <<: *if-develop-branch
 354   script:
 355     - cd $ANSIBLE_DIR && ansible-playbook -i beta --vault-password-file "/root/.vault_pass" playbooks/gitter/webapp-deploy.yml
 356   environment:
 357     name: beta
 358     url: https://beta.gitter.im
 359
 360 deploy_beta_staging:
 361   extends: .deploy_job
 362   dependencies:
 363     - distribute_beta_staging
 364   rules:
 365     # Don't run this job for forks since it uses the 'internal' runner.
 366     - <<: *if-not-canonical-project
 367       when: never
 368     - when: manual
 369       allow_failure: true
 370   script:
 371     - cd $ANSIBLE_DIR && ansible-playbook -i beta --vault-password-file "/root/.vault_pass" playbooks/gitter/webapp-staging-deploy.yml
 372   environment:
 373     name: beta_staging
 374     url: https://beta.gitter.im?gitter_next=true
 375
 376 deploy_staging:
 377   extends:
 378     - .deploy_job
 379     - .rules:staging
 380   dependencies:
 381     - distribute_staging
 382   script:
 383     - cd $ANSIBLE_DIR && ansible-playbook -i prod --vault-password-file "/root/.vault_pass" playbooks/gitter/webapp-staging-deploy.yml
 384   environment:
 385     name: staging
 386     url: https://gitter.im?gitter_next=true
 387
 388 deploy_prod:
 389   extends: .deploy_job
 390   dependencies:
 391     - distribute_prod
 392   rules:
 393     # Don't run this job for forks since it uses the 'internal' runner.
 394     - <<: *if-not-canonical-project
 395       when: never
 396     - <<: *if-tag
 397       when: manual
 398   script:
 399     - cd $ANSIBLE_DIR && ansible-playbook -i prod --vault-password-file "/root/.vault_pass" playbooks/gitter/webapp-deploy.yml
 400   environment:
 401     name: prod
 402     url: https://gitter.im
 403
 404 docker-base:
 405   extends: .use-docker-in-docker
 406   stage: docker_images
 407   rules:
 408     - <<: *if-schedule
 409   script:
 410     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 411     - docker build -t $GITTER_CI_REGISTRY_IMAGE:latest .
 412     - docker push $GITTER_CI_REGISTRY_IMAGE:latest
 413
 414 mongo-base:
 415   extends: .use-docker-in-docker
 416   stage: docker_images
 417   rules:
 418     - <<: *if-schedule
 419   script:
 420     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 421     - docker build -t $GITTER_CI_REGISTRY_IMAGE/mongo:latest scripts/docker/mongo-image/
 422     - docker push $GITTER_CI_REGISTRY_IMAGE/mongo:latest
 423
 424 elasticsearch-base:
 425   extends: .use-docker-in-docker
 426   stage: docker_images
 427   rules:
 428     - <<: *if-schedule
 429   script:
 430     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 431     - docker build -t $GITTER_CI_REGISTRY_IMAGE/elasticsearch:latest scripts/docker/elasticsearch-image/
 432     - docker push $GITTER_CI_REGISTRY_IMAGE/elasticsearch:latest
 433
 434 synapse-base:
 435   extends: .use-docker-in-docker
 436   stage: docker_images
 437   rules:
 438     - <<: *if-schedule
 439   script:
 440     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com
 441     - docker build -t $GITTER_CI_REGISTRY_IMAGE/synapse:latest scripts/docker/matrix/synapse/
 442     - docker push $GITTER_CI_REGISTRY_IMAGE/synapse:latest
 443
 444 renovate:
 445   extends: .use-docker-in-docker
 446   stage: docker_images
 447   rules:
 448     - <<: *if-schedule
 449       allow_failure: true
 450   variables:
 451     # GitLab access token with api access
 452     #RENOVATE_GITLAB_TOKEN: xxx
 453     # GitHub access token for getting release notes of package updates,
 454     # https://github.com/renovatebot/renovate/blob/master/docs/self-hosting.md#githubcom-token-for-release-notes
 455     #RENOVATE_GITHUB_TOKEN: xxx
 456     RENOVATE_CONFIG_FILE: renovate-config.js
 457   script:
 458     - docker run -e GITLAB_TOKEN="$RENOVATE_GITLAB_TOKEN" -e GITHUB_COM_TOKEN="$RENOVATE_GITHUB_TOKEN" -v $PWD/renovate.json:/usr/src/app/renovate.json -v $PWD/renovate-config.js:/usr/src/app/config.js renovate/renovate:13 gitterHQ/webapp
 459 #
 460 # Security reports
 461 # =========================================================
 462 include:
 463   - template: Security/SAST.gitlab-ci.yml
 464   - template: Security/DAST.gitlab-ci.yml
 465   - template: Security/Dependency-Scanning.gitlab-ci.yml
 466   - template: Security/Container-Scanning.gitlab-ci.yml
 467   - template: Security/Secret-Detection.gitlab-ci.yml
 468
 469 dependency_scanning:
 470   stage: security
 471   variables:
 472     SECURE_LOG_LEVEL: debug
 473     RETIREJS_ADVISORY_DB_INSECURE: 'true'
 474
 475 gemnasium-dependency_scanning:
 476   rules:
 477     - when: always
 478
 479 retire-js-dependency_scanning:
 480   rules:
 481     - when: always
 482
 483 container_scanning:
 484   stage: security
 485   rules:
 486     - when: always
 487   variables:
 488     DOCKERFILE_PATH: Dockerfile-app-base
 489     GIT_STRATEGY: fetch
 490     CI_APPLICATION_REPOSITORY: $GITTER_CI_REGISTRY_IMAGE/app
 491     CI_APPLICATION_TAG: $CI_COMMIT_REF_SLUG
 492
 493 sast:
 494   stage: security
 495   variables:
 496     SAST_EXCLUDED_PATHS: '/test/**,**/*-test.js,/scripts,/build-scripts'
 497     SEARCH_MAX_DEPTH: 4
 498
 499 eslint-sast:
 500   rules:
 501     - when: always
 502
 503 nodejs-scan-sast:
 504   rules:
 505     - when: always
 506
 507 semgrep-sast:
 508   rules:
 509     - when: always
 510
 511 dast:
 512   stage: security
 513   rules:
 514     - when: always
 515   dependencies:
 516     - containerize
 517   variables:
 518     <<: *webapp_variables
 519     NODE_ENV: test-docker
 520     DAST_WEBSITE: http://webapp-server:5000/
 521     # Pass-through webapp secrets from our project-level secret variables
 522     web__sessionSecret: '${web__sessionSecret}'
 523     web__statePassphrase: '${web__statePassphrase}'
 524     ws__superClientPassword: '${ws__superClientPassword}'
 525     web__messageSecret: '${web__messageSecret}'
 526     email__unsubscribeNotificationsSecret: '${email__unsubscribeNotificationsSecret}'
 527     integrations__secret: '${integrations__secret}'
 528     github__statePassphrase: '${github__statePassphrase}'
 529     twitteroauth__consumer_key: '${twitteroauth__consumer_key}'
 530     twitteroauth__consumer_secret: '${twitteroauth__consumer_secret}'
 531     gitlaboauth__client_id: '${gitlaboauth__client_id}'
 532     gitlaboauth__client_secret: '${gitlaboauth__client_secret}'
 533     gitlab__public_token_pool: '${gitlab__public_token_pool}'
 534     github__client_id: '${github__client_id}'
 535     github__client_secret: '${github__client_secret}'
 536     github__user_client_id: '${github__user_client_id}'
 537     github__user_client_secret: '${github__user_client_secret}'
 538     github__anonymous_app__client_id: '${github__anonymous_app__client_id}'
 539     github__anonymous_app__client_secret: '${github__anonymous_app__client_secret}'
 540     tokens__anonymousPassword: '${tokens__anonymousPassword}'
 541   before_script:
 542     # Used to communicate with the other services and share the other hostnames we can connect to
 543     # See https://gitlab.com/gitlab-org/gitlab-runner/issues/1042#note_144420147
 544     - cp /etc/hosts "${CI_PROJECT_DIR}/"
 545   services:
 546     - name: $GITTER_CI_REGISTRY_IMAGE/app:$CI_COMMIT_REF_SLUG
 547       alias: webapp-server
 548       # Used to communicate with the other services and share the other hostnames we can connect to
 549       # See https://gitlab.com/gitlab-org/gitlab-runner/issues/1042#note_144420147
 550       entrypoint:
 551         [
 552           '/bin/sh',
 553           '-c',
 554           'while [ ! -f /$CI_PROJECT_DIR/hosts ]; do sleep 1; done && cat /$CI_PROJECT_DIR/hosts > /etc/hosts && node web.js',
 555         ]
 556     - *webapp_service_mongo
 557     - *webapp_service_redis
 558     - *webapp_service_elasticsearch
 559     - *webapp_service_neo4j
 560     - *webapp_service_synapse
 561
 562 .secret-analyzer:
 563   extends: .use-docker-in-docker
 564   stage: security
 565
 566 secret_detection:
 567   rules:
 568     - when: always