.gitlab-ci.yml

   1 stages:
   2   - test
   3   - build-website
   4   - deploy-website
   5
   6 workflow:
   7   rules:
   8     - if: $CI_FORCE_RUN
   9     - if: $CI_MERGE_REQUEST_IID
  10     - if: $CI_COMMIT_TAG
  11     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
  12
  13 # Please, make sure to only use images from `registry.gitlab.tails.boum.org`,
  14 # as we give significant privileges to our Runners wrt. what they can do in our
  15 # repositories.
  16 image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:bookworm
  17
  18 variables:
  19     GET_SOURCES_ATTEMPTS: 10
  20
  21 before_script:
  22   - export DEBIAN_FRONTEND=noninteractive
  23   - apt-get update -qq
  24
  25 .prepare-lint-po: &prepare-lint-po
  26   - apt-get -qy install git i18nspector
  27   - git clone https://gitlab.tails.boum.org/tails/jenkins-tools.git /tmp/jenkins-tools
  28
  29 build-website:
  30   stage: build-website
  31   rules:
  32     - if: '$CI_COMMIT_BRANCH == "master"'
  33       variables:
  34         IKIWIKI_RCS_OPTION: --rcs git
  35     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
  36     - changes:
  37         - .gitlab-ci.yml
  38   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/ikiwiki:bookworm
  39   cache:
  40     key: website-$CI_COMMIT_REF_SLUG
  41     paths:
  42       - config/chroot_local-includes/usr/share/doc/tails/website
  43       - wiki/src/.ikiwiki
  44       - underlays
  45   artifacts:
  46     name: website
  47     paths:
  48       - config/chroot_local-includes/usr/share/doc/tails/website
  49     when: on_success
  50     expire_in: 1 day
  51   variables:
  52     UNDERLAYS_DIR: ${CI_PROJECT_DIR}/underlays
  53   script:
  54     - mkdir -p ${UNDERLAYS_DIR}
  55     - |
  56       for i in etcher-binary promotion-material; do
  57         test -d ${UNDERLAYS_DIR}/${i} || git clone --depth=1 https://gitlab.tails.boum.org/tails/${i}.git ${UNDERLAYS_DIR}/${i}
  58         git -C ${UNDERLAYS_DIR}/${i} switch master
  59         git -C ${UNDERLAYS_DIR}/${i} pull
  60       done
  61     - |
  62       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
  63         git checkout -B master $CI_COMMIT_SHA
  64         git branch -u origin/master
  65         git remote set-url origin https://user:$WEBSITE_BUILD_PROJECT_ACCESS_TOKEN@gitlab.tails.boum.org/tails/tails.git
  66       fi
  67     - |
  68       ./build-website ${IKIWIKI_RCS_OPTION} \
  69         --url "https://tails.net" \
  70         --usedirs \
  71         --sslcookie \
  72         --historyurl "https://gitlab.tails.boum.org/tails/tails/-/commits/master/wiki/src/[[file]]" \
  73         --diffurl "https://gitlab.tails.boum.org/tails/tails/-/commit/[[sha1_commit]]" \
  74         --rss \
  75         --atom \
  76         --set exclude='^misc\/.*|\.(ar|fa|id|pl|sr_Latn|tr|zh|zh_TW)\.po$' \
  77         --include '^contribute\/how\/promote\/material\/.*' \
  78         --set-yaml add_underlays="['${UNDERLAYS_DIR}/etcher-binary','${UNDERLAYS_DIR}/promotion-material']"
  79
  80 deploy-website:
  81   stage: deploy-website
  82   rules:
  83     - if: '$CI_COMMIT_BRANCH == "master"'
  84   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/rsync:bookworm
  85   dependencies:
  86     - build-website
  87   environment: production
  88   resource_group: production/$SSH_SERVER
  89   parallel:
  90     matrix:
  91       - SSH_SERVER: www.lizard
  92         SSH_PORT: 3009
  93       - SSH_SERVER: www2.chameleon
  94         SSH_PORT: 22
  95   script:
  96     - apt-get install -y openssh-client rsync
  97     - test -e .ssh || mkdir .ssh
  98     - cp "$WEBSITE_DEPLOY_SSH_PRIVATE_KEY" .ssh/private_key
  99     - cp "$WEBSITE_DEPLOY_SSH_KNOWN_HOSTS" .ssh/known_hosts
 100     - chmod 400 .ssh/known_hosts .ssh/private_key
 101     - echo "variables often lack a trailing newline, which breaks SSH, detect and fix"
 102     - ssh-keygen -y -f .ssh/private_key || echo >> .ssh/private_key
 103     - echo "here is the SSH key we will deploy with"
 104     - ssh-keygen -y -f .ssh/private_key
 105     - echo -n "Begin rsync, time is " && date '+%Y-%m-%d %H-%M-%S%z'
 106     - rsync --rsh="ssh -p ${SSH_PORT} -o UserKnownHostsFile=.ssh/known_hosts -i .ssh/private_key" --checksum --archive --no-times --verbose --mkpath --delete "config/chroot_local-includes/usr/share/doc/tails/website/" www-data@${SSH_SERVER}:/
 107
 108 lint-po:
 109   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:trixie
 110   rules:
 111     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 112     - changes:
 113         - .gitlab-ci.yml
 114         - ./**.po
 115   script:
 116     - *prepare-lint-po
 117     - /tmp/jenkins-tools/slaves/lint_po
 118
 119 ruff-lint-changed-files:
 120   only:
 121     - merge_requests
 122
 123   variables:
 124     GIT_STRATEGY: clone
 125     GIT_DEPTH: 0
 126
 127   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 128
 129   script:
 130     - apt-get -qy install ruff findutils git python3
 131     - ruff --version
 132     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 133     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" check --output-format=junit --output-file=ruff.xml
 134   artifacts:
 135     when: always
 136     reports:
 137       junit: ruff.xml
 138
 139 ruff-format-changed-files:
 140   only:
 141     - merge_requests
 142
 143   variables:
 144     GIT_STRATEGY: clone
 145     GIT_DEPTH: 0
 146
 147   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 148
 149   script:
 150     - apt-get -qy install ruff findutils git python3
 151     - ruff --version
 152     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 153     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" format --check
 154
 155 check-website-core-pages:
 156   script:
 157     - apt-get -qy install git
 158     - ./bin/check-core-pages
 159
 160 check-po-msgfmt:
 161   rules:
 162     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 163     - changes:
 164         - .gitlab-ci.yml
 165         - ./**.po
 166   script:
 167     - apt-get -qy install python3 gettext
 168     - ./bin/check-po-msgfmt
 169
 170 check-po-meta-date:
 171   rules:
 172     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 173     - changes:
 174         - .gitlab-ci.yml
 175         - ./**.po
 176   script:
 177     - apt-get -qy install git ruby
 178     - ./bin/sanity-check-website
 179
 180 check-translatable-live-website-urls:
 181   script:
 182     - apt-get -qy install python3-polib
 183     - ./bin/check-translatable-live-website-urls po/tails.pot
 184
 185 check-locale-descriptions:
 186   script:
 187     - apt-get -qy install python3 python3-requests python3-toml python3-bs4
 188     - echo 'If this fails, look at https://tails.net/contribute/release_process/update_locale_descriptions/'
 189     - ./bin/locale-descriptions suggest
 190
 191 rubocop:
 192   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:bookworm
 193   script:
 194   - apt-get -qy install rubocop
 195   - rubocop --version
 196   - rubocop --format junit --out rubocop.xml --format markdown
 197   artifacts:
 198     when: always
 199     reports:
 200       junit: rubocop.xml
 201
 202 test-iuk:
 203   rules:
 204     - if: '$CI_COMMIT_BRANCH != "master"'
 205   script:
 206   - './bin/test-utils/test-iuk'
 207
 208 test-perl5lib:
 209   rules:
 210     - if: '$CI_COMMIT_BRANCH != "master"'
 211   script:
 212   - 'cat config/chroot_local-packageslists/tails-perl5lib.list
 213        | grep -E -v "^#"
 214        | xargs apt-get -qy install'
 215   - 'apt-get -qy install
 216        apt-file
 217        libdist-zilla-plugin-test-notabs-perl
 218        libdist-zilla-plugin-test-perl-critic-perl
 219        libdist-zilla-app-command-authordebs-perl
 220        libmodule-build-perl
 221        sudo'
 222   - apt-get update -qq # Take into account APT configuration added by apt-file
 223   # Otherwise, apt-get called by "dzil authordebs --install" asks confirmation
 224   - echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/yes
 225   - cd $CI_PROJECT_DIR/config/chroot_local-includes/usr/src/perl5lib
 226   - dzil authordebs --install
 227   - dzil test --all
 228
 229 shellcheck:
 230   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:trixie
 231   script:
 232   - apt-get -qy install python3 shellcheck xmlstarlet git
 233   - shellcheck --version
 234   - 'git ls-files -z | ./bin/test-utils/is-file-type filter --zero shell | xargs --verbose --no-run-if-empty -0 shellcheck --format=checkstyle
 235        | xmlstarlet tr config/ci/shellcheck/checkstyle2junit.xslt
 236        > shellcheck.xml'
 237   artifacts:
 238     when: always
 239     reports:
 240       junit: shellcheck.xml
 241
 242 test-persistent-storage-config-file:
 243   script:
 244     - apt-get -qy install python3 python3-gi acl
 245     - config/chroot_local-includes/usr/lib/python3/dist-packages/tps/configuration/config_file_test.py
 246
 247 test-python-doctest:
 248   script:
 249     - apt-get -qy install python3 python3-sh python3-toml python3-requests python3-bs4
 250     - config/chroot_local-includes/usr/local/lib/tails-gdm-error-message doctest --verbose
 251     - env PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages python3 config/chroot_local-includes/usr/local/bin/tails-documentation --doctest
 252     - ./bin/locale-descriptions doctest
 253
 254 test-tca:
 255   rules:
 256     - if: '$CI_COMMIT_BRANCH != "master"'
 257   script:
 258     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 259        | grep -E -v "^#"
 260        | xargs apt-get -qy install'
 261     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tca -name "*.py" -print0 | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 262
 263 test-tca-portal:
 264   rules:
 265     - if: '$CI_COMMIT_BRANCH != "master"'
 266   script:
 267     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 268        | grep -E -v "^#"
 269        | xargs apt-get -qy install'
 270     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages env python3 ./config/chroot_local-includes/usr/local/lib/tca-portal --doctest-only --log-level DEBUG'
 271
 272
 273 test-tailslib:
 274   rules:
 275     - if: '$CI_COMMIT_BRANCH != "master"'
 276   script:
 277     - apt-get -qy install python3 python3-atomicwrites python3-sh python3-gi git
 278     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tailslib -name "*.py" -print0 | grep --null-data -v -e netnsdrop.py -e gnome.py | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 279
 280 test-whisperback:
 281   rules:
 282     - if: '$CI_COMMIT_BRANCH != "master"'
 283   script:
 284     - 'cat config/chroot_local-packageslists/whisperback.list | grep -E -v "^#"
 285          | xargs apt-get -qy install'
 286     - apt-get -qy install python3-pytest
 287     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages
 288          pytest-3 --verbose --junit-xml=report.xml
 289          config/chroot_local-includes/usr/lib/python3/dist-packages/whisperBack/test.py'
 290   artifacts:
 291     when: always
 292     reports:
 293       junit: report.xml
 294
 295 apt-snapshots-expiry:
 296   script:
 297     - apt-get -qy install curl git
 298     - ./bin/apt-snapshots-expiry
 299   rules:
 300     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 301     - changes:
 302         - .gitlab-ci.yml
 303         - config/APT_snapshots.d/*/serial
 304         - vagrant/definitions/tails-builder/config/APT_snapshots.d/*/serial
 305
 306 .install-https-get-expired-build-deps: &install-https-get-expired-build-deps
 307   - apt-get -qy install --no-install-recommends golang-go ca-certificates
 308
 309 .build-https-get-expired: &build-https-get-expired
 310   - go build -o ./https-get-expired config/chroot_local-includes/usr/src/https-get-expired.go
 311
 312 .test-https-get-expired: &test-https-get-expired
 313   - echo "Basic check:"
 314   - ./https-get-expired -reject-expired https://tails.net/
 315   - echo "Let's pretend we are in the past. Then, this certificate is still good."
 316   - ./https-get-expired -current-time 2000-01-01 -reject-expired https://tails.net/
 317   - echo "Let's pretend we are in the future. Then, this certificate is expired"
 318   - "! ./https-get-expired -current-time 2090-01-01 -reject-expired https://tails.net/"
 319   - "! ./https-get-expired -reject-expired https://wrong.host.badssl.com/"
 320   - "! ./https-get-expired -reject-expired https://self-signed.badssl.com/"
 321   - "! ./https-get-expired -reject-expired https://untrusted-root.badssl.com/"
 322   - "! ./https-get-expired -reject-expired https://expired.badssl.com/"
 323   - echo "Invalid host"
 324   - "! ./https-get-expired -reject-expired https://nxdomain.tails.net/"
 325   - "./bin/test-utils/https-get-expired-test-all"
 326
 327 https-get-expired:
 328   rules:
 329     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 330     - changes:
 331         - .gitlab-ci.yml
 332         - config/chroot_local-includes/usr/src/https-get-expired.go
 333         - config/chroot_local-includes/etc/default/htpdate.pools
 334   script:
 335     - *install-https-get-expired-build-deps
 336     - *build-https-get-expired
 337     - *test-https-get-expired
 338
 339 https-get-expired-sid:
 340   # this job gives us results using a future version of Golang compared to the one we actually use
 341   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 342   rules:
 343     - if: '$CI_COMMIT_BRANCH == "devel"'
 344     - changes:
 345         - .gitlab-ci.yml
 346         - config/chroot_local-includes/usr/src/https-get-expired.go
 347         - config/chroot_local-includes/etc/default/htpdate.pools
 348   script:
 349     - *install-https-get-expired-build-deps
 350     - *build-https-get-expired
 351     - *test-https-get-expired