.gitlab-ci.yml

   1 workflow:
   2   rules:
   3     - if: $CI_FORCE_RUN
   4     - if: $CI_MERGE_REQUEST_IID
   5     - if: $CI_COMMIT_TAG
   6     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
   7
   8 # Please, make sure to only use images from `registry.gitlab.tails.boum.org`,
   9 # as we give significant privileges to our Runners wrt. what they can do in our
  10 # repositories.
  11 image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:bookworm
  12
  13 variables:
  14     GET_SOURCES_ATTEMPTS: 10
  15
  16 before_script:
  17   - export DEBIAN_FRONTEND=noninteractive
  18   - apt-get update -qq
  19
  20 .prepare-lint-po: &prepare-lint-po
  21   - apt-get -qy install git i18nspector
  22   - git clone https://gitlab.tails.boum.org/tails/jenkins-tools.git /tmp/jenkins-tools
  23
  24 build-website:
  25   rules:
  26     - if: '$CI_COMMIT_BRANCH == "master"'
  27       variables:
  28         IKIWIKI_RCS_OPTION: --rcs git
  29     - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
  30     - changes:
  31         - .gitlab-ci.yml
  32   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/ikiwiki:bookworm
  33   cache:
  34     key: website-$CI_COMMIT_REF_SLUG
  35     paths:
  36       - config/chroot_local-includes/usr/share/doc/tails/website
  37       - wiki/src/.ikiwiki
  38       - underlays
  39   artifacts:
  40     name: website
  41     paths:
  42       - config/chroot_local-includes/usr/share/doc/tails/website
  43     when: on_success
  44     expire_in: 1 day
  45   variables:
  46     UNDERLAYS_DIR: ${CI_PROJECT_DIR}/underlays
  47   script:
  48     - mkdir -p ${UNDERLAYS_DIR}
  49     - |
  50       for i in etcher-binary promotion-material; do
  51         test -d ${UNDERLAYS_DIR}/${i} || git clone --depth=1 https://gitlab.tails.boum.org/tails/${i}.git ${UNDERLAYS_DIR}/${i}
  52         git -C ${UNDERLAYS_DIR}/${i} switch master
  53         git -C ${UNDERLAYS_DIR}/${i} pull
  54       done
  55     - |
  56       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
  57         git checkout -B master $CI_COMMIT_SHA
  58         git branch -u origin/master
  59         git remote set-url origin https://user:$WEBSITE_BUILD_PROJECT_ACCESS_TOKEN@gitlab.tails.boum.org/tails/tails.git
  60       fi
  61     - |
  62       ./build-website ${IKIWIKI_RCS_OPTION} \
  63         --url "https://tails.net" \
  64         --usedirs \
  65         --sslcookie \
  66         --historyurl "https://gitlab.tails.boum.org/tails/tails/-/commits/master/wiki/src/[[file]]" \
  67         --diffurl "https://gitlab.tails.boum.org/tails/tails/-/commit/[[sha1_commit]]" \
  68         --rss \
  69         --atom \
  70         --set exclude='^misc\/.*|\.(ar|fa|id|pl|sr_Latn|tr|zh|zh_TW)\.po$' \
  71         --include '^contribute\/how\/promote\/material\/.*' \
  72         --set-yaml add_underlays="['${UNDERLAYS_DIR}/etcher-binary','${UNDERLAYS_DIR}/promotion-material']"
  73
  74 deploy-website:
  75   needs: [build-website, lint-po, check-website-core-pages, check-po-msgfmt, check-po-meta-date, check-translatable-live-website-urls]
  76   rules:
  77     - if: '$CI_COMMIT_BRANCH == "master"'
  78   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/rsync:bookworm
  79   dependencies:
  80     - build-website
  81   environment: production
  82   resource_group: production/$SSH_SERVER
  83   parallel:
  84     matrix:
  85       - SSH_SERVER: www.lizard
  86         SSH_PORT: 3009
  87       - SSH_SERVER: www2.chameleon
  88         SSH_PORT: 22
  89   script:
  90     - apt-get install -y openssh-client rsync
  91     - test -e .ssh || mkdir .ssh
  92     - cp "$WEBSITE_DEPLOY_SSH_PRIVATE_KEY" .ssh/private_key
  93     - cp "$WEBSITE_DEPLOY_SSH_KNOWN_HOSTS" .ssh/known_hosts
  94     - chmod 400 .ssh/known_hosts .ssh/private_key
  95     - echo "variables often lack a trailing newline, which breaks SSH, detect and fix"
  96     - ssh-keygen -y -f .ssh/private_key || echo >> .ssh/private_key
  97     - echo "here is the SSH key we will deploy with"
  98     - ssh-keygen -y -f .ssh/private_key
  99     - echo -n "Begin rsync, time is " && date '+%Y-%m-%d %H-%M-%S%z'
 100     - rsync --rsh="ssh -p ${SSH_PORT} -o UserKnownHostsFile=.ssh/known_hosts -i .ssh/private_key" --checksum --archive --no-times --verbose --mkpath --delete "config/chroot_local-includes/usr/share/doc/tails/website/" www-data@${SSH_SERVER}:/
 101
 102 lint-po:
 103   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:trixie
 104   rules:
 105     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 106     - changes:
 107         - .gitlab-ci.yml
 108         - ./**.po
 109   script:
 110     - *prepare-lint-po
 111     - /tmp/jenkins-tools/slaves/lint_po
 112
 113 ruff-lint-changed-files:
 114   only:
 115     - merge_requests
 116
 117   variables:
 118     GIT_STRATEGY: clone
 119     GIT_DEPTH: 0
 120
 121   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 122
 123   script:
 124     - apt-get -qy install ruff findutils git python3
 125     - ruff --version
 126     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 127     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" check --output-format=junit --output-file=ruff.xml
 128   artifacts:
 129     when: always
 130     reports:
 131       junit: ruff.xml
 132
 133 ruff-format-changed-files:
 134   only:
 135     - merge_requests
 136
 137   variables:
 138     GIT_STRATEGY: clone
 139     GIT_DEPTH: 0
 140
 141   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 142
 143   script:
 144     - apt-get -qy install ruff findutils git python3
 145     - ruff --version
 146     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 147     - ./bin/test-utils/ruff "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}" format --check
 148
 149 check-website-core-pages:
 150   script:
 151     - apt-get -qy install git
 152     - ./bin/check-core-pages
 153
 154 check-po-msgfmt:
 155   rules:
 156     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 157     - changes:
 158         - .gitlab-ci.yml
 159         - ./**.po
 160   script:
 161     - apt-get -qy install python3 gettext
 162     - ./bin/check-po-msgfmt
 163
 164 check-po-meta-date:
 165   rules:
 166     - if: '$CI_COMMIT_BRANCH =~ /^master|stable|testing|devel$/'
 167     - changes:
 168         - .gitlab-ci.yml
 169         - ./**.po
 170   script:
 171     - apt-get -qy install git ruby
 172     - ./bin/sanity-check-website
 173
 174 check-translatable-live-website-urls:
 175   script:
 176     - apt-get -qy install python3-polib
 177     - ./bin/check-translatable-live-website-urls po/tails.pot
 178
 179 check-locale-descriptions:
 180   script:
 181     - apt-get -qy install python3 python3-requests python3-toml python3-bs4
 182     - echo 'If this fails, look at https://tails.net/contribute/release_process/update_locale_descriptions/'
 183     - ./bin/locale-descriptions suggest
 184
 185 rubocop:
 186   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:bookworm
 187   script:
 188   - apt-get -qy install rubocop
 189   - rubocop --version
 190   - rubocop --format junit --out rubocop.xml --format markdown
 191   artifacts:
 192     when: always
 193     reports:
 194       junit: rubocop.xml
 195
 196 test-iuk:
 197   rules:
 198     - if: '$CI_COMMIT_BRANCH != "master"'
 199   script:
 200   - './bin/test-utils/test-iuk'
 201
 202 test-perl5lib:
 203   rules:
 204     - if: '$CI_COMMIT_BRANCH != "master"'
 205   script:
 206   - 'cat config/chroot_local-packageslists/tails-perl5lib.list
 207        | grep -E -v "^#"
 208        | xargs apt-get -qy install'
 209   - 'apt-get -qy install
 210        apt-file
 211        libdist-zilla-plugin-test-notabs-perl
 212        libdist-zilla-plugin-test-perl-critic-perl
 213        libdist-zilla-app-command-authordebs-perl
 214        libmodule-build-perl
 215        sudo'
 216   - apt-get update -qq # Take into account APT configuration added by apt-file
 217   # Otherwise, apt-get called by "dzil authordebs --install" asks confirmation
 218   - echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/yes
 219   - cd $CI_PROJECT_DIR/config/chroot_local-includes/usr/src/perl5lib
 220   - dzil authordebs --install
 221   - dzil test --all
 222
 223 shellcheck:
 224   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:trixie
 225   script:
 226   - apt-get -qy install python3 shellcheck xmlstarlet git
 227   - shellcheck --version
 228   - 'git ls-files -z | ./bin/test-utils/is-file-type filter --zero shell | xargs --verbose --no-run-if-empty -0 shellcheck --format=checkstyle
 229        | xmlstarlet tr config/ci/shellcheck/checkstyle2junit.xslt
 230        > shellcheck.xml'
 231   artifacts:
 232     when: always
 233     reports:
 234       junit: shellcheck.xml
 235
 236 shfmt-changed-files:
 237   only:
 238     - merge_requests
 239
 240   variables:
 241     GIT_STRATEGY: clone
 242     GIT_DEPTH: 0
 243
 244   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 245
 246   script:
 247     - apt-get -qy install shfmt findutils git python3
 248     - shfmt --version
 249     - git fetch origin "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 250     - ./bin/test-utils/shfmt -s "origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:?}"
 251
 252 test-persistent-storage-config-file:
 253   script:
 254     - apt-get -qy install python3 python3-gi acl
 255     - config/chroot_local-includes/usr/lib/python3/dist-packages/tps/configuration/config_file_test.py
 256
 257 test-python-doctest:
 258   script:
 259     - apt-get -qy install python3 python3-sh python3-toml python3-requests python3-bs4
 260     - config/chroot_local-includes/usr/local/lib/tails-gdm-error-message doctest --verbose
 261     - env PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages python3 config/chroot_local-includes/usr/local/bin/tails-documentation --doctest
 262     - ./bin/locale-descriptions doctest
 263
 264 test-tca:
 265   rules:
 266     - if: '$CI_COMMIT_BRANCH != "master"'
 267   script:
 268     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 269        | grep -E -v "^#"
 270        | xargs apt-get -qy install'
 271     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tca -name "*.py" -print0 | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 272
 273 test-tca-portal:
 274   rules:
 275     - if: '$CI_COMMIT_BRANCH != "master"'
 276   script:
 277     - 'cat config/chroot_local-packageslists/tor-connection-assistant.list
 278        | grep -E -v "^#"
 279        | xargs apt-get -qy install'
 280     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages env python3 ./config/chroot_local-includes/usr/local/lib/tca-portal --doctest-only --log-level DEBUG'
 281
 282
 283 test-tailslib:
 284   rules:
 285     - if: '$CI_COMMIT_BRANCH != "master"'
 286   script:
 287     - apt-get -qy install python3 python3-atomicwrites python3-sh python3-gi git
 288     - 'cd config/chroot_local-includes/usr/lib/python3/dist-packages ; find tailslib -name "*.py" -print0 | grep --null-data -v -e netnsdrop.py -e gnome.py | xargs -0 -L1 env PYTHONPATH=. python3 -m doctest'
 289
 290 test-whisperback:
 291   rules:
 292     - if: '$CI_COMMIT_BRANCH != "master"'
 293   script:
 294     - 'cat config/chroot_local-packageslists/whisperback.list | grep -E -v "^#"
 295          | xargs apt-get -qy install'
 296     - apt-get -qy install python3-pytest
 297     - 'PYTHONPATH=config/chroot_local-includes/usr/lib/python3/dist-packages
 298          pytest-3 --verbose --junit-xml=report.xml
 299          config/chroot_local-includes/usr/lib/python3/dist-packages/whisperBack/test.py'
 300   artifacts:
 301     when: always
 302     reports:
 303       junit: report.xml
 304
 305 apt-snapshots-expiry:
 306   script:
 307     - apt-get -qy install curl git
 308     - ./bin/apt-snapshots-expiry
 309   rules:
 310     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 311     - changes:
 312         - .gitlab-ci.yml
 313         - config/APT_snapshots.d/*/serial
 314         - vagrant/definitions/tails-builder/config/APT_snapshots.d/*/serial
 315
 316 .install-https-get-expired-build-deps: &install-https-get-expired-build-deps
 317   - apt-get -qy install --no-install-recommends golang-go ca-certificates
 318
 319 .build-https-get-expired: &build-https-get-expired
 320   - go build -o ./https-get-expired config/chroot_local-includes/usr/src/https-get-expired.go
 321
 322 .test-https-get-expired: &test-https-get-expired
 323   - echo "Basic check:"
 324   - ./https-get-expired -reject-expired https://tails.net/
 325   - echo "Let's pretend we are in the past. Then, this certificate is still good."
 326   - ./https-get-expired -current-time 2000-01-01 -reject-expired https://tails.net/
 327   - echo "Let's pretend we are in the future. Then, this certificate is expired"
 328   - "! ./https-get-expired -current-time 2090-01-01 -reject-expired https://tails.net/"
 329   - "! ./https-get-expired -reject-expired https://wrong.host.badssl.com/"
 330   - "! ./https-get-expired -reject-expired https://self-signed.badssl.com/"
 331   - "! ./https-get-expired -reject-expired https://untrusted-root.badssl.com/"
 332   - "! ./https-get-expired -reject-expired https://expired.badssl.com/"
 333   - echo "Invalid host"
 334   - "! ./https-get-expired -reject-expired https://nxdomain.tails.net/"
 335   - "./bin/test-utils/https-get-expired-test-all"
 336
 337 https-get-expired:
 338   rules:
 339     - if: '$CI_COMMIT_BRANCH =~ /^stable|testing|devel$/'
 340     - changes:
 341         - .gitlab-ci.yml
 342         - config/chroot_local-includes/usr/src/https-get-expired.go
 343         - config/chroot_local-includes/etc/default/htpdate.pools
 344   script:
 345     - *install-https-get-expired-build-deps
 346     - *build-https-get-expired
 347     - *test-https-get-expired
 348
 349 https-get-expired-sid:
 350   # this job gives us results using a future version of Golang compared to the one we actually use
 351   image: registry.gitlab.tails.boum.org/sysadmin-team/container-images/debian:sid
 352   rules:
 353     - if: '$CI_COMMIT_BRANCH == "devel"'
 354     - changes:
 355         - .gitlab-ci.yml
 356         - config/chroot_local-includes/usr/src/https-get-expired.go
 357         - config/chroot_local-includes/etc/default/htpdate.pools
 358   script:
 359     - *install-https-get-expired-build-deps
 360     - *build-https-get-expired
 361     - *test-https-get-expired