From b75f9e4bf7be88a9177aa90f08861e38b630240e Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Sat, 21 Mar 2020 18:32:56 +0300 Subject: [PATCH] Normalize some more job names. Also moves a few jobs between files as seems appropriate. Eliminates some trivially inherited jobs. Change-Id: Ib8e989ef560cc14387d14eb0083b5d678ee28651 --- admin/gitlab-ci/archive.gitlab-ci.yml | 71 ++++++++---- admin/gitlab-ci/documentation.gitlab-ci.yml | 170 ++++++++++------------------ admin/gitlab-ci/global.gitlab-ci.yml | 11 ++ admin/gitlab-ci/gromacs.gitlab-ci.yml | 36 +++--- admin/gitlab-ci/lint.gitlab-ci.yml | 22 ++-- 5 files changed, 150 insertions(+), 160 deletions(-) diff --git a/admin/gitlab-ci/archive.gitlab-ci.yml b/admin/gitlab-ci/archive.gitlab-ci.yml index 4070608f00..895e2e1347 100644 --- a/admin/gitlab-ci/archive.gitlab-ci.yml +++ b/admin/gitlab-ci/archive.gitlab-ci.yml @@ -31,7 +31,7 @@ prepare-release-version: # Special job to package regressiontest files and have them available for testing # Runs during pre-build # Set up to only fetch the files and prepare everything for merge requests -prepare-regressiontests: +regressiontests:prepare: extends: - .variables:default cache: {} @@ -73,7 +73,7 @@ prepare-regressiontests: paths: - gmx-regressiontests.tar.gz -package-regressiontests-release: +regressiontests:package: extends: - .variables:default cache: {} @@ -126,7 +126,7 @@ package-regressiontests-release: paths: - regressiontests-*.tar.gz -configure-archive-nightly: +archive:configure:nightly: stage: nightly-configure only: - web @@ -139,14 +139,14 @@ configure-archive-nightly: variables: - $GROMACS_RELEASE extends: - - .configure-docs + - .docs:configure variables: BUILD_DIR: build-package CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_USE_RDTSCP=OFF CMAKE_MPI_OPTIONS: -DGMX_THREAD_MPI=OFF -DGMX_OPENMP=OFF -configure-archive-release: +archive:configure:release: only: refs: - web @@ -155,7 +155,7 @@ configure-archive-release: variables: - $GROMACS_RELEASE extends: - - .configure-docs + - .docs:configure variables: BUILD_DIR: build-package CMAKE_SIMD_OPTIONS: -DGMX_SIMD=None @@ -167,13 +167,13 @@ configure-archive-release: # the custom build directory due to https://gitlab.com/gitlab-org/gitlab-runner/issues/4203 # As soon as this is resolved we should use the separate build directories instead of # manually hacking things. TODO: #3361 -.build-archive: +.archive:build: extends: - .gromacs:base:build - .use-ccache - .variables:default - - .documentation-before-script-template - - .build-docs-binary-template + - .before_script:default + - .docs:build # Dockerfiles are from dockerhub, user eriklindahl image: gromacs/ci-docs-clang:master only: @@ -223,14 +223,14 @@ configure-archive-release: - $BUILD_DIR/*log - gromacs*tar.gz -build-archive-nightly: +archive:build: extends: - - .build-archive + - .archive:build stage: nightly-build needs: - job: simple-build artifacts: false - - job: configure-archive-nightly + - job: archive:configure:nightly artifacts: true only: - web @@ -242,12 +242,12 @@ build-archive-nightly: variables: BUILD_DIR: build-package -package-archive-release: +archive:package: extends: - - .build-archive + - .archive:build stage: release-package needs: - - job: configure-archive-release + - job: archive:configure:release artifacts: true only: refs: @@ -300,13 +300,40 @@ release-verify: exit 1; fi dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version -webpage-archive-release: +archive:nightly-webpage: extends: - - .build-docs-webpage + - .webpage:build + stage: nightly-deploy + only: + - web + - triggers + - schedules + except: + variables: + - $GROMACS_RELEASE + needs: + - job: simple-build + artifacts: false + - job: linkchecker + artifacts: false + - job: webpage:build + artifacts: true + variables: + BUILD_DIR: build-docs + script: + - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ + artifacts: + when: always + paths: + - webpage.tar.gz + +archive:full-release: + extends: + - .webpage:build stage: release-deploy only: refs: @@ -316,9 +343,9 @@ webpage-archive-release: variables: - $GROMACS_RELEASE dependencies: - - webpage-build-release - - package-archive-release - - package-regressiontests-release + - webpage:build:release + - archive:package + - regressiontests:package variables: BUILD_DIR: release-doc-builds script: diff --git a/admin/gitlab-ci/documentation.gitlab-ci.yml b/admin/gitlab-ci/documentation.gitlab-ci.yml index 72f1f37481..d620ea5673 100644 --- a/admin/gitlab-ci/documentation.gitlab-ci.yml +++ b/admin/gitlab-ci/documentation.gitlab-ci.yml @@ -1,21 +1,14 @@ # Build documentation and web page. -.documentation-before-script-template: - before_script: - - echo $EXTRA_INSTALLS - - time apt-get -qq update - - time apt-get -qqy --no-install-suggests --no-install-recommends install - $EXTRA_INSTALLS - - mkdir -p ccache - - export CCACHE_BASEDIR=${PWD} - - export CCACHE_DIR=${PWD}/ccache -.configure-docs-template: +.docs:configure: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master stage: configure-build extends: + - .gromacs:base:configure - .variables:default + - .before_script:default only: refs: - web @@ -78,13 +71,16 @@ paths: - $BUILD_DIR -.configure-docs-release-template: +.docs:configure:release: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master stage: release-configure extends: + - .gromacs:base:configure - .variables:default + - .before_script:default + cache: {} variables: KUBERNETES_CPU_LIMIT: 1 KUBERNETES_CPU_REQUEST: 1 @@ -190,22 +186,9 @@ - $RELEASE_REGRESSIONTESTS - $RELEASE_SOURCE -.configure-docs: - extends: - - .gromacs:base:configure - - .configure-docs-template - - .documentation-before-script-template - -.configure-docs-release: +docs:configure: extends: - - .gromacs:base:configure - - .configure-docs-release-template - - .documentation-before-script-template - cache: {} - -configure-docs: - extends: - - .configure-docs + - .docs:configure needs: - job: simple-build artifacts: false @@ -216,7 +199,7 @@ configure-docs: variables: - $GROMACS_RELEASE -.build-docs-binary-template: +.docs:build: # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master @@ -237,7 +220,38 @@ configure-docs: paths: - $BUILD_DIR -.build-docs-webpage-template: +docs:build: + extends: + - .gromacs:base:build + - .variables:default + - .use-ccache + - .docs:build + - .before_script:default + stage: documentation + needs: + - job: simple-build + artifacts: false + - job: docs:configure + artifacts: true + variables: + BUILD_DIR: build-docs + except: + variables: + - $GROMACS_RELEASE + +# The manual build is built separately so that errors in converting +# Sphinx to LaTeX and compiling can always be found in the +# all-output.txt file, while avoiding many thousands of lines of spam +# from pdflatex for normal builds. This does reduce the available +# parallel utilization, and so increases the build time. +# +# TODO why are the doxygen and sphinx log files disappearing +# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job +.webpage:build: + extends: + - .gromacs:base:build + - .variables:default + - .before_script:default # Built by admin/dockerfiles/ci-docs-clang # TODO this should be organized more like the current documentation.py script image: gromacs/ci-docs-clang:master @@ -277,53 +291,14 @@ configure-docs: - $BUILD_DIR/docs/doxygen/doxygen-full.log - $BUILD_DIR/docs/sphinx-html.log -.build-docs-binary: - extends: - - .gromacs:base:build - - .use-ccache - - .variables:default - - .use-ccache - - .build-docs-binary-template - - .documentation-before-script-template - -.build-docs-webpage: - extends: - - .gromacs:base:build - - .variables:default - - .build-docs-webpage-template - - .documentation-before-script-template - -build-docs: +webpage:build: extends: - - .build-docs-binary - stage: documentation - needs: - - job: simple-build - artifacts: false - - job: configure-docs - artifacts: true - variables: - BUILD_DIR: build-docs - except: - variables: - - $GROMACS_RELEASE - -# The manual build is built separately so that errors in converting -# Sphinx to LaTeX and compiling can always be found in the -# all-output.txt file, while avoiding many thousands of lines of spam -# from pdflatex for normal builds. This does reduce the available -# parallel utilization, and so increases the build time. -# -# TODO why are the doxygen and sphinx log files disappearing -# TODO use a feature from gitlab runner instead of using pipefail to get timings for the job -webpage: - extends: - - .build-docs-webpage + - .webpage:build stage: post-test needs: - - job: build-docs + - job: docs:build artifacts: true - - job: run-check-source + - job: check-source artifacts: false variables: BUILD_DIR: build-docs @@ -331,36 +306,9 @@ webpage: variables: - $GROMACS_RELEASE -webpage-nightly: +webpage:configure: extends: - - .build-docs-webpage - stage: nightly-deploy - only: - - web - - triggers - - schedules - except: - variables: - - $GROMACS_RELEASE - needs: - - job: simple-build - artifacts: false - - job: linkchecker - artifacts: false - - job: webpage - artifacts: true - variables: - BUILD_DIR: build-docs - script: - - tar czf webpage.tar.gz $BUILD_DIR/docs/html/ - artifacts: - when: always - paths: - - webpage.tar.gz - -webpage-configure-release: - extends: - - .configure-docs-release + - .docs:configure:release cache: {} only: refs: @@ -370,19 +318,23 @@ webpage-configure-release: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version variables: CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -webpage-binary-release: +webpage:dependencies: extends: - - .build-docs-binary + - .gromacs:base:build + - .variables:default + - .use-ccache + - .docs:build + - .before_script:default stage: release-build needs: - - job: webpage-configure-release + - job: webpage:configure artifacts: true variables: BUILD_DIR: release-doc-builds @@ -394,12 +346,12 @@ webpage-binary-release: variables: - $GROMACS_RELEASE -webpage-build-release: +webpage:build:release: extends: - - .build-docs-webpage + - .webpage:build dependencies: - - webpage-binary-release - - webpage-configure-release + - webpage:dependencies + - webpage:configure stage: release-prepare-deploy only: refs: diff --git a/admin/gitlab-ci/global.gitlab-ci.yml b/admin/gitlab-ci/global.gitlab-ci.yml index 730a532836..62c4471695 100644 --- a/admin/gitlab-ci/global.gitlab-ci.yml +++ b/admin/gitlab-ci/global.gitlab-ci.yml @@ -16,6 +16,17 @@ EXTRA_INSTALLS: "python3-dev python3-setuptools python3-pip python3-pytest" # TODO: Get these from updated Docker images. +# Our generic before_script to install dependencies and prepare the ccache directory. +.before_script:default: + before_script: + - echo $EXTRA_INSTALLS + - time apt-get -qq update + - time apt-get -qqy --no-install-suggests --no-install-recommends install + $EXTRA_INSTALLS + - mkdir -p ccache + - export CCACHE_BASEDIR=${PWD} + - export CCACHE_DIR=${PWD}/ccache + # Behavioral templates # Use a persistent compiler cache to speed up rebuilds for a single job. diff --git a/admin/gitlab-ci/gromacs.gitlab-ci.yml b/admin/gitlab-ci/gromacs.gitlab-ci.yml index 64af06e442..b9ecb75289 100644 --- a/admin/gitlab-ci/gromacs.gitlab-ci.yml +++ b/admin/gitlab-ci/gromacs.gitlab-ci.yml @@ -262,8 +262,8 @@ gromacs:gcc-7:release:configure: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version gromacs:gcc-7-simd-sse41:configure: @@ -308,8 +308,8 @@ gromacs:gcc-7-double:release:configure: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version gromacs:gcc-8:configure: @@ -338,8 +338,8 @@ gromacs:gcc-8:release:configure: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version gromacs:gcc-8-cuda-10.2:configure: @@ -400,8 +400,8 @@ gromacs:clang-7:release:configure: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version gromacs:clang-8:configure: @@ -430,8 +430,8 @@ gromacs:clang-8:release:configure: variables: - $GROMACS_RELEASE dependencies: - - package-archive-release - - package-regressiontests-release + - archive:package + - regressiontests:package - prepare-release-version # Jobs running during build stage @@ -741,7 +741,7 @@ gromacs:gcc-7:test: - job: simple-build artifacts: false - job: gromacs:gcc-7:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -774,7 +774,7 @@ gromacs:gcc-7-simd-sse41:test: - job: simple-build artifacts: false - job: gromacs:gcc-7-simd-sse41:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -789,7 +789,7 @@ gromacs:gcc-7-double:test: - job: simple-build artifacts: false - job: gromacs:gcc-7-double:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -822,7 +822,7 @@ gromacs:gcc-8:test: - job: simple-build artifacts: false - job: gromacs:gcc-8:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -864,7 +864,7 @@ gromacs:gcc-8-cuda-10.2:test: - job: simple-build artifacts: false - job: gromacs:gcc-8-cuda-10.2:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -879,7 +879,7 @@ gromacs:clang-6:test: - job: simple-build artifacts: false - job: gromacs:clang-6:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -892,7 +892,7 @@ gromacs:clang-7:test: - job: simple-build artifacts: false - job: gromacs:clang-7:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE @@ -925,7 +925,7 @@ gromacs:clang-8:test: - job: simple-build artifacts: false - job: gromacs:clang-8:build - - job: prepare-regressiontests + - job: regressiontests:prepare except: variables: - $GROMACS_RELEASE diff --git a/admin/gitlab-ci/lint.gitlab-ci.yml b/admin/gitlab-ci/lint.gitlab-ci.yml index 9ac85814bf..19ce0c7c38 100644 --- a/admin/gitlab-ci/lint.gitlab-ci.yml +++ b/admin/gitlab-ci/lint.gitlab-ci.yml @@ -1,6 +1,6 @@ # Repository cleanliness. Source tidiness, linting, and policy compliance. -configure-clang-tidy: +clang-tidy:configure: extends: - .gromacs:base:configure - .use-clang7 @@ -17,7 +17,7 @@ configure-clang-tidy: variables: - $GROMACS_RELEASE -build-clang-tidy: +clang-tidy:build: extends: - .gromacs:base:build - .use-ccache @@ -27,7 +27,7 @@ build-clang-tidy: needs: - job: simple-build artifacts: false - - job: configure-clang-tidy + - job: clang-tidy:configure artifacts: true variables: BUILD_DIR: build-clang-tidy @@ -36,7 +36,7 @@ build-clang-tidy: variables: - $GROMACS_RELEASE -run-clang-format: +clang-format: extends: - .gromacs:base:build - .variables:default @@ -66,7 +66,7 @@ run-clang-format: - clang-format.log - formatting.txt -run-copyright-check: +copyright-check: extends: - .gromacs:base:build - .variables:default @@ -99,17 +99,17 @@ run-copyright-check: - years.log - headers.log -run-check-source: +check-source: extends: - .variables:default - - .build-docs-binary-template - - .documentation-before-script-template + - .docs:build + - .before_script:default cache: {} stage: source-check needs: - job: simple-build artifacts: false - - job: build-docs + - job: docs:build artifacts: true variables: KUBERNETES_CPU_LIMIT: 1 @@ -135,10 +135,10 @@ run-check-source: linkchecker: extends: - - .build-docs-webpage + - .webpage:build stage: nightly-build dependencies: - - webpage + - webpage:build only: - web - schedules -- 2.11.4.GIT