admin/gitlab-ci/documentation.gitlab-ci.yml

   1 # Build documentation and web page.
   2
   3 .docs:configure:
   4   extends:
   5     - .variables:default
   6     - .gromacs:base:configure
   7     - .before_script:default
   8   # TODO (#3480) this should be organized more like the current documentation.py script
   9   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-llvm-7-docs
  10   stage: configure-build
  11   cache: {}
  12   variables:
  13     KUBERNETES_CPU_LIMIT: 1
  14     KUBERNETES_CPU_REQUEST: 1
  15     KUBERNETES_MEMORY_REQUEST: 2Gi
  16     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  17     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  18     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=OFF"
  19     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  20     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  21     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  22     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  23   script:
  24     - if [[ ! -z $GROMACS_RELEASE && $GROMACS_RELEASE == "true" ]] ; then
  25       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  26       else
  27       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  28       fi
  29     - echo $CMAKE_COMPILER_SCRIPT
  30     - echo $CMAKE_EXTRA_OPTIONS
  31     - echo $CMAKE_SIMD_OPTIONS
  32     - echo $CMAKE_GPU_OPTIONS
  33     - echo $CMAKE_MPI_OPTIONS
  34     - echo $CMAKE_PRECISION_OPTIONS
  35     - echo $CMAKE_BUILD_TYPE_OPTIONS
  36     - echo $CMAKE_GMXAPI_OPTIONS
  37     - echo $REL_OPTION
  38     - if [[ -d $BUILD_DIR ]] ; then
  39       rm -rf $BUILD_DIR && mkdir $BUILD_DIR ;
  40       else
  41       echo "Preparing new build directory" ;
  42       mkdir $BUILD_DIR ;
  43       fi
  44     - cd $BUILD_DIR
  45     - cmake ..
  46       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  47       $CMAKE_COMPILER_SCRIPT
  48       $CMAKE_EXTRA_OPTIONS
  49       $CMAKE_SIMD_OPTIONS
  50       $CMAKE_MPI_OPTIONS
  51       $CMAKE_PRECISION_OPTIONS
  52       $CMAKE_BUILD_TYPE_OPTIONS
  53       $CMAKE_GPU_OPTIONS
  54       $CMAKE_GMXAPI_OPTIONS
  55       $REL_OPTION
  56       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
  57       2>&1 | tee cmakeLog.log
  58     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
  59     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
  60     - cd ..
  61   artifacts:
  62     name: docs-artifacts-$CI_COMMIT_REF_SLUG
  63     when: always
  64     expire_in: 1 week
  65     paths:
  66       - $BUILD_DIR
  67
  68 .docs:configure:release:
  69   extends:
  70     - .variables:default
  71     - .gromacs:base:configure
  72     - .before_script:default
  73     - .rules:nightly-only-for-release
  74   # TODO (#3480) this should be organized more like the current documentation.py script
  75   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-llvm-7-docs
  76   stage: release-configure
  77   cache: {}
  78   variables:
  79     KUBERNETES_CPU_LIMIT: 1
  80     KUBERNETES_CPU_REQUEST: 1
  81     KUBERNETES_MEMORY_REQUEST: 2Gi
  82     CMAKE_COMPILER_SCRIPT: "-DCMAKE_C_COMPILER=clang-7 -DCMAKE_CXX_COMPILER=clang++-7"
  83     CMAKE_SIMD_OPTIONS: "-DGMX_SIMD=None"
  84     CMAKE_MPI_OPTIONS: "-DGMX_THREAD_MPI=ON"
  85     CMAKE_PRECISION_OPTIONS: "-DGMX_DOUBLE=OFF"
  86     CMAKE_BUILD_TYPE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
  87     CMAKE_GPU_OPTIONS: "-DGMX_GPU=OFF"
  88     CMAKE_GMXAPI_OPTIONS: "-DGMX_PYTHON_PACKAGE=ON"
  89     RELEASE_BUILD_DIR: release-doc-builds
  90     RELEASE_SOURCE: release-source-from-tarball
  91     RELEASE_REGRESSIONTESTS: release-regressiontests-from-tarball
  92   script:
  93     - if [[ $GROMACS_RELEASE == "true" ]] ; then
  94       REL_OPTION="-DGMX_BUILD_TARBALL=ON" ;
  95       else
  96       REL_OPTION="-DGMX_BUILD_TARBALL=OFF" ;
  97       fi
  98     - echo $CMAKE_COMPILER_SCRIPT
  99     - echo $CMAKE_EXTRA_OPTIONS
 100     - echo $CMAKE_SIMD_OPTIONS
 101     - echo $CMAKE_GPU_OPTIONS
 102     - echo $CMAKE_MPI_OPTIONS
 103     - echo $CMAKE_PRECISION_OPTIONS
 104     - echo $CMAKE_BUILD_TYPE_OPTIONS
 105     - echo $CMAKE_GMXAPI_OPTIONS
 106     - echo $REL_OPTION
 107     - VERSION=`cat version.json |
 108       python3 -c "import json,sys; print(json.load(sys.stdin)['version'])"`
 109     - if [[ $GROMACS_RELEASE != "true" ]] ; then
 110       VERSION=$VERSION-dev ;
 111       fi
 112     - REGTEST_COMPARE=`cat version.json |
 113       python3 -c "import json,sys; print(json.load(sys.stdin)['regressiontest-md5sum'])"`
 114     - SOURCENAME=gromacs-$VERSION
 115     - SOURCETARBALL=$SOURCENAME.tar.gz
 116     - REGTESTNAME=regressiontests-$VERSION
 117     - REGTESTTARBALL=$REGTESTNAME.tar.gz
 118     - SOURCE_MD5SUM=`md5sum $SOURCETARBALL | awk '{print $1}'`
 119     - RELEASE_TARBALLS=release-tarballs
 120     - rm -rf $RELEASE_TARBALLS $RELEASE_SOURCE $RELEASE_REGRESSIONTESTS && mkdir $RELEASE_TARBALLS
 121     - mv $SOURCETARBALL $RELEASE_TARBALLS
 122     - mv $REGTESTTARBALL $RELEASE_TARBALLS
 123     - cd $RELEASE_TARBALLS
 124     # We rename the source and regressiontest directories
 125     # to have names for them that don't change for different versions.
 126     - tar -xf $SOURCETARBALL
 127     - mv $SOURCENAME ../$RELEASE_SOURCE
 128     - tar -xf $REGTESTTARBALL
 129     - mv $REGTESTNAME ../$RELEASE_REGRESSIONTESTS
 130     - rm $SOURCETARBALL $REGTESTTARBALL
 131     - cd ..
 132     - echo $CMAKE_COMPILER_SCRIPT
 133     - echo $CMAKE_EXTRA_OPTIONS
 134     - echo $CMAKE_SIMD_OPTIONS
 135     - echo $CMAKE_GPU_OPTIONS
 136     - echo $CMAKE_MPI_OPTIONS
 137     - echo $CMAKE_PRECISION_OPTIONS
 138     - echo $CMAKE_BUILD_TYPE_OPTIONS
 139     - echo $CMAKE_GMXAPI_OPTIONS
 140     - if [[ -d $RELEASE_BUILD_DIR ]] ; then
 141       echo "Cleaning up build directory" ;
 142       rm -rf $RELEASE_BUILD_DIR && mkdir $RELEASE_BUILD_DIR ;
 143       else
 144       echo "Preparing new build directory" ;
 145       mkdir $RELEASE_BUILD_DIR ;
 146       fi
 147     - cd $RELEASE_BUILD_DIR
 148     - cmake ../$RELEASE_SOURCE/
 149       -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 150       $CMAKE_COMPILER_SCRIPT
 151       $CMAKE_EXTRA_OPTIONS
 152       $CMAKE_SIMD_OPTIONS
 153       $CMAKE_MPI_OPTIONS
 154       $CMAKE_PRECISION_OPTIONS
 155       $CMAKE_BUILD_TYPE_OPTIONS
 156       $CMAKE_GPU_OPTIONS
 157       $CMAKE_GMXAPI_OPTIONS
 158       $REL_OPTION
 159       "-DSOURCE_MD5SUM=$SOURCE_MD5SUM"
 160       "-DREGRESSIONTEST_PATH=../$RELEASE_REGRESSIONTESTS"
 161       -DCMAKE_INSTALL_PREFIX=../$INSTALL_DIR -DGMX_COMPILER_WARNINGS=ON
 162       2>&1 | tee cmakeLog.log
 163     - awk '/CMake Warning/,/^--|^$/' cmakeLog.log | tee cmakeErrors.log
 164     - if [ -s cmakeErrors.log  ] ; then echo "Found CMake warning while processing build"; cat cmakeErrors.log ; exit 1; fi
 165     - cd ..
 166   artifacts:
 167     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 168     when: always
 169     expire_in: 1 week
 170     paths:
 171       - $RELEASE_BUILD_DIR
 172       - $RELEASE_REGRESSIONTESTS
 173       - $RELEASE_SOURCE
 174
 175 docs:configure:
 176   extends:
 177     - .docs:configure
 178     - .rules:basic-push
 179   variables:
 180     BUILD_DIR: build-docs
 181     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 182
 183 .docs:build:
 184   # TODO (#3480) this should be organized more like the current documentation.py script
 185   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-llvm-7-docs
 186   script:
 187     - cd $BUILD_DIR
 188     - cmake --build . --target gmx -- -j8
 189     - cd ..
 190   artifacts:
 191     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 192     when: always
 193     expire_in: 1 week
 194     paths:
 195       - $BUILD_DIR
 196
 197 docs:build:
 198   extends:
 199     - .variables:default
 200     - .gromacs:base:build
 201     - .use-ccache
 202     - .docs:build
 203     - .before_script:default
 204     - .rules:basic-push
 205   stage: documentation
 206   needs:
 207     - job: docs:configure
 208   variables:
 209     BUILD_DIR: build-docs
 210
 211 # The manual build is built separately so that errors in converting
 212 # Sphinx to LaTeX and compiling can always be found in the
 213 # all-output.txt file, while avoiding many thousands of lines of spam
 214 # from pdflatex for normal builds. This does reduce the available
 215 # parallel utilization, and so increases the build time.
 216 #
 217 # TODO why are the doxygen and sphinx log files disappearing
 218 # TODO use a feature from gitlab runner instead of using pipefail to get timings for the job
 219 .webpage:build:
 220   extends:
 221     - .variables:default
 222     - .gromacs:base:build
 223     - .before_script:default
 224   # TODO (#3480) this should be organized more like the current documentation.py script
 225   image: ${CI_REGISTRY}/gromacs/gromacs/ci-ubuntu-18.04-llvm-7-docs
 226   cache: {}
 227   variables:
 228     KUBERNETES_CPU_LIMIT: 4
 229     KUBERNETES_CPU_REQUEST: 2
 230     KUBERNETES_MEMORY_REQUEST: 4Gi
 231   script:
 232     - cd $BUILD_DIR
 233     - cmake --build . --target manual -- -j4 | tee docs/manual/all-output.log
 234       | grep -i "warning" | grep -v "TeX" | grep -v "Package" || true
 235     - cmake --build . --target webpage -- -j4
 236     - grep "WARNING:" docs/manual/all-output.log | tee pythonErrors.log || true
 237     - grep -i "error" docs/sphinx-*.log | tee sphinxDiagnostics.log || true
 238     - grep -i "warning" docs/sphinx-*.log | tee -a sphinxDiagnostics.log || true
 239     - if [ -s pythonErrors.log ] ; then echo "Found Python Errors during build"; exit 1; fi
 240     - if [ -s sphinxErrors.log ] ; then echo "Found errors during Sphinx documentation build"; cat sphinxErrors.log; exit 1; fi
 241     - if [ -s sphinxDiagnostics.log ] ; then echo "Found diagnostic warnings during Sphinx documentation build"; cat sphinxDiagnostics.log; exit 1; fi
 242     - cd ..
 243   artifacts:
 244     name: docs-artifacts-$CI_COMMIT_REF_SLUG
 245     when: always
 246     expire_in: 1 week
 247     paths:
 248       - $BUILD_DIR/*.log
 249       - $BUILD_DIR/docs/html
 250       - $BUILD_DIR/docs/manual/gromacs.log
 251       - $BUILD_DIR/docs/manual/all-output.log
 252       - $BUILD_DIR/docs/doxygen/doxygen-user.log
 253       - $BUILD_DIR/docs/doxygen/doxygen-lib.log
 254       - $BUILD_DIR/docs/doxygen/doxygen-full.log
 255       - $BUILD_DIR/docs/sphinx-html.log
 256
 257 webpage:build:
 258   extends:
 259     - .webpage:build
 260     - .rules:basic-push
 261   stage: post-test
 262   needs:
 263     - job: docs:build
 264     - job: check-source
 265       artifacts: false
 266   variables:
 267     BUILD_DIR: build-docs
 268
 269 webpage:configure:
 270   extends:
 271     - .docs:configure:release
 272     - .rules:nightly-only-for-release
 273   cache: {}
 274   tags:
 275     - k8s-scilifelab
 276   dependencies:
 277     - archive:package
 278     - regressiontests:package
 279     - prepare-release-version
 280   variables:
 281     CMAKE_EXTRA_OPTIONS: -DGMX_BUILD_HELP=on -DGMX_BUILD_MANUAL=on -DGMX_USE_SIMD_KERNELS=off
 282
 283
 284 webpage:dependencies:
 285   extends:
 286     - .gromacs:base:build
 287     - .variables:default
 288     - .use-ccache
 289     - .docs:build
 290     - .before_script:default
 291     - .rules:nightly-only-for-release
 292   stage: release-build
 293   tags:
 294     - k8s-scilifelab
 295   needs:
 296     - job: webpage:configure
 297   variables:
 298     BUILD_DIR: release-doc-builds
 299
 300 webpage:build:release:
 301   extends:
 302     - .webpage:build
 303     - .rules:nightly-only-for-release
 304   stage: release-prepare-deploy
 305   tags:
 306     - k8s-scilifelab
 307   dependencies:
 308     - webpage:dependencies
 309     - webpage:configure
 310   variables:
 311     BUILD_DIR: release-doc-builds