From 788c882029a43dc7d53cf42dbb6aa3e20bcf8c74 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:39:49 -0700 Subject: [PATCH] Compilation Tests for ARW (#2115) TYPE: enhancement KEYWORDS: testing, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The testing frameworks (github actions nor Jenkins) don't run a wide array of compilation tests per compiler family commonly available. Solution: Use the github actions / hpc-workflows testing framework to run serial, sm, dm, and dm+sm for GNU, Intel Classic & OneAPI, and PGI/nvhpc on Derecho. Tests will run collectively in one node in parallel to maximize core hour allocations. TESTS CONDUCTED: 1. Tests can be run locally on Derecho by using the .ci/hpc-workflows runner.py to launch all tests as separate jobs, but this is inefficient and not exactly as the tests run. To emulate the tests as done in the github actions workflow use the additional `args` set in the respective testSet of .github/workflows/.ci.yml and provide alternate directories from which to compile. This can be facilitated via : * clone WRF branch that features these changes into `` * list names of tests you want space-delimited in variable, e.g. `export tests="make-gnu-serial make-gnu-dm"` * create copy test dirs with something like `printf "%s\n" $tests | xargs -i -P 4 cp -Rp {}` * Create alt dirs run locations from test definitions. In this case one directory up from each, this can be done with `export altDirs=$( printf "../%s/.ci " $tests )` (note the extra space!) * launch tests using runner and join args along with `-alt $altDirs` * Final command should look like ``` /.ci/hpc-workflows/.ci/runner.py /.ci/wrf_compilation_tests-make.json -t $tests -a -p -tp 1 -j='{"node_select":{"-l ":{"select":1}}}' -jn -alt $altDirs ``` --- .ci/hpc-workflows | 2 +- .ci/wrf_compilation_tests-make.json | 92 +++++++++++++++++++++++-------------- .github/workflows/ci.yml | 18 +++++++- .github/workflows/test_workflow.yml | 20 ++++---- 4 files changed, 86 insertions(+), 46 deletions(-) diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows index ba839344..dfc8e6d8 160000 --- a/.ci/hpc-workflows +++ b/.ci/hpc-workflows @@ -1 +1 @@ -Subproject commit ba8393447c8a2cef23952c01425154ceb34d64e4 +Subproject commit dfc8e6d823b80497ea41bab94e1fdf3f4594ad18 diff --git a/.ci/wrf_compilation_tests-make.json b/.ci/wrf_compilation_tests-make.json index b80a2b8b..e261afbf 100644 --- a/.ci/wrf_compilation_tests-make.json +++ b/.ci/wrf_compilation_tests-make.json @@ -17,53 +17,75 @@ "queue" : "main", "hpc_arguments" : { - "node_select" : { "-l " : { "select" : 1, "ncpus" : 16 } }, + "node_select" : { "-l " : { "select" : 1, "ncpus" : 8 } }, "priority" : { "-l " : { "job_priority" : "economy" } } }, "arguments" : { - "base_env_numprocs" : [ "-e", "NUM_PROCS=16" ], + "base_env_numprocs" : [ "-e", "NUM_PROCS=8" ], "very_last_modules" : [ "netcdf" ], ".*gnu.*::test_modules" : [ "gcc" ], - ".*intel(?!-llvm).*::test_modules" : [ "intel-classic" ], + ".*intel-classic.*::test_modules" : [ "intel-classic" ], ".*intel-llvm.*::test_modules" : [ "intel-oneapi" ], ".*pgi.*::test_modules" : [ "nvhpc" ], ".*dm.*::test_mpi_module" : [ "cray-mpich" ] } } }, - "make-gnu" : - { - "steps" : - { - "serial" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "32" ] - }, - "sm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "33" ], - "dependencies" : { "serial" : "afterany" } - } - } + + "make-gnu-serial" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "32" ] } } }, + "make-gnu-sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "33" ] } } }, + "make-gnu-dm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "34" ] } } }, + "make-gnu-dm+sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "35" ] } } }, + + "make-intel-classic-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "13" ] } } }, - "make-gnu-mpi" : - { - "steps" : - { - "dm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "34" ] - }, - "dm+sm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "35" ], - "dependencies" : { "dm" : "afterany" } - } - } + "make-intel-classic-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "14" ] } } + }, + "make-intel-classic-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "15" ] } } + }, + "make-intel-classic-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "16" ] } } + }, + + "make-intel-llvm-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "76" ] } } + }, + "make-intel-llvm-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "77" ] } } + }, + "make-intel-llvm-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "78" ] } } + }, + "make-intel-llvm-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "79" ] } } + }, + + "make-pgi-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "1" ] } } + }, + "make-pgi-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "2" ] } } + }, + "make-pgi-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "3" ] } } + }, + "make-pgi-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "4" ] } } } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec396e2c..087f9eba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,22 @@ jobs: tpool : 1 mkdirs : true tests : - - make-gnu - - make-gnu-mpi + - make-gnu-serial + - make-gnu-sm + - make-gnu-dm + - make-gnu-dm+sm + - make-intel-classic-serial + - make-intel-classic-sm + - make-intel-classic-dm + - make-intel-classic-dm+sm + - make-intel-llvm-serial + - make-intel-llvm-sm + - make-intel-llvm-dm + - make-intel-llvm-dm+sm + - make-pgi-serial + - make-pgi-sm + - make-pgi-dm + - make-pgi-dm+sm # add new compilation tests here uses : ./.github/workflows/test_workflow.yml diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index abcb901c..e80d6f33 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -69,12 +69,15 @@ jobs: if : ${{ inputs.mkdirs }} id : cpTestDirs run : | - for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do - echo "Creating duplicate directory for $testDir" - # Remove if it exists to get a fresh start - rm -rf $testDir - cp -Rp main/ $testDir - done + # Limit parallel ops to 4 at a time to not thrash login nodes + + # Remove if it exists to get a fresh start + echo "Cleaning old test directories..." + tests="${{ join( fromJson( inputs.tests ), ' ' ) }}" + time printf "%s\n" $tests | xargs -i -P 4 rm -rf {} + + echo "Creating duplicate directory for $tests" + time printf "%s\n" $tests | xargs -i -P 4 cp -Rp main/ {} - name: Test ${{ inputs.name }} id : runTest @@ -87,6 +90,7 @@ jobs: -t ${{ join( fromJson( inputs.tests ), ' ' ) }} \ -a "${{ inputs.account }}" \ -p ${{ inputs.pool}} -tp ${{ inputs.tpool }} \ + -jn ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }} \ ${{ inputs.args }} $ALT_DIRS @@ -129,8 +133,8 @@ jobs: # *documented* functionality doesn't work as expected. Wow, bravo # can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex # and expands to nothing - name: ${{ github.event_name == 'push' && 'master' || github.event.number }}-${{ inputs.id }}_logfiles - path: ${{ inputs.archive }}/${{ github.event_name == 'push' && 'master' || github.event.number }}/${{ inputs.id }}/ + name: ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }}_logfiles + path: ${{ inputs.archive }}/${{ github.event_name == 'push' && github.ref_name || github.event.number }}/${{ inputs.id }}/ # As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target # and strictly controlled usage of the GH token -- 2.11.4.GIT