validate.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 # We use the default ghc in PATH as default
   8 # Use the ghc-x.y.z trigger several errors in windows:
   9 # * It triggers the max path length issue:
  10 #   See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
  11 # * It triggers a `createProcess: does not exist` error in units tests
  12 #   See https://github.com/haskell/cabal/issues/8049
  13 HC=ghc
  14 CABAL=cabal
  15 JOBS=4
  16 LIBTESTS=true
  17 CLITESTS=true
  18 CABALSUITETESTS=true
  19 LIBONLY=false
  20 DEPSONLY=false
  21 DOCTEST=false
  22 BENCHMARKS=false
  23 VERBOSE=false
  24 HACKAGETESTSALL=false
  25
  26 TARGETS=""
  27 STEPS=""
  28 EXTRAHCS=""
  29
  30 LISTSTEPS=false
  31
  32 # Help
  33 #######################################################################
  34
  35 show_usage() {
  36 cat <<EOF
  37 ./validate.sh - build & test
  38
  39 Usage: ./validate.sh [options]
  40   A script which runs all the tests.
  41
  42 Available options:
  43   -j, --jobs JOBS                   cabal build -j argument (default:  $JOBS)
  44       --libonly                     Test only Cabal-the-library
  45       --cli                         Test both Cabal-the-library and cabal-install
  46       --(no-)run-lib-tests          Run library tests
  47       --(no-)run-cli-tests          Run client tests
  48       --(no-)run-lib-suite          Run cabal-testsuite with library
  49       --(no-)run-cli-suite          Run cabal-testsuite with client
  50   -w, --with-compiler HC            With compiler
  51       --with-cabal CABAL            With cabal-install
  52       --extra-hc HC                 Extra compiler to run test-suite with
  53       --(no-)doctest                Run doctest on library
  54       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  55       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  56       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  57   -v, --verbose                     Verbose output
  58   -q, --quiet                       Less output
  59   -s, --step STEP                   Run only specific step (can be specified multiple times)
  60       --list-steps                  List steps and build-targets and exit
  61       --help                        Print this message and exit
  62 EOF
  63 }
  64
  65 # "library"
  66 #######################################################################
  67
  68 OUTPUT=$(mktemp)
  69
  70 # `red` and `green` are used to output also the spent time in white at the end.
  71 # `blue` and `cyan` are used to print the spawned command, so they only have one
  72 # argument.
  73 red () {
  74   printf "\033[0;31m%s\033[0m %s \n" "$1" "$2"
  75 }
  76 green () {
  77   printf "\033[0;32m%s\033[0m %s \n" "$1" "$2"
  78 }
  79 blue () {
  80   printf "\033[0;34m%s\033[0m\n" "$1"
  81 }
  82 cyan () {
  83   printf "\033[0;96m%s\033[0m\n" "$1"
  84 }
  85
  86 JOB_START_TIME=$(date +%s)
  87
  88 timed() {
  89     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  90     blue "$PRETTYCMD"
  91     start_time=$(date +%s)
  92
  93     if $VERBOSE; then
  94         "$@" 2>&1
  95     else
  96         "$@" > "$OUTPUT" 2>&1
  97     fi
  98     # echo "MOCK" > "$OUTPUT"
  99     RET=$?
 100
 101     end_time=$(date +%s)
 102     duration=$((end_time - start_time))
 103     tduration=$((end_time - JOB_START_TIME))
 104
 105     if [ $RET -eq 0 ]; then
 106         if ! $VERBOSE; then
 107             # if output is relatively short, show everything
 108             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
 109                 cat "$OUTPUT"
 110             else
 111                 echo "..."
 112                 tail -n 20 "$OUTPUT"
 113             fi
 114
 115             rm -f "$OUTPUT"
 116         fi
 117
 118         green "<<< $PRETTYCMD" "($duration/$tduration sec)"
 119
 120         # bottom-margin
 121         echo ""
 122     else
 123         if ! $VERBOSE; then
 124             cat "$OUTPUT"
 125         fi
 126
 127         red "<<< $PRETTYCMD" "($duration/$tduration sec, $RET)"
 128         red "<<< $*" "($duration/$tduration sec, $RET)"
 129         rm -f "$OUTPUT"
 130         exit 1
 131     fi
 132 }
 133
 134 print_header() {
 135     TITLE=$1
 136     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 137     cyan "===X========================================================================== $(date +%T) ===" \
 138       | sed "s#X$TITLEPAT=# $TITLE #"
 139
 140 }
 141
 142 # getopt
 143 #######################################################################
 144
 145 while [ $# -gt 0 ]; do
 146     arg=$1
 147     case $arg in
 148         --help)
 149             show_usage
 150             exit
 151             ;;
 152         -j|--jobs)
 153             JOBS="$2"
 154             shift
 155             shift
 156             ;;
 157         --lib-only)
 158             LIBONLY=true
 159             shift
 160             ;;
 161         --cli)
 162             LIBONLY=false
 163             shift
 164             ;;
 165         --run-lib-tests)
 166             LIBTESTS=true
 167             shift
 168             ;;
 169         --no-run-lib-tests)
 170             LIBTESTS=false
 171             shift
 172             ;;
 173         --run-cli-tests)
 174             CLITESTS=true
 175             shift
 176             ;;
 177         --no-run-cli-tests)
 178             CLITESTS=false
 179             shift
 180             ;;
 181         --run-lib-suite)
 182             LIBSUITE=true
 183             shift
 184             ;;
 185         --no-run-lib-suite)
 186             LIBSUITE=false
 187             shift
 188             ;;
 189         --run-cli-suite)
 190             CLISUITE=true
 191             shift
 192             ;;
 193         --no-run-cli-suite)
 194             CLISUITE=false
 195             shift
 196             ;;
 197         -w|--with-compiler)
 198             HC=$2
 199             shift
 200             shift
 201             ;;
 202         --with-cabal)
 203             CABAL=$2
 204             shift
 205             shift
 206             ;;
 207         --extra-hc)
 208             EXTRAHCS="$EXTRAHCS $2"
 209             shift
 210             shift
 211             ;;
 212         --doctest)
 213             DOCTEST=true
 214             shift
 215             ;;
 216         --no-doctest)
 217             DOCTEST=false
 218             shift
 219             ;;
 220         --solver-benchmarks)
 221             BENCHMARKS=true
 222             shift
 223             ;;
 224         --no-solver-benchmarks)
 225             BENCHMARKS=false
 226             shift
 227             ;;
 228         --complete-hackage-tests)
 229             HACKAGETESTSALL=true
 230             shift
 231             ;;
 232         --partial-hackage-tests)
 233             HACKAGETESTSALL=false
 234             shift
 235             ;;
 236         -v|--verbose)
 237             VERBOSE=true
 238             shift
 239             ;;
 240         -q|--quiet)
 241             VERBOSE=false
 242             shift
 243             ;;
 244         -s|--step)
 245             STEPS="$STEPS $2"
 246             shift
 247             shift
 248             ;;
 249         --list-steps)
 250             LISTSTEPS=true
 251             shift
 252             ;;
 253         *)
 254             echo "Unknown option $arg"
 255             exit 1
 256     esac
 257 done
 258
 259 # calculate steps and build targets
 260 #######################################################################
 261
 262 # If there are no explicit steps given calculate them
 263 if $LIBONLY; then
 264     CLITESTS=false
 265     CLISUITE=false
 266     BENCHMARKS=false
 267 fi
 268
 269 if [ -z "$STEPS" ]; then
 270     STEPS="print-config print-tool-versions"
 271     STEPS="$STEPS build"
 272     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 273     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 274     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 275     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 276                     then STEPS="$STEPS lib-suite-extras"; fi
 277     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 278     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 279     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 280     STEPS="$STEPS time-summary"
 281 fi
 282
 283 TARGETS="Cabal Cabal-hooks cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 284 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 285 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 286
 287 if $LISTSTEPS; then
 288   echo "Targets: $TARGETS"
 289   echo "Steps:   $STEPS"
 290   exit
 291 fi
 292
 293 # Adjust runtime configuration
 294 #######################################################################
 295
 296 TESTSUITEJOBS="-j$JOBS"
 297 JOBS="-j$JOBS"
 298
 299 # assume compiler is GHC
 300 RUNHASKELL=$(echo "$HC" | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 301
 302 case "$(uname)" in
 303     MINGW64*)
 304         ARCH="x86_64-windows"
 305         ;;
 306     Linux   )
 307         ARCH="x86_64-linux"
 308         ;;
 309     *)
 310         ARCH="x86_64-osx"
 311         ;;
 312 esac
 313
 314 if $LIBONLY; then
 315     PROJECTFILE=cabal.validate-libonly.project
 316 else
 317     PROJECTFILE=cabal.validate.project
 318 fi
 319
 320 BASEHC=ghc-$($HC --numeric-version)
 321 BUILDDIR=dist-newstyle-validate-$BASEHC
 322 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 323
 324 CABALNEWBUILD="${CABAL} build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 325 CABALLISTBIN="${CABAL} list-bin --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 326
 327 # This was needed in some local Windows MSYS2 environments
 328 # but breaks CI for Windows + GHC 9.0.2, thus it is set only on non-CI executions
 329 # of validate.sh
 330 # https://github.com/haskell/cabal/issues/9571
 331 # https://github.com/haskell/cabal/pull/10114
 332 RTSOPTS="$([[ $ARCH = "x86_64-windows" && -z "$CI" ]] && echo "+RTS --io-manager=native" || echo "")"
 333
 334 # header
 335 #######################################################################
 336
 337 step_print_config() {
 338 print_header print-config
 339
 340 cat <<EOF
 341 compiler:            $HC
 342 runhaskell:          $RUNHASKELL
 343 cabal-install:       $CABAL
 344 jobs:                $JOBS
 345 Cabal tests:         $LIBTESTS
 346 cabal-install tests: $CLITESTS
 347 cabal-testsuite:     $CABALSUITETESTS
 348 library only:        $LIBONLY
 349 dependencies only:   $DEPSONLY
 350 doctest:             $DOCTEST
 351 benchmarks:          $BENCHMARKS
 352 verbose:             $VERBOSE
 353 extra compilers:     $EXTRAHCS
 354 extra RTS options:   $RTSOPTS
 355
 356 EOF
 357 }
 358
 359 step_print_tool_versions() {
 360 print_header print-tool-versions
 361
 362 timed "$HC" --version
 363 timed "$CABAL" --version
 364
 365 for EXTRAHC in $EXTRAHCS; do
 366     timed "$EXTRAHC" --version
 367 done
 368 }
 369
 370 step_time_summary() {
 371     print_header END
 372
 373     JOB_END_TIME=$(date +%s)
 374     tduration=$((JOB_END_TIME - JOB_START_TIME))
 375
 376     cyan "!!! Validation took $tduration seconds."
 377 }
 378
 379 # build
 380 #######################################################################
 381
 382 step_build() {
 383 print_header "build"
 384 print_header "Step Build: dry run"
 385 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 386 print_header "Step Build: full build plan (cached and to-be-built dependencies):"
 387 jq -r '."install-plan" | map(."pkg-name" + "-" + ."pkg-version" + " " + ."component-name") | join("\n")' "$BUILDDIR/cache/plan.json"
 388 print_header "Step Build: actual build"
 389 timed $CABALNEWBUILD $TARGETS || exit 1
 390 }
 391
 392 # Cabal lib
 393 #######################################################################
 394
 395 step_doctest() {
 396 print_header "Cabal: doctest"
 397 cabal-env --name doctest-Cabal --transitive QuickCheck
 398 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 399 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 400 }
 401
 402 step_lib_tests() {
 403 print_header "Cabal: tests"
 404
 405 CMD="$($CABALLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 406 (cd Cabal-tests && timed $CMD) || exit 1
 407
 408 CMD="$($CABALLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 409 (cd Cabal-tests && timed $CMD) || exit 1
 410
 411 CMD="$($CABALLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 412 (cd Cabal-tests && timed $CMD) || exit 1
 413
 414 CMD="$($CABALLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 415 (cd Cabal-tests && timed $CMD) || exit 1
 416
 417 CMD="$($CABALLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 418 (cd Cabal-tests && timed $CMD) || exit 1
 419
 420 CMD=$($CABALLISTBIN Cabal-tests:test:hackage-tests)
 421 (cd Cabal-tests && timed $CMD read-fields) || exit 1
 422 if $HACKAGETESTSALL; then
 423     (cd Cabal-tests && timed $CMD parsec)    || exit 1
 424     (cd Cabal-tests && timed $CMD roundtrip) || exit 1
 425 else
 426     (cd Cabal-tests && timed $CMD parsec d)    || exit 1
 427     (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
 428 fi
 429 }
 430
 431 # Cabal cabal-testsuite
 432 #######################################################################
 433
 434 step_lib_suite() {
 435 print_header "Cabal: cabal-testsuite"
 436
 437 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes $RTSOPTS"
 438 (cd cabal-testsuite && timed $CMD) || exit 1
 439 }
 440
 441 step_lib_suite_extras() {
 442 for EXTRAHC in $EXTRAHCS; do
 443
 444 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 445 (cd cabal-testsuite && timed $CMD) || exit 1
 446
 447 done
 448 }
 449
 450 # cabal-install
 451 #######################################################################
 452
 453 step_cli_tests() {
 454 print_header "cabal-install: tests"
 455
 456 # this are sorted in asc time used, quicker tests first.
 457 CMD="$($CABALLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 458 (cd cabal-install && timed $CMD) || exit 1
 459
 460 # This doesn't work in parallel either
 461 CMD="$($CABALLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 462 (cd cabal-install && timed $CMD) || exit 1
 463
 464 # Only single job, otherwise we fail with "Heap exhausted"
 465 CMD="$($CABALLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
 466 (cd cabal-install && timed $CMD) || exit 1
 467
 468 # This test-suite doesn't like concurrency
 469 CMD="$($CABALLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 470 (cd cabal-install && timed $CMD) || exit 1
 471 }
 472
 473 # cabal-install cabal-testsuite
 474 #######################################################################
 475
 476 step_cli_suite() {
 477 print_header "cabal-install: cabal-testsuite"
 478
 479 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS  --with-ghc=$HC --hide-successes --intree-cabal-lib=$PWD --test-tmp=$PWD/testdb $RTSOPTS"
 480 (cd cabal-testsuite && timed $CMD) || exit 1
 481 }
 482
 483 # solver-benchmarks
 484 #######################################################################
 485
 486 step_solver_benchmarks_tests() {
 487 print_header "solver-benchmarks: test"
 488
 489 CMD="$($CABALLISTBIN solver-benchmarks:test:unit-tests)"
 490 (cd Cabal && timed $CMD) || exit 1
 491 }
 492
 493 step_solver_benchmarks_run() {
 494 print_header "solver-benchmarks: run"
 495
 496 SOLVEPKG=Chart-diagrams
 497 CMD="$($CABALLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 498 (cd Cabal && timed $CMD) || exit 1
 499 }
 500
 501 # Steps dispatcher
 502 #######################################################################
 503
 504 for step in $STEPS; do
 505     case $step in
 506         print-config)             step_print_config            ;;
 507         print-tool-versions)      step_print_tool_versions     ;;
 508         build)                    step_build                   ;;
 509         doctest)                  step_doctest                 ;;
 510         lib-tests)                step_lib_tests               ;;
 511         cli-tests)                step_cli_tests               ;;
 512         lib-suite)                step_lib_suite               ;;
 513         lib-suite-extras)         step_lib_suite_extras        ;;
 514         cli-suite)                step_cli_suite               ;;
 515         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 516         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 517         time-summary)             step_time_summary            ;;
 518         *)
 519             echo "Invalid step $step"
 520             exit 1
 521             ;;
 522     esac
 523 done
 524
 525 #######################################################################