validate.sh

   1 #!/usr/bin/env bash
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 # We use the default ghc in PATH as default
   8 # Use the ghc-x.y.z trigger several errors in windows:
   9 # * It triggers the max path length issue:
  10 #   See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
  11 # * It triggers a `createProcess: does not exist` error in units tests
  12 #   See https://github.com/haskell/cabal/issues/8049
  13 HC=ghc
  14 CABAL=cabal
  15 JOBS=""
  16 LIBTESTS=true
  17 CLITESTS=true
  18 CABALSUITETESTS=true
  19 LIBONLY=false
  20 DEPSONLY=false
  21 DOCTEST=false
  22 BENCHMARKS=false
  23 VERBOSE=false
  24 HACKAGETESTSALL=false
  25
  26 TARGETS=""
  27 STEPS=""
  28 EXTRAHCS=""
  29
  30 LISTSTEPS=false
  31
  32 # Help
  33 #######################################################################
  34
  35 show_usage() {
  36 cat <<EOF
  37 ./validate.sh - build & test
  38
  39 Usage: ./validate.sh [options]
  40   A script which runs all the tests.
  41
  42 Available options:
  43   -j, --jobs JOBS                   cabal build -j argument (default:  $JOBS)
  44       --libonly                     Test only Cabal-the-library
  45       --cli                         Test both Cabal-the-library and cabal-install
  46       --(no-)run-lib-tests          Run library tests
  47       --(no-)run-cli-tests          Run client tests
  48       --(no-)run-lib-suite          Run cabal-testsuite with library
  49       --(no-)run-cli-suite          Run cabal-testsuite with client
  50   -w, --with-compiler HC            With compiler
  51       --with-cabal CABAL            With cabal-install
  52       --extra-hc HC                 Extra compiler to run test-suite with
  53       --(no-)doctest                Run doctest on library
  54       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  55       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  56       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  57   -v, --verbose                     Verbose output
  58   -q, --quiet                       Less output
  59   -s, --step STEP                   Run only specific step (can be specified multiple times)
  60       --list-steps                  List steps and build-targets and exit
  61       --help                        Print this message and exit
  62 EOF
  63 }
  64
  65 # "library"
  66 #######################################################################
  67
  68 OUTPUT=$(mktemp)
  69
  70 # `red` and `green` are used to output also the spent time in white at the end.
  71 # `blue` and `cyan` are used to print the spawned command, so they only have one
  72 # argument.
  73 red () {
  74   printf "\033[0;31m%s\033[0m %s \n" "$1" "$2"
  75 }
  76 green () {
  77   printf "\033[0;32m%s\033[0m %s \n" "$1" "$2"
  78 }
  79 blue () {
  80   printf "\033[0;34m%s\033[0m\n" "$1"
  81 }
  82 cyan () {
  83   printf "\033[0;96m%s\033[0m\n" "$1"
  84 }
  85
  86 JOB_START_TIME=$(date +%s)
  87
  88 timed() {
  89     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  90     blue "$PRETTYCMD"
  91     start_time=$(date +%s)
  92
  93     if $VERBOSE; then
  94         "$@" 2>&1
  95     else
  96         "$@" > "$OUTPUT" 2>&1
  97     fi
  98     # echo "MOCK" > "$OUTPUT"
  99     RET=$?
 100
 101     end_time=$(date +%s)
 102     duration=$((end_time - start_time))
 103     tduration=$((end_time - JOB_START_TIME))
 104
 105     if [ $RET -eq 0 ]; then
 106         if ! $VERBOSE; then
 107             # if output is relatively short, show everything
 108             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
 109                 cat "$OUTPUT"
 110             else
 111                 echo "..."
 112                 tail -n 20 "$OUTPUT"
 113             fi
 114
 115             rm -f "$OUTPUT"
 116         fi
 117
 118         green "<<< $PRETTYCMD" "($duration/$tduration sec)"
 119
 120         # bottom-margin
 121         echo ""
 122     else
 123         if ! $VERBOSE; then
 124             cat "$OUTPUT"
 125         fi
 126
 127         red "<<< $PRETTYCMD" "($duration/$tduration sec, $RET)"
 128         red "<<< $*" "($duration/$tduration sec, $RET)"
 129         rm -f "$OUTPUT"
 130         exit 1
 131     fi
 132 }
 133
 134 print_header() {
 135     TITLE=$1
 136     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 137     cyan "===X========================================================================== $(date +%T) ===" \
 138       | sed "s#X$TITLEPAT=# $TITLE #"
 139
 140 }
 141
 142 # getopt
 143 #######################################################################
 144
 145 while [ $# -gt 0 ]; do
 146     arg=$1
 147     case $arg in
 148         --help)
 149             show_usage
 150             exit
 151             ;;
 152         -j|--jobs)
 153             JOBS="$2"
 154             shift
 155             shift
 156             ;;
 157         --lib-only)
 158             LIBONLY=true
 159             shift
 160             ;;
 161         --cli)
 162             LIBONLY=false
 163             shift
 164             ;;
 165         --run-lib-tests)
 166             LIBTESTS=true
 167             shift
 168             ;;
 169         --no-run-lib-tests)
 170             LIBTESTS=false
 171             shift
 172             ;;
 173         --run-cli-tests)
 174             CLITESTS=true
 175             shift
 176             ;;
 177         --no-run-cli-tests)
 178             CLITESTS=false
 179             shift
 180             ;;
 181         --run-lib-suite)
 182             LIBSUITE=true
 183             shift
 184             ;;
 185         --no-run-lib-suite)
 186             LIBSUITE=false
 187             shift
 188             ;;
 189         --run-cli-suite)
 190             CLISUITE=true
 191             shift
 192             ;;
 193         --no-run-cli-suite)
 194             CLISUITE=false
 195             shift
 196             ;;
 197         -w|--with-compiler)
 198             HC=$2
 199             shift
 200             shift
 201             ;;
 202         --with-cabal)
 203             CABAL=$2
 204             shift
 205             shift
 206             ;;
 207         --extra-hc)
 208             EXTRAHCS="$EXTRAHCS $2"
 209             shift
 210             shift
 211             ;;
 212         --doctest)
 213             DOCTEST=true
 214             shift
 215             ;;
 216         --no-doctest)
 217             DOCTEST=false
 218             shift
 219             ;;
 220         --solver-benchmarks)
 221             BENCHMARKS=true
 222             shift
 223             ;;
 224         --no-solver-benchmarks)
 225             BENCHMARKS=false
 226             shift
 227             ;;
 228         --complete-hackage-tests)
 229             HACKAGETESTSALL=true
 230             shift
 231             ;;
 232         --partial-hackage-tests)
 233             HACKAGETESTSALL=false
 234             shift
 235             ;;
 236         -v|--verbose)
 237             VERBOSE=true
 238             shift
 239             ;;
 240         -q|--quiet)
 241             VERBOSE=false
 242             shift
 243             ;;
 244         -s|--step)
 245             STEPS="$STEPS $2"
 246             shift
 247             shift
 248             ;;
 249         --list-steps)
 250             LISTSTEPS=true
 251             shift
 252             ;;
 253         *)
 254             echo "Unknown option $arg"
 255             exit 1
 256     esac
 257 done
 258
 259 # calculate steps and build targets
 260 #######################################################################
 261
 262 # If there are no explicit steps given calculate them
 263 if $LIBONLY; then
 264     CLITESTS=false
 265     CLISUITE=false
 266     BENCHMARKS=false
 267 fi
 268
 269 if [ -z "$STEPS" ]; then
 270     STEPS="print-config print-tool-versions"
 271     STEPS="$STEPS build"
 272     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 273     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 274     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 275     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 276                     then STEPS="$STEPS lib-suite-extras"; fi
 277     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 278     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 279     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 280     STEPS="$STEPS time-summary"
 281 fi
 282
 283 TARGETS="Cabal Cabal-hooks cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 284 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 285 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 286
 287 if $LISTSTEPS; then
 288   echo "Targets: $TARGETS"
 289   echo "Steps:   $STEPS"
 290   exit
 291 fi
 292
 293 # Adjust runtime configuration
 294 #######################################################################
 295
 296 if [ -z "$JOBS" ]; then
 297     if command -v nproc >/dev/null; then
 298         JOBS=$(nproc)
 299     else
 300         echo "Warning: \`nproc\` not found, setting \`--jobs\` to default of 4."
 301         JOBS=4
 302     fi
 303 fi
 304
 305 TESTSUITEJOBS="-j$JOBS"
 306 JOBS="-j$JOBS"
 307
 308 # assume compiler is GHC
 309 RUNHASKELL=$(echo "$HC" | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 310
 311 ARCH=$(uname -m)
 312
 313 case "$ARCH" in
 314     arm64)
 315         ARCH=aarch64
 316         ;;
 317     x86_64)
 318         ARCH=x86_64
 319         ;;
 320     *)
 321         echo "Warning: Unknown architecture '$ARCH'"
 322         ;;
 323 esac
 324
 325 OS=$(uname)
 326
 327 case "$OS" in
 328     MINGW64*)
 329         ARCH="$ARCH-windows"
 330         ;;
 331     Linux)
 332         ARCH="$ARCH-linux"
 333         ;;
 334     Darwin)
 335         ARCH="$ARCH-osx"
 336         ;;
 337     *)
 338         echo "Warning: Unknown operating system '$OS'"
 339         ARCH="$ARCH-$OS"
 340         ;;
 341 esac
 342
 343 if $LIBONLY; then
 344     PROJECTFILE=cabal.validate-libonly.project
 345 else
 346     PROJECTFILE=cabal.validate.project
 347 fi
 348
 349 BASEHC=ghc-$($HC --numeric-version)
 350 BUILDDIR=dist-newstyle-validate-$BASEHC
 351 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 352
 353 CABALNEWBUILD="${CABAL} build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 354 CABALLISTBIN="${CABAL} list-bin --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 355
 356 # See https://github.com/haskell/cabal/issues/9571 for why we set this for Windows
 357 RTSOPTS="$([ $ARCH = "x86_64-windows" ] && [ "$($HC --numeric-version)" != "9.0.2" ] && [ "$(echo -e "$(ghc --numeric-version)\n9.0.2" | sort -V | head -n1)" = "9.0.2" ] && echo "+RTS --io-manager=native" || echo "")"
 358
 359 # header
 360 #######################################################################
 361
 362 step_print_config() {
 363 print_header print-config
 364
 365 cat <<EOF
 366 compiler:            $HC
 367 runhaskell:          $RUNHASKELL
 368 cabal-install:       $CABAL
 369 jobs:                $JOBS
 370 Cabal tests:         $LIBTESTS
 371 cabal-install tests: $CLITESTS
 372 cabal-testsuite:     $CABALSUITETESTS
 373 library only:        $LIBONLY
 374 dependencies only:   $DEPSONLY
 375 doctest:             $DOCTEST
 376 benchmarks:          $BENCHMARKS
 377 verbose:             $VERBOSE
 378 extra compilers:     $EXTRAHCS
 379 extra RTS options:   $RTSOPTS
 380
 381 EOF
 382 }
 383
 384 step_print_tool_versions() {
 385 print_header print-tool-versions
 386
 387 timed "$HC" --version
 388 timed "$CABAL" --version
 389
 390 for EXTRAHC in $EXTRAHCS; do
 391     timed "$EXTRAHC" --version
 392 done
 393 }
 394
 395 step_time_summary() {
 396     print_header END
 397
 398     JOB_END_TIME=$(date +%s)
 399     tduration=$((JOB_END_TIME - JOB_START_TIME))
 400
 401     cyan "!!! Validation took $tduration seconds."
 402 }
 403
 404 # build
 405 #######################################################################
 406
 407 step_build() {
 408 print_header "build"
 409 print_header "Step Build: dry run"
 410 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 411 print_header "Step Build: full build plan (cached and to-be-built dependencies):"
 412 jq -r '."install-plan" | map(."pkg-name" + "-" + ."pkg-version" + " " + ."component-name") | join("\n")' "$BUILDDIR/cache/plan.json"
 413 print_header "Step Build: actual build"
 414 timed $CABALNEWBUILD $TARGETS || exit 1
 415 }
 416
 417 # Cabal lib
 418 #######################################################################
 419
 420 step_doctest() {
 421 print_header "Cabal: doctest"
 422 cabal-env --name doctest-Cabal --transitive QuickCheck
 423 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 424 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 425 }
 426
 427 step_lib_tests() {
 428 print_header "Cabal: tests"
 429
 430 CMD="$($CABALLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 431 (cd Cabal-tests && timed $CMD) || exit 1
 432
 433 CMD="$($CABALLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 434 (cd Cabal-tests && timed $CMD) || exit 1
 435
 436 CMD="$($CABALLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 437 (cd Cabal-tests && timed $CMD) || exit 1
 438
 439 CMD="$($CABALLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 440 (cd Cabal-tests && timed $CMD) || exit 1
 441
 442 CMD="$($CABALLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 443 (cd Cabal-tests && timed $CMD) || exit 1
 444
 445
 446 # See #10284 for why this value is pinned.
 447 HACKAGE_TESTS_INDEX_STATE="--index-state=2024-08-25"
 448
 449 CMD=$($CABALLISTBIN Cabal-tests:test:hackage-tests)
 450 (cd Cabal-tests && timed $CMD read-fields $HACKAGE_TESTS_INDEX_STATE) || exit 1
 451 if $HACKAGETESTSALL; then
 452     (cd Cabal-tests && timed $CMD parsec $HACKAGE_TESTS_INDEX_STATE)    || exit 1
 453     (cd Cabal-tests && timed $CMD roundtrip $HACKAGE_TESTS_INDEX_STATE) || exit 1
 454 else
 455     (cd Cabal-tests && timed $CMD parsec d $HACKAGE_TESTS_INDEX_STATE)    || exit 1
 456     (cd Cabal-tests && timed $CMD roundtrip k $HACKAGE_TESTS_INDEX_STATE) || exit 1
 457 fi
 458 }
 459
 460 # Cabal cabal-testsuite
 461 #######################################################################
 462
 463 step_lib_suite() {
 464 print_header "Cabal: cabal-testsuite"
 465
 466 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes $RTSOPTS"
 467 (cd cabal-testsuite && timed $CMD) || exit 1
 468 }
 469
 470 step_lib_suite_extras() {
 471 for EXTRAHC in $EXTRAHCS; do
 472
 473 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 474 (cd cabal-testsuite && timed $CMD) || exit 1
 475
 476 done
 477 }
 478
 479 # cabal-install
 480 #######################################################################
 481
 482 step_cli_tests() {
 483 print_header "cabal-install: tests"
 484
 485 # this are sorted in asc time used, quicker tests first.
 486 CMD="$($CABALLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 487 (cd cabal-install && timed $CMD) || exit 1
 488
 489 # This doesn't work in parallel either
 490 CMD="$($CABALLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 491 (cd cabal-install && timed $CMD) || exit 1
 492
 493 # Only single job, otherwise we fail with "Heap exhausted"
 494 CMD="$($CABALLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
 495 (cd cabal-install && timed $CMD) || exit 1
 496
 497 # This test-suite doesn't like concurrency
 498 CMD="$($CABALLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 499 (cd cabal-install && timed $CMD) || exit 1
 500 }
 501
 502 # cabal-install cabal-testsuite
 503 #######################################################################
 504
 505 step_cli_suite() {
 506 print_header "cabal-install: cabal-testsuite"
 507
 508 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS  --with-ghc=$HC --hide-successes --intree-cabal-lib=$PWD --test-tmp=$PWD/testdb $RTSOPTS"
 509 (cd cabal-testsuite && timed $CMD) || exit 1
 510 }
 511
 512 # solver-benchmarks
 513 #######################################################################
 514
 515 step_solver_benchmarks_tests() {
 516 print_header "solver-benchmarks: test"
 517
 518 CMD="$($CABALLISTBIN solver-benchmarks:test:unit-tests)"
 519 (cd Cabal && timed $CMD) || exit 1
 520 }
 521
 522 step_solver_benchmarks_run() {
 523 print_header "solver-benchmarks: run"
 524
 525 SOLVEPKG=Chart-diagrams
 526 CMD="$($CABALLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 527 (cd Cabal && timed $CMD) || exit 1
 528 }
 529
 530 # Steps dispatcher
 531 #######################################################################
 532
 533 for step in $STEPS; do
 534     case $step in
 535         print-config)             step_print_config            ;;
 536         print-tool-versions)      step_print_tool_versions     ;;
 537         build)                    step_build                   ;;
 538         doctest)                  step_doctest                 ;;
 539         lib-tests)                step_lib_tests               ;;
 540         cli-tests)                step_cli_tests               ;;
 541         lib-suite)                step_lib_suite               ;;
 542         lib-suite-extras)         step_lib_suite_extras        ;;
 543         cli-suite)                step_cli_suite               ;;
 544         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 545         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 546         time-summary)             step_time_summary            ;;
 547         *)
 548             echo "Invalid step $step"
 549             exit 1
 550             ;;
 551     esac
 552 done
 553
 554 #######################################################################