validate.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 HC=ghc-8.2.2
   8 CABAL=cabal
   9 CABALPLAN=cabal-plan
  10 JOBS=4
  11 LIBTESTS=true
  12 CLITESTS=true
  13 CABALSUITETESTS=true
  14 LIBONLY=false
  15 DEPSONLY=false
  16 DOCTEST=false
  17 BENCHMARKS=false
  18 VERBOSE=false
  19 HACKAGETESTSALL=false
  20
  21 TARGETS=""
  22 STEPS=""
  23 EXTRAHCS=""
  24
  25 LISTSTEPS=false
  26
  27 # Help
  28 #######################################################################
  29
  30 show_usage() {
  31 cat <<EOF
  32 ./validate.sh - build & test
  33
  34 Usage: ./validate.sh [options]
  35   A script which runs all the tests.
  36
  37 Available options:
  38   -j, --jobs JOBS                   cabal v2-build -j argument (default:  $JOBS)
  39       --libonly                     Test onlt Cabal-the-library
  40       --cli                         Test both Cabal-the-library and cabal-install
  41       --(no-)run-lib-tests          Run library tests
  42       --(no-)run-cli-tests          Run client tests
  43       --(no-)run-lib-suite          Run cabal-testsuite with library
  44       --(no-)run-cli-suite          Run cabal-testsuite with client
  45   -w, --with-compiler HC            With compiler
  46       --with-cabal CABAL            With cabal-install
  47       --with-cabal-plan CABALPLAN   With cabal-plan
  48       --extra-hc HC                 Extra compiler to run test-suite with
  49       --(no-)doctest                Run doctest on library
  50       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  51       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  52       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  53   -v, --verbose                     Verbose output
  54   -q, --quiet                       Less output
  55   -s, --step STEP                   Run only specific step (can be specified mutliple times)
  56       --list-steps                  List steps and build-targets and exit
  57       --help                        Print this message and exit
  58 EOF
  59 }
  60
  61 # "library"
  62 #######################################################################
  63
  64 OUTPUT=$(mktemp)
  65
  66 RED='\033[0;31m'
  67 GREEN='\033[0;32m'
  68 BLUE='\033[0;34m'
  69 CYAN='\033[0;96m'
  70 RESET='\033[0m' # No Color
  71
  72 JOB_START_TIME=$(date +%s)
  73
  74 timed() {
  75     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  76     echo "$BLUE>>> $PRETTYCMD $RESET"
  77     start_time=$(date +%s)
  78
  79     if $VERBOSE; then
  80         "$@" 2>&1
  81     else
  82         "$@" > "$OUTPUT" 2>&1
  83     fi
  84     # echo "MOCK" > "$OUTPUT"
  85     RET=$?
  86
  87     end_time=$(date +%s)
  88     duration=$((end_time - start_time))
  89     tduration=$((end_time - JOB_START_TIME))
  90
  91     if [ $RET -eq 0 ]; then
  92         if ! $VERBOSE; then
  93             # if output is relatively short, show everything
  94             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
  95                 cat "$OUTPUT"
  96             else
  97                 echo "..."
  98                 tail -n 20 "$OUTPUT"
  99             fi
 100
 101             rm -f "$OUTPUT"
 102         fi
 103
 104         echo "$GREEN<<< $PRETTYCMD $RESET ($duration/$tduration sec)"
 105
 106         # bottom-margin
 107         echo ""
 108     else
 109         if ! $VERBOSE; then
 110             cat "$OUTPUT"
 111         fi
 112
 113         echo "$RED<<< $PRETTYCMD $RESET ($duration/$tduration sec, $RET)"
 114         echo "$RED<<< $* $RESET ($duration/$tduration sec, $RET)"
 115         rm -f "$OUTPUT"
 116         exit 1
 117     fi
 118 }
 119
 120 print_header() {
 121     TITLE=$1
 122     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 123     echo "$CYAN===X============================================================ $(date +%T) ===$RESET" \
 124       | sed "s#X$TITLEPAT=# $TITLE #"
 125
 126 }
 127
 128 # getopt
 129 #######################################################################
 130
 131 while [ $# -gt 0 ]; do
 132     arg=$1
 133     case $arg in
 134         --help)
 135             show_usage
 136             exit
 137             ;;
 138         -j|--jobs)
 139             JOBS="$2"
 140             shift
 141             shift
 142             ;;
 143         --lib-only)
 144             LIBONLY=true
 145             shift
 146             ;;
 147         --cli)
 148             LIBONLY=false
 149             shift
 150             ;;
 151         --run-lib-tests)
 152             LIBTESTS=true
 153             shift
 154             ;;
 155         --no-run-lib-tests)
 156             LIBTESTS=false
 157             shift
 158             ;;
 159         --run-cli-tests)
 160             CLITESTS=true
 161             shift
 162             ;;
 163         --no-run-cli-tests)
 164             CLITESTS=false
 165             shift
 166             ;;
 167         --run-lib-suite)
 168             LIBSUITE=true
 169             shift
 170             ;;
 171         --no-run-lib-suite)
 172             LIBSUITE=false
 173             shift
 174             ;;
 175         --run-cli-suite)
 176             CLISUITE=true
 177             shift
 178             ;;
 179         --no-run-cli-suite)
 180             CLISUITE=false
 181             shift
 182             ;;
 183         -w|--with-compiler)
 184             HC=$2
 185             shift
 186             shift
 187             ;;
 188         --with-cabal)
 189             CABAL=$2
 190             shift
 191             shift
 192             ;;
 193         --with-cabal-plan)
 194             CABALPLAN=$2
 195             shift
 196             shift
 197             ;;
 198         --extra-hc)
 199             EXTRAHCS="$EXTRAHCS $2"
 200             shift
 201             shift
 202             ;;
 203         --doctest)
 204             DOCTEST=true
 205             shift
 206             ;;
 207         --no-doctest)
 208             DOCTEST=false
 209             shift
 210             ;;
 211         --solver-benchmarks)
 212             BENCHMARKS=true
 213             shift
 214             ;;
 215         --no-solver-benchmarks)
 216             BENCHMARKS=false
 217             shift
 218             ;;
 219         --complete-hackage-tests)
 220             HACKAGETESTSALL=true
 221             shift
 222             ;;
 223         --partial-hackage-tests)
 224             HACKAGETESTSALL=false
 225             shift
 226             ;;
 227         -v|--verbose)
 228             VERBOSE=true
 229             shift
 230             ;;
 231         -q|--quiet)
 232             VERBOSE=false
 233             shift
 234             ;;
 235         -s|--step)
 236             STEPS="$STEPS $2"
 237             shift
 238             shift
 239             ;;
 240         --list-steps)
 241             LISTSTEPS=true
 242             shift
 243             ;;
 244         *)
 245             echo "Unknown option $arg"
 246             exit 1
 247     esac
 248 done
 249
 250 # calculate steps and build targets
 251 #######################################################################
 252
 253 # If there are no explicit steps given calculate them
 254 if $LIBONLY; then
 255     CLITESTS=false
 256     CLISUITE=false
 257     BENCHMARKS=false
 258 fi
 259
 260 if [ -z "$STEPS" ]; then
 261     STEPS="print-config print-tool-versions"
 262     STEPS="$STEPS build"
 263     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 264     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 265     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 266     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 267                     then STEPS="$STEPS lib-suite-extras"; fi
 268     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 269     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 270     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 271     STEPS="$STEPS time-summary"
 272 fi
 273
 274 TARGETS="Cabal cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 275 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 276 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 277
 278 if $LISTSTEPS; then
 279   echo "Targets: $TARGETS"
 280   echo "Steps:   $STEPS"
 281   exit
 282 fi
 283
 284 # Adjust runtime configuration
 285 #######################################################################
 286
 287 TESTSUITEJOBS="-j$JOBS"
 288 JOBS="-j$JOBS"
 289
 290 # assume compiler is GHC
 291 RUNHASKELL=$(echo $HC | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 292
 293 if [ "$(uname)" = "Linux" ]; then
 294     ARCH="x86_64-linux"
 295 else
 296     ARCH="x86_64-osx"
 297 fi
 298
 299 if $LIBONLY; then
 300     PROJECTFILE=cabal.project.validate.libonly
 301 else
 302     PROJECTFILE=cabal.project.validate
 303 fi
 304
 305 BASEHC=ghc-$($HC --numeric-version)
 306 BUILDDIR=dist-newstyle-validate-$BASEHC
 307 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 308
 309 CABALNEWBUILD="${CABAL} v2-build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 310 CABALPLANLISTBIN="${CABALPLAN} list-bin --builddir=$BUILDDIR"
 311
 312 # header
 313 #######################################################################
 314
 315 step_print_config() {
 316 print_header print-config
 317
 318 cat <<EOF
 319 compiler:            $HC
 320 runhaskell           $RUNHASKELL
 321 cabal-install:       $CABAL
 322 cabal-plan:          $CABALPLAN
 323 jobs:                $JOBS
 324 Cabal tests:         $LIBTESTS
 325 cabal-install tests: $CLITESTS
 326 cabal-testsuite:     $CABALSUITETESTS
 327 library only:        $LIBONLY
 328 dependencies only:   $DEPSONLY
 329 doctest:             $DOCTEST
 330 benchmarks:          $BENCHMARKS
 331 verbose:             $VERBOSE
 332 extra complers:      $EXTRAHCS
 333
 334 EOF
 335 }
 336
 337 step_print_tool_versions() {
 338 print_header print-tool-versions
 339
 340 timed $HC --version
 341 timed $CABAL --version
 342 timed $CABALPLAN --version
 343
 344 for EXTRAHC in $EXTRAHCS; do
 345     timed $EXTRAHC --version
 346 done
 347 }
 348
 349 step_time_summary() {
 350     print_header END
 351
 352     JOB_END_TIME=$(date +%s)
 353     tduration=$((JOB_END_TIME - JOB_START_TIME))
 354
 355     echo "$CYAN!!! Validation took $tduration seconds. $RESET"
 356 }
 357
 358 # build
 359 #######################################################################
 360
 361 step_build() {
 362 print_header "build"
 363 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 364 timed $CABALNEWBUILD $TARGETS || exit 1
 365 }
 366
 367 # Cabal lib
 368 #######################################################################
 369
 370 step_doctest() {
 371 print_header "Cabal: doctest"
 372 cabal-env --name doctest-Cabal --transitive QuickCheck
 373 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 374 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 375 }
 376
 377 step_lib_tests() {
 378 print_header "Cabal: tests"
 379
 380 CMD="$($CABALPLANLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 381 (cd Cabal-tests && timed $CMD) || exit 1
 382
 383 CMD="$($CABALPLANLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 384 (cd Cabal-tests && timed $CMD) || exit 1
 385
 386 CMD="$($CABALPLANLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 387 (cd Cabal-tests && timed $CMD) || exit 1
 388
 389 CMD="$($CABALPLANLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 390 (cd Cabal-tests && timed $CMD) || exit 1
 391
 392 CMD="$($CABALPLANLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 393 (cd Cabal-tests && timed $CMD) || exit 1
 394
 395 CMD=$($CABALPLANLISTBIN Cabal-tests:test:hackage-tests)
 396 (cd Cabal-tests && timed $CMD read-fields) || exit 1
 397
 398 if $HACKAGETESTSALL; then
 399   (cd Cabal-tests && timed $CMD parsec)    || exit 1
 400   (cd Cabal-tests && timed $CMD roundtrip) || exit 1
 401 else
 402   (cd Cabal-tests && timed $CMD parsec d)    || exit 1
 403   (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
 404 fi
 405 }
 406
 407 # Cabal cabal-testsuite
 408 #######################################################################
 409
 410 step_lib_suite() {
 411 print_header "Cabal: cabal-testsuite"
 412
 413 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes"
 414 (cd cabal-testsuite && timed $CMD) || exit 1
 415 }
 416
 417 step_lib_suite_extras() {
 418 for EXTRAHC in $EXTRAHCS; do
 419
 420 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 421 (cd cabal-testsuite && timed $CMD) || exit 1
 422
 423 done
 424 }
 425
 426 # cabal-install
 427 #######################################################################
 428
 429 step_cli_tests() {
 430 print_header "cabal-install: tests"
 431
 432 # this are sorted in asc time used, quicker tests first.
 433 CMD="$($CABALPLANLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 434 (cd cabal-install && timed $CMD) || exit 1
 435
 436 # This doesn't work in parallel either
 437 CMD="$($CABALPLANLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 438 (cd cabal-install && timed $CMD) || exit 1
 439
 440 # Only single job, otherwise we fail with "Heap exhausted"
 441 CMD="$($CABALPLANLISTBIN cabal-install:test:memory-usage-tests) -j1 --hide-successes"
 442 (cd cabal-install && timed $CMD) || exit 1
 443
 444 # This test-suite doesn't like concurrency
 445 CMD="$($CABALPLANLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 446 (cd cabal-install && timed $CMD) || exit 1
 447 }
 448
 449 # cabal-install cabal-testsuite
 450 #######################################################################
 451
 452 step_cli_suite() {
 453 print_header "cabal-install: cabal-testsuite"
 454
 455 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALPLANLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS --hide-successes"
 456 (cd cabal-testsuite && timed $CMD) || exit 1
 457 }
 458
 459 # solver-benchmarks
 460 #######################################################################
 461
 462 step_solver_benchmarks_tests() {
 463 print_header "solver-benchmarks: test"
 464
 465 CMD="$($CABALPLANLISTBIN solver-benchmarks:test:unit-tests)"
 466 (cd Cabal && timed $CMD) || exit 1
 467 }
 468
 469 step_solver_benchmarks_run() {
 470 print_header "solver-benchmarks: run"
 471
 472 SOLVEPKG=Chart-diagrams
 473 CMD="$($CABALPLANLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALPLANLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 474 (cd Cabal && timed $CMD) || exit 1
 475 }
 476
 477 # Steps dispatcher
 478 #######################################################################
 479
 480 for step in $STEPS; do
 481     case $step in
 482         print-config)             step_print_config            ;;
 483         print-tool-versions)      step_print_tool_versions     ;;
 484         build)                    step_build                   ;;
 485         doctest)                  step_doctest                 ;;
 486         lib-tests)                step_lib_tests               ;;
 487         cli-tests)                step_cli_tests               ;;
 488         lib-suite)                step_lib_suite               ;;
 489         lib-suite-extras)         step_lib_suite_extras        ;;
 490         cli-suite)                step_cli_suite               ;;
 491         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 492         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 493         time-summary)             step_time_summary            ;;
 494         *)
 495             echo "Invalid step $step"
 496             exit 1
 497             ;;
 498     esac
 499 done
 500
 501 #######################################################################