src/testsuite.lisp

   1 ;;********************************************************
   2 ;; file:        testsuite.lisp
   3 ;; description: Initialize the list of testsuite files
   4 ;; date:        Tue Dec 18 2018 - 14:38
   5 ;; author:      Various
   6 ;;********************************************************
   7
   8 (in-package :maxima)
   9
  10 ;;; The Maxima test suite is defined to be the sum of all the tests in
  11 ;;; the files listed in the Maxima option variable $testsuite_files.
  12 ;;; Each entry can either be a plain file name (minus extension), e.g.,
  13 ;;; "testfile.mac", or a Maxima list containing a file name followed by
  14 ;;; another Maxima list of the numbers of the test problems that are
  15 ;;; expected to fail, e.g.  ((mlist simp) "testfile.mac" ((mlist) 7 9 13)).
  16 (defparameter $testsuite_files
  17       `((mlist simp)
  18         ((mlist simp) "rtest_rules"
  19          #+allegro ((mlist simp) 11 13))
  20         "rtestnset"
  21         ((mlist simp) "rtest1"
  22          ;; 115 sometimes fails on ECL 16.1.2 on AMD 64/Ubuntu.
  23          #+ecl ((mlist simp) 115 183 185 186)
  24          #-ecl ((mlist simp) 183 185 186))
  25         ((mlist simp) "rtest1a" ((mlist simp) 33))
  26         ((mlist simp) "rtest2" ((mlist simp) 86 95))
  27         "rtest4"
  28         ;; Mark tests that require the documentation as known failures
  29         ;; if this was a lisp-only build
  30         ((mlist simp) "rtest5"
  31                  ,@(and (boundp '*autoconf-lisp-only-build*)
  32                         (symbol-value '*autoconf-lisp-only-build*)
  33                         (list (list '(mlist simp) 80))))
  34         ;; 46 = ECL bug #437: mishandling float format
  35         ;; 45 sporadically fails in all tested ECL versions (15.3.7-16.1.3)
  36         ;; 43 fails in ECL up to version 15.3.7
  37         ((mlist simp) "rtest6"
  38          #+ecl ((mlist simp) 43 45 46))
  39         "rtest6a"
  40         "rtest6b"
  41         "rtest7"
  42         "rtest9"
  43         ((mlist simp) "rtest9a"
  44          #+allegro ((mlist simp) 24 27 30 31 35 36 47 48 51 52 55 56 59 60 63 64 67 68 71 72))
  45         ((mlist simp) "rtest10" ((mlist simp) 24 25))
  46         ((mlist simp) "rtest11"
  47          #+allegro ((mlist simp) 136 137 158 174)
  48          #+ccl64 ((mlist simp) 158 174)
  49          #+gcl ((mlist simp) 158 174 175))
  50         "rtest13"
  51         "rtest13s"
  52         ;; ECL 16.1.2 still reliably fails in #307 + #310
  53         ;; and sporadically in 201, 234, 249, 250, 251, 252, 267, 297, 298, 312, 315
  54         ;; and 319
  55         ;; ECL 13.5.1 sporadically fails in 233
  56         ((mlist simp) "rtest14"
  57          #+ecl ((mlist simp) 145 201 233 234 249 250 251 252 267 297 298 307 310 312 315 319))
  58         "rtest15"
  59         ;; ccl versions 1.11 and earlier fail test 50.  Mark it as a
  60         ;; known failure.  Presumably 1.12 will have this fixed.
  61         ;; Test 561 tickles bug in ECL if signed zero enabled; see ECL
  62         ;; bug #329. Fixed post-16.1.3.
  63         ;; Test 50 still sometimes fails in ecl 16.1.2
  64         ((mlist simp) "rtest16"
  65          #-(or ecl allegro) ((mlist simp) 524 525)
  66          #+ecl ((mlist simp) 50 524 525 561)
  67          #+allegro ((mlist simp) 50 241 524 525))
  68         "rtestode"
  69         "rtestode_zp"
  70         ((mlist simp) "rtest3" ((mlist simp) 146))
  71         ;; ECL 16.1.2 still fails in #104
  72         ((mlist simp) "rtest8"
  73          #+ecl ((mlist simp) 104))
  74         "rtest12"
  75         "rexamples"
  76         ((mlist simp) "rtesthyp"
  77          ((mlist simp) 105 112 113 123 124 128))
  78         ((mlist simp) "rtest_hypgeo"
  79          ((mlist simp) 143))
  80         "rtestmt19937"
  81         "rtest_allnummod"
  82         ((mlist simp) "rtest_maxmin"
  83                 ((mlist simp) 16 17 40 52 53 57 97 109))
  84         "rtestconjugate"
  85         ((mlist simp) "rtestsum"
  86          ((mlist simp) 23 24 38 95))
  87         ;; Tested with acl 10.1
  88         ((mlist simp) "rtest_trig"
  89          #+allegro ((mlist simp) 58))
  90         "rtest_zeta"
  91         "rtest_diff_invtrig"
  92         "rtest_scalarp"
  93         "rtest_everysome"
  94         ((mlist simp) "rtestint"
  95          ((mlist simp) 232))
  96         "rtest_numth"
  97         "rtestifactor"
  98         ((mlist simp) "rtest_equal"
  99          ((mlist simp) 157 160))
 100         "rtest_abs"
 101         ((mlist simp) "rtest_taylor"
 102          ((mlist simp) 88 91 97 104 128 129))
 103         ((mlist simp) "rtest_dot")
 104         "rtest_mset"
 105         "rtest_boolean"
 106         "rtest_round"
 107         ((mlist simp) "rtest_map"
 108          ((mlist simp) 2 3 4))
 109         ((mlist simp) "rtest_sign"
 110          ((mlist simp) 21 25 30 40 65 72 79))
 111         "rtest_algebraic"
 112         ;; Using the gcl version that is shipped with Ubuntu 16.04 (Long term Support)
 113         ;; test 307 of rtest_gamma results in an error-catch on i386, but not in
 114         ;; i64.
 115         ;; Unfortunately many gcl versions have the same version number so we cannot
 116         ;; test for the buggy version in order to get full support for the lts version.
 117         ;;
 118         ;; On ECL 15.3.7 (but not on ECL versions from 2014 or 2016) rtest_gamma
 119         ;; most of the times crashes on ia32 and sometimes crashes on x64.
 120         ((mlist simp) "rtest_gamma"
 121          #+gcl ((mlist simp) 307)
 122          #+allegro   ((mlist simp) 48 198 663 745))
 123         "rtest_expintegral"
 124         "rtest_signum"
 125         "rtest_lambert_w"
 126         ((mlist simp) "rtest_elliptic"
 127          #-allegro ((mlist simp) 135 149)
 128          #+allegro ((mlist simp) 92 135 149))
 129         "rtest_integrate"
 130         "rtest_integrate_special"
 131         ((mlist simp) "rtest_sqrt"
 132          ((mlist simp) 89))
 133         ((mlist simp) "rtest_carg"
 134          ((mlist simp) 40 41))
 135         ((mlist simp) "rtest_log")
 136         ((mlist simp) "rtest_power"
 137          ((mlist simp) 19 20 26))
 138         "rtestdefstruct"
 139         ;; Tested with acl 10.1
 140         ((mlist simp) "rtest_limit"
 141          #+allegro ((mlist simp) 184 185))
 142         "rtest_powerseries"
 143         ((mlist simp) "rtest_laplace"
 144          ((mlist simp) 29 49 50 51 59 60 61 62 78 80))
 145         "rtest_plotoptions"
 146         "rtest_algsys"
 147         "rtest_trace"
 148         "rtest_polynomialp"
 149         ((mlist simp) "rtest_limit_extra"
 150           ((mlist simp)  42 52 53 59 61 64 67 70 80 82 83 84 87 89 90 91
 151                          94 96 98 100 102 103 104 106 108 111
 152                          124 125 126 127 132 133 135 136 137 139 187))
 153         "rtest_gcd"
 154           ))
 155
 156 ;; The list of share testsuite files. As they are given without a path
 157 ;; this assumes that file_search_tests is set appropriately so that maxima
 158 ;; can actually find these files. (file_search_maxima is a good choice.)
 159 (defparameter $share_testsuite_files
 160   '((mlist simp)
 161     "rtest_pslq"
 162     "rtestflatten"
 163     "rtest_z_transform"
 164     "rtest_zeilberger_extreme"
 165     "rtest_zeilberger"
 166     "rtest_boolsimp"
 167     "rtest_eigen"
 168     "rtest_lsquares"
 169     "rtest_pytranslate"
 170
 171     ;; These tests of diff eq code mostly fail.
 172     ;; Probably it's just a matter of loading the
 173     ;; appropriate package or packages for each test.
 174
 175     ;; "rtest_ode1_abel"
 176     ;; "rtest_ode1_riccati"
 177     ;; "rtest_sym"
 178     ;; "rtest_sym2"
 179     ;; "rtestode_kamke_1_1"
 180     ;; "rtestode_kamke_1_2"
 181     ;; "rtestode_kamke_1_3"
 182     ;; "rtestode_kamke_1_4"
 183     ;; "rtestode_kamke_1_5"
 184     ;; "rtestode_kamke_1_6"
 185     ;; "rtestode_murphy_1_1"
 186     ;; "rtestode_murphy_1_2"
 187     ;; "rtestode_murphy_1_3"
 188     ;; "rtestode_murphy_1_4"
 189     ;; "rtestode_murphy_1_5"
 190     ;; "rtestode_murphy_1_6"
 191     ;; "rtestode_murphy_2_1"
 192     ;; "rtestode_murphy_2_2"
 193     ;; "rtestode_murphy_2_3"
 194     ;; "rtestode_murphy_2_4"
 195     ;; "rtestode_murphy_2_5"
 196     ;; "rtestode_kamke_1_1.mac"
 197     ;; "rtestode_kamke_1_2.mac"
 198     ;; "rtestode_kamke_1_3.mac"
 199     ;; "rtestode_kamke_1_4.mac"
 200     ;; "rtestode_kamke_1_5.mac"
 201     ;; ((mlist simp) "rtestode_kamke_1_6.mac"
 202     ;;  ((mlist simp)  20))   ; OK - testsuite issues
 203     ;; "rtestode_kamke_2_1.mac"
 204     ;; ((mlist simp) "rtestode_kamke_2_2.mac"
 205     ;;  ((mlist simp) 132 )) ; OK - testsuite issues
 206     ;; ((mlist simp) "rtestode_kamke_2_3.mac"
 207     ;;  ((mlist simp) 83 107)) ; OK - testsuite issues
 208     ;; "rtestode_kamke_2_4.mac"
 209     ;; "rtestode_kamke_2_5.mac"
 210     ;; "rtest_sym.mac"
 211     ;; "rtest_sym2.mac"
 212     ;; ((mlist simp) "rtest_ode1_riccati.mac"
 213     ;;  (mlist simp 138)) ; OK - testsuite issues
 214     ;; ((mlist simp) "rtest_ode1_abel.mac"
 215     ;;  (mlist simp 45)) ; OK - testsuite issues
 216     ;; "rtestode_odelin.mac"
 217     ;; "rtestode_utils.mac"
 218
 219     "rtest_odelin"
 220     "rtestezunits"
 221     ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 222     #-allegro
 223     "rtest_numericalio"
 224     ((mlist simp) "rtest_simplify_sum"
 225      ((mlist simp) 57))
 226     "rtest_solve_rec"
 227     ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 228     #-allegro
 229     ((mlist simp) "rtest_stringproc"
 230      ;; 12 fails in ECL 15.3.7, 69 in ECL 13.5.1
 231      #+ecl ((mlist simp) 12 69)
 232      #+gcl ((mlist simp) 14))
 233     #-allegro "rtest_md5sum"
 234     "rtest_opproperties"
 235     "rtest_stats"
 236     "rtest_distrib"
 237     ((mlist simp) "rtest_descriptive"
 238      ;; Tests that failed for gcl 2.6.12
 239      #+gcl
 240      ((mlist simp) 98 109)
 241      ;; Tests that failed for ACL 10.1
 242      #+allegro
 243      ((mlist simp) 98 99 109 110))
 244     "rtest_interpol"
 245     ((mlist simp) "rtest_levin"
 246      ;; Tested with allegro 10.1
 247      #+allegro ((mlist simp) 71 75 77))
 248     "rtest_fractals"
 249     "rtest_bernstein"
 250     "rtest_atensor"
 251     "rtest_ctensor"
 252     "rtest_itensor"
 253     ;; On sbcl 1.4.10 we still get out-of-memory errors on many
 254     ;; computers on loading lapack => commented these tests out
 255     ;; for SBCL.
 256     ;;
 257     ;;  The following functions were used but not defined: ODEPACK::DUMACH in gcl 2.6.12
 258     ;;  and abcl 1.5.0
 259
 260     ;; Floating-point exception with ECL 16.1.2 in 4 7 8 9 10 12
 261     ;; On a different computer with ECL 16.1.2 test 5 failed, too. No idea why.
 262     ;; On another computer that used the same ECL version test step 15 resulted in an
 263     ;; error-catch without making further information visible in the log.
 264     ;; ECL 13.5.1 errored out in 7-12,14+15 and ECL 13.5.1 to 16.1.2
 265     ;; are known to sporadically error out in steps 6 and 13.
 266     ;;
 267     ;; But ecl 16.1.3 passes all of these tests
 268     #-sbcl
 269     ((mlist simp) "rtest_dgeqrf"
 270      #+ecl ((mlist simp) 1 4 5 6 7 8 9 10 11 12 13 14 15))
 271     #-sbcl
 272     ((mlist simp) "rtest_dgesv"
 273      #+ecl ((mlist simp) 1 3 4 5 6))
 274     ;;  The following functions were used but not defined: ODEPACK::DUMACH in gcl 2.6.12
 275     ;;  and abcl 1.5.0
 276     #-(or gcl abcl gcl)
 277     "rtest_dlsode"
 278     ((mlist simp) "rtest_fourier_elim"
 279      ((mlist simp) 146 147 148 149))
 280     ((mlist simp) "rtest_sequence"
 281      ((mlist simp) 55))
 282     "rtest_cholesky"
 283     "rtest_eigens_by_jacobi"
 284     "rtest_lu"
 285     "rtest_linalg"
 286     "rtest_matrixexp"
 287     ((mlist simp) "rtest_romberg"
 288      ((mlist simp) 18 20))
 289     "rtest_wilcoxon"
 290     "rtest_bitwise"
 291     "rtest_gf"
 292     "rtest_arag"
 293     ((mlist simp) "rtest_pdiff"
 294      #-(or ccl cmucl ecl sbcl gcl clisp)
 295      ((mlist simp) 62))
 296     ((mlist simp) "rtest_to_poly"
 297      #-(or ccl cmucl sbcl gcl ecl clisp)
 298      ((mlist simp) 13 14 15 16 17 18 19 20 25))
 299 ;; Tested with acl 10.1
 300     ((mlist simp) "rtestprintf"
 301      #+allegro
 302      ((mlist simp) 1 2 3 4 5 6 7 8 9 10 11 12 13 14
 303       15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
 304       35 37 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 57
 305       58 59 66 67 68)
 306      #+ecl
 307      ;; ECL 15.3.7 and earlier fails in 54, 66 and 70
 308      ((mlist simp) 38 54 61 63 65 66 69 70)
 309      #+clisp
 310      ((mlist simp) 27 38 61 63 65 69)
 311      #+gcl
 312      ((mlist simp) 7 29 38 39 40 47 48 61 69 70)
 313      ;; The tests that failed with abcl 1.5.0
 314      #+abcl
 315      ((mlist simp) 38 40 61 63 65 69)
 316      #+ccl
 317      ((mlist simp) 27 61)
 318      #+cmucl
 319      ((mlist simp) 61)
 320      #-(or clisp cmucl gcl ecl abcl ccl allegro)
 321      ((mlist simp) 38 61 63 65 69)
 322      )
 323     "rtest_simplex"
 324     ((mlist simp) "rtest_graphs"
 325      ;; Tested with acl 10.1
 326      #+allegro ((mlist simp) 3 4 5))
 327     ((mlist simp) "rtest_abs_integrate" ((mlist) 177 253))
 328     "rtest_pochhammer"
 329     ((mlist simp) "rtest_to_poly_solve"
 330      #-(or cmucl ccl ecl gcl sbcl sbcl clisp)
 331      ((mlist simp) 64 74 80 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 55 70 72 73 76 77 78 83 86 87 88 89 90 96 97 102 116 120 124 125 126 127 128 129 130 131 133 134 135 136 137 138 139 140 141 142 143 144 146 147 148 149 157 158 159 162 163 164 165 166 167 168 169 173 179 180 181 183 184 196 197 198 199 200 201 202 203 204 207 208 210 214 215 216 217 222 233 234 240 241 242 243 244 245 253 262 263 265 268 273 274 277 278 282 283 284 285 286 287 288 289 290 291 292 293 295 296 299 300 311 312 318 319 320 322)
 332      #+sbcl
 333      ((mlist simp) 48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240 242 245 265 293 312 319 320 322)
 334      #+clisp
 335      ((mlist simp) 48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 210 216 240 242 245 265 293 312 319 320 322)
 336      #+gcl
 337      ((mlist simp) 64 74 80 48 55 83 102 116 131 133 137 140 141 147 149 166 167 168 183 184 212 216 240 242 245 265 293 312 319 320 322)
 338      #+cmucl
 339      ((mlist simp)
 340       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240
 341       242 245 265 293 312 319 320 322)
 342      #+ecl
 343      ((mlist simp)
 344       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 199
 345       200 201 202 203 204 214 216 222 240 242 245 265 268 277 278 293 295 311
 346       312 319 320 322)
 347      #+ccl
 348      ((mlist simp)
 349       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240 242 245 265 293 312 319 320 322))
 350     ;; The tests that failed with abcl 1.5.0
 351     ((mlist simp) "rtest_hg"
 352      #+(or gcl abcl) ((mlist simp) 87 120)
 353      #-(or gcl abcl) ((mlist simp) 87))
 354     ((mlist simp) "rtest_sym"
 355      #-(or sbcl gcl clisp cmucl) ((mlist simp) 12 15 58 64)
 356      #+(or clisp gcl)  ((mlist simp) 15 64)
 357      #+sbcl ((mlist simp) 15 64)
 358      #+cmucl ((mlist simp) 15 64))
 359     ((mlist simp) "rtest_nfloat"
 360      #-(or ecl gcl)
 361      ((mlist simp) 25))
 362     "rtest_mnewton"
 363     "rtest_solve_rat_ineq"
 364     ((mlist simp) "rtest_vect"
 365      #-(or cmucl)
 366      ((mlist simp) 4 9 10 13 16 19 20 21 24 25)
 367      #+cmucl
 368      ((mlist simp) 4 9 10 13 16 20 21 24 25))
 369      "rtest_antid"
 370      "rtest_bffac"
 371      "rtest_diff_form"
 372      "rtest_grobner"
 373      ((mlist simp) "rtest_finance"
 374       ;; Tested with acl 10.1
 375       ;; ECL newer than 15.3.7 doesn't fail any more
 376       #+(or gcl allegro ecl)
 377       ((mlist simp) 9 10 11))
 378      "rtest_fft"
 379      "rtest_rfft"
 380      ((mlist simp) "rtest_decfp"
 381       #+gcl ((mlist simp) 1 2 3 4))
 382      "rtest_wrstcse"
 383      ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 384      #-(or ecl abcl)
 385      "rtest_draw"
 386      ((mlist simp) "rtest_engineering_format"
 387       #+abcl
 388       ((mlist simp) 6)
 389       #+ccl
 390       ((mlist simp) 6 8 10 12)
 391       ;; Tested with acl 10.1
 392       #+allegro
 393       ((mlist simp) 1 6 8 10 12 14))
 394      )
 395     )