src/testsuite.lisp

   1 ;;********************************************************
   2 ;; file:        testsuite.lisp
   3 ;; description: Initialize the list of testsuite files
   4 ;; date:        Tue Dec 18 2018 - 14:38
   5 ;; author:      Various
   6 ;;********************************************************
   7
   8 (in-package :maxima)
   9
  10 ;;; The Maxima test suite is defined to be the sum of all the tests in
  11 ;;; the files listed in the Maxima option variable $testsuite_files.
  12 ;;; Each entry can either be a plain file name (minus extension), e.g.,
  13 ;;; "testfile.mac", or a Maxima list containing a file name followed by
  14 ;;; another Maxima list of the numbers of the test problems that are
  15 ;;; expected to fail, e.g.  ((mlist simp) "testfile.mac" ((mlist) 7 9 13)).
  16 (defvar $testsuite_files
  17       `((mlist simp)
  18         ((mlist simp) "rtest_rules"
  19          #+allegro ((mlist simp) 11 13))
  20         "rtestnset"
  21         ((mlist simp) "rtest1"
  22          ;; 115 sometimes fails on ECL 16.1.2 on AMD 64/Ubuntu.
  23          #+ecl ((mlist simp) 115 183 185 186)
  24          #-ecl ((mlist simp) 183 185 186))
  25         ((mlist simp) "rtest1a" ((mlist simp) 33))
  26         ((mlist simp) "rtest2" ((mlist simp) 86 95))
  27         "rtest4"
  28         ;; Mark tests that require the documentation as known failures
  29         ;; if this was a lisp-only build
  30         ((mlist simp) "rtest5"
  31                  ,@(and (boundp '*autoconf-lisp-only-build*)
  32                         (symbol-value '*autoconf-lisp-only-build*)
  33                         (list (list '(mlist simp) 78 80))))
  34         ;; 46 = ECL bug #437: mishandling float format
  35         ;; 45 sporadically fails in all tested ECL versions (15.3.7-16.1.3)
  36         ;; 43 fails in ECL up to version 15.3.7
  37         ((mlist simp) "rtest6"
  38          #+ecl ((mlist simp) 43 45 46))
  39         "rtest6a"
  40         "rtest6b"
  41         "rtest7"
  42         "rtest9"
  43         ((mlist simp) "rtest9a"
  44          #+allegro ((mlist simp) 24 27 30 31 35 36 47 48 51 52 55 56 59 60 63 64 67 68 71 72))
  45         ((mlist simp) "rtest10" ((mlist simp) 24 25))
  46         ((mlist simp) "rtest11"
  47          #+allegro ((mlist simp) 136 137 158 174)
  48          #+ccl64 ((mlist simp) 158 174)
  49          #+gcl ((mlist simp) 158 174 175))
  50         "rtest13"
  51         "rtest13s"
  52         ;; ECL 16.1.2 still reliably fails in #307 + #310
  53         ;; and sporadically in 201, 234, 249, 250, 251, 252, 267, 297, 298, 312, 315
  54         ;; and 319
  55         ;; ECL 13.5.1 sporadically fails in 233
  56         ((mlist simp) "rtest14"
  57          #+ecl ((mlist simp) 145 201 233 234 249 250 251 252 267 297 298 307 310 312 315 319))
  58         "rtest15"
  59         ;; ccl versions 1.11 and earlier fail test 50.  Mark it as a
  60         ;; known failure.  Presumably 1.12 will have this fixed.
  61         ;; Test 561 tickles bug in ECL if signed zero enabled; see ECL
  62         ;; bug #329. Fixed post-16.1.3.
  63         ;; Test 50 still sometimes fails in ecl 16.1.2
  64         ((mlist simp) "rtest16"
  65          #-(or ecl allegro) ((mlist simp) 524 525)
  66          #+ecl ((mlist simp) 50 524 525 561)
  67          #+allegro ((mlist simp) 50 241 524 525))
  68         "rtestode"
  69         "rtestode_zp"
  70         "rtest3"
  71         ;; ECL 16.1.2 still fails in #104
  72         ((mlist simp) "rtest8"
  73          #+ecl ((mlist simp) 104))
  74         ((mlist simp) "rtest12"
  75          ((mlist simp) 76 78))
  76         "rexamples"
  77         ((mlist simp) "rtesthyp"
  78          ((mlist simp) 105 112 113 123 124 128))
  79         ((mlist simp) "rtest_hypgeo"
  80          ((mlist simp) 143))
  81         "rtestmt19937"
  82         "rtest_allnummod"
  83         "rtestconjugate"
  84         ((mlist simp) "rtestsum"
  85          ((mlist simp) 3 4 18 75))
  86         ;; Tested with acl 10.1
  87         ((mlist simp) "rtest_trig"
  88          #+allegro ((mlist simp) 58))
  89         "rtest_zeta"
  90         "rtest_diff_invtrig"
  91         "rtest_scalarp"
  92         "rtest_everysome"
  93         ((mlist simp) "rtestint"
  94          ((mlist simp) 232))
  95         "rtest_numth"
  96         "rtestifactor"
  97         ((mlist simp) "rtest_equal"
  98          ((mlist simp) 157 160))
  99         "rtest_abs"
 100         ((mlist simp) "rtest_taylor"
 101          ((mlist simp) 88 91 97 104 128 129))
 102         ((mlist simp) "rtest_dot")
 103         "rtest_mset"
 104         "rtest_boolean"
 105         "rtest_round"
 106         ((mlist simp) "rtest_map"
 107          ((mlist simp) 2 3 4))
 108         ((mlist simp) "rtest_sign"
 109          ((mlist simp) 21 25 30 40 65 72 79))
 110         "rtest_algebraic"
 111         ;; Using the gcl version that is shipped with Ubuntu 16.04 (Long term Support)
 112         ;; test 307 of rtest_gamma results in an error-catch on i386, but not in
 113         ;; i64.
 114         ;; Unfortunately many gcl versions have the same version number so we cannot
 115         ;; test for the buggy version in order to get full support for the lts version.
 116         ;;
 117         ;; On ECL 15.3.7 (but not on ECL versions from 2014 or 2016) rtest_gamma
 118         ;; most of the times crashes on ia32 and sometimes crashes on x64.
 119         ((mlist simp) "rtest_gamma"
 120          #+gcl ((mlist simp) 307)
 121          #+allegro   ((mlist simp) 48 198 663 745))
 122         "rtest_expintegral"
 123         "rtest_signum"
 124         "rtest_lambert_w"
 125         ((mlist simp) "rtest_elliptic"
 126          #-allegro ((mlist simp) 129 143)
 127          #+allegro ((mlist simp) 92 129 143))
 128         "rtest_integrate"
 129         "rtest_integrate_special"
 130         ((mlist simp) "rtest_sqrt"
 131          ((mlist simp) 89))
 132         ((mlist simp) "rtest_carg"
 133          ((mlist simp) 40 41))
 134         ((mlist simp) "rtest_log")
 135         ((mlist simp) "rtest_power"
 136          ((mlist simp) 19 20 26 58))
 137         "rtestdefstruct"
 138         ;; Tested with acl 10.1
 139         ((mlist simp) "rtest_limit"
 140          #+allegro ((mlist simp) 184 185))
 141         "rtest_powerseries"
 142         ((mlist simp) "rtest_laplace"
 143          ((mlist simp) 29 49 50 51 59 60 61 62 78 80))
 144         "rtest_plotoptions"
 145         "rtest_algsys"
 146         "rtest_trace"
 147         "rtest_polynomialp"
 148         ))
 149
 150 ;; The list of share testsuite files. As they are given withut a path
 151 ;; this assumes that file_search_tests is set appropriately so that maxima
 152 ;; can actually find these files. (file_search_maxima is a good choice.)
 153 (defvar $share_testsuite_files
 154   '((mlist simp)
 155     "rtestflatten"
 156     "rtest_z_transform"
 157     "rtest_zeilberger_extreme"
 158     "rtest_zeilberger"
 159     "rtest_boolsimp"
 160     "rtest_eigen"
 161     "rtest_lsquares"
 162     "rtest_pytranslate"
 163
 164     ;; These tests of diff eq code mostly fail.
 165     ;; Probably it's just a matter of loading the
 166     ;; appropriate package or packages for each test.
 167
 168     ;; "rtest_ode1_abel"
 169     ;; "rtest_ode1_riccati"
 170     ;; "rtest_sym"
 171     ;; "rtest_sym2"
 172     ;; "rtestode_kamke_1_1"
 173     ;; "rtestode_kamke_1_2"
 174     ;; "rtestode_kamke_1_3"
 175     ;; "rtestode_kamke_1_4"
 176     ;; "rtestode_kamke_1_5"
 177     ;; "rtestode_kamke_1_6"
 178     ;; "rtestode_murphy_1_1"
 179     ;; "rtestode_murphy_1_2"
 180     ;; "rtestode_murphy_1_3"
 181     ;; "rtestode_murphy_1_4"
 182     ;; "rtestode_murphy_1_5"
 183     ;; "rtestode_murphy_1_6"
 184     ;; "rtestode_murphy_2_1"
 185     ;; "rtestode_murphy_2_2"
 186     ;; "rtestode_murphy_2_3"
 187     ;; "rtestode_murphy_2_4"
 188     ;; "rtestode_murphy_2_5"
 189     ;; "rtestode_kamke_1_1.mac"
 190     ;; "rtestode_kamke_1_2.mac"
 191     ;; "rtestode_kamke_1_3.mac"
 192     ;; "rtestode_kamke_1_4.mac"
 193     ;; "rtestode_kamke_1_5.mac"
 194     ;; ((mlist simp) "rtestode_kamke_1_6.mac"
 195     ;;  ((mlist simp)  20))   ; OK - testsuite issues
 196     ;; "rtestode_kamke_2_1.mac"
 197     ;; ((mlist simp) "rtestode_kamke_2_2.mac"
 198     ;;  ((mlist simp) 132 )) ; OK - testsuite issues
 199     ;; ((mlist simp) "rtestode_kamke_2_3.mac"
 200     ;;  ((mlist simp) 83 107)) ; OK - testsuite issues
 201     ;; "rtestode_kamke_2_4.mac"
 202     ;; "rtestode_kamke_2_5.mac"
 203     ;; "rtest_sym.mac"
 204     ;; "rtest_sym2.mac"
 205     ;; ((mlist simp) "rtest_ode1_riccati.mac"
 206     ;;  (mlist simp 138)) ; OK - testsuite issues
 207     ;; ((mlist simp) "rtest_ode1_abel.mac"
 208     ;;  (mlist simp 45)) ; OK - testsuite issues
 209     ;; "rtestode_odelin.mac"
 210     ;; "rtestode_utils.mac"
 211
 212     "rtest_odelin"
 213     "rtestezunits"
 214     ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 215     #-allegro
 216     "rtest_numericalio"
 217     ((mlist simp) "rtest_simplify_sum"
 218      ((mlist simp) 57))
 219     "rtest_solve_rec"
 220     ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 221     #-allegro
 222     ((mlist simp) "rtest_stringproc"
 223      ;; 12 fails in ECL 15.3.7, 69 in ECL 13.5.1
 224      #+ecl ((mlist simp) 12 69)
 225      #+gcl ((mlist simp) 14))
 226     "rtest_opproperties"
 227     "rtest_stats"
 228     "rtest_distrib"
 229     ((mlist simp) "rtest_descriptive"
 230      ;; Tests that failed for gcl 2.6.12
 231      #+gcl
 232      ((mlist simp) 98 109)
 233      ;; Tests that failed for ACL 10.1
 234      #+allegro
 235      ((mlist simp) 98 99 109 110))
 236     "rtest_interpol"
 237     ((mlist simp) "rtest_levin"
 238      ;; Tested with allegro 10.1
 239      #+allegro ((mlist simp) 71 75 77))
 240     "rtest_fractals"
 241     "rtest_bernstein"
 242     "rtest_atensor"
 243     "rtest_ctensor"
 244     "rtest_itensor"
 245     ;; On sbcl 1.4.10 we still get out-of-memory errors on many
 246     ;; computers on loading lapack => commented these tests out
 247     ;; for SBCL.
 248     ;;
 249     ;;  The following functions were used but not defined: ODEPACK::DUMACH in gcl 2.6.12
 250     ;;  and abcl 1.5.0
 251
 252     ;; Floating-point exception with ECL 16.1.2 in 4 7 8 9 10 12
 253     ;; On a different computer with ECL 16.1.2 test 5 failed, too. No idea why.
 254     ;; On another computer that used the same ECL version test step 15 resulted in an
 255     ;; error-catch without making further information visible in the log.
 256     ;; ECL 13.5.1 errored out in 7-12,14+15 and ECL 13.5.1 to 16.1.2
 257     ;; are known to sporadically error out in steps 6 and 13.
 258     ;;
 259     ;; But ecl 16.1.3 passes all of these tests
 260     #-sbcl
 261     ((mlist simp) "rtest_dgeqrf"
 262      #+ecl ((mlist simp) 1 4 5 6 7 8 9 10 11 12 13 14 15))
 263     #-sbcl
 264     ((mlist simp) "rtest_dgesv"
 265      #+ecl ((mlist simp) 1 3 4 5 6))
 266     ;;  The following functions were used but not defined: ODEPACK::DUMACH in gcl 2.6.12
 267     ;;  and abcl 1.5.0
 268     #-(or gcl abcl gcl)
 269     "rtest_dlsode"
 270     ((mlist simp) "rtest_fourier_elim"
 271      ((mlist simp) 146 147 148 149))
 272     ((mlist simp) "rtest_sequence"
 273      ((mlist simp) 55))
 274     "rtest_cholesky"
 275     "rtest_eigens_by_jacobi"
 276     "rtest_lu"
 277     "rtest_linalg"
 278     "rtest_matrixexp"
 279     ((mlist simp) "rtest_romberg"
 280      ((mlist simp) 18 20))
 281     "rtest_wilcoxon"
 282     "rtest_bitwise"
 283     "rtest_gf"
 284     ((mlist simp) "rtest_namespaces"
 285      #+(or clisp sbcl ccl cmucl ecl)
 286      ((mlist simp) 7))
 287     "rtest_arag"
 288     ((mlist simp) "rtest_pdiff"
 289      #-(or ccl cmucl ecl sbcl gcl clisp)
 290      ((mlist simp) 62))
 291     ((mlist simp) "rtest_to_poly"
 292      #-(or ccl cmucl sbcl gcl ecl)
 293      ((mlist simp) 13 14 15 16 17 18 19 20 25))
 294 ;; Tested with acl 10.1
 295     ((mlist simp) "rtestprintf"
 296      #+allegro
 297      ((mlist simp) 1 2 3 4 5 6 7 8 9 10 11 12 13 14
 298       15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
 299       35 37 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 57
 300       58 59 66 67 68)
 301      #+ecl
 302      ;; ECL 15.3.7 and earlier fails in 54, 66 and 70
 303      ((mlist simp) 38 54 61 63 65 66 69 70)
 304      #+clisp
 305      ((mlist simp) 27 38 61 63 65 69)
 306      #+gcl
 307      ((mlist simp) 7 29 38 39 40 47 48 61 69 70)
 308      ;; The tests that failed with abcl 1.5.0
 309      #+abcl
 310      ((mlist simp) 38 40 61 63 65 69)
 311      #+ccl
 312      ((mlist simp) 27 61)
 313      #+cmucl
 314      ((mlist simp) 61)
 315      #-(or clisp cmucl gcl ecl abcl ccl allegro)
 316      ((mlist simp) 38 61 63 65 69)
 317      )
 318     "rtest_simplex"
 319     ((mlist simp) "rtest_graphs"
 320      ;; Tested with acl 10.1
 321      #+allegro ((mlist simp) 3 4 5))
 322     ((mlist simp) "rtest_abs_integrate"
 323      #-(or cmucl ccl ecl sbcl gcl)
 324      ((mlist) 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 123 125 126 127 178))
 325     "rtest_pochhammer"
 326     ((mlist simp) "rtest_to_poly_solve"
 327      #-(or cmucl ccl ecl gcl sbcl sbcl)
 328      ((mlist simp) 64 74 80 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 55 70 72 73 76 77 78 83 86 87 88 89 90 96 97 102 116 120 124 125 126 127 128 129 130 131 133 134 135 136 137 138 139 140 141 142 143 144 146 147 148 149 157 158 159 162 163 164 165 166 167 168 169 173 179 180 181 183 184 196 197 198 199 200 201 202 203 204 207 208 210 214 215 216 217 222 233 234 240 241 242 243 244 245 253 262 263 265 268 273 274 277 278 282 283 284 285 286 287 288 289 290 291 292 293 295 296 299 300 311 312 318 319 320 322)
 329      #+sbcl
 330      ((mlist simp) 48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240 242 245 265 293 312 319 320 322)
 331      #+clisp
 332      ((mlist simp) 55 102 116 131 133 137 140 141 147 166 167 168 183 184 216
 333       240 242 245 265 293 312 319 320 322)
 334      #+gcl
 335      ((mlist simp) 64 74 80 48 55 83 102 116 131 133 137 140 141 147 149 166 167 168 183 184 212 216 240 242 245 265 293 312 319 320 322)
 336      #+cmucl
 337      ((mlist simp)
 338       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240
 339       242 245 265 293 312 319 320 322)
 340      #+ecl
 341      ((mlist simp)
 342       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 199
 343       200 201 202 203 204 214 216 222 240 242 245 265 268 277 278 293 295 311
 344       312 319 320 322)
 345      #+ccl
 346      ((mlist simp)
 347       48 55 64 74 80 83 102 116 131 133 137 140 141 147 166 167 168 183 184 216 240 242 245 265 293 312 319 320 322))
 348     ;; The tests that failed with abcl 1.5.0
 349     ((mlist simp) "rtest_hg"
 350      #+(or gcl abcl) ((mlist simp) 87 120)
 351      #-(or gcl abcl) ((mlist simp) 87))
 352     ((mlist simp) "rtest_sym"
 353      #-(or sbcl gcl clisp cmucl) ((mlist simp) 12 15 58 64)
 354      #+(or clisp gcl)  ((mlist simp) 15 64)
 355      #+sbcl ((mlist simp) 15 64)
 356      #+cmucl ((mlist simp) 15 64))
 357     ((mlist simp) "rtest_nfloat"
 358      #-(or ecl gcl)
 359      ((mlist simp) 25))
 360     "rtest_mnewton"
 361     "rtest_solve_rat_ineq"
 362     ((mlist simp) "rtest_vect"
 363      #-(or cmucl)
 364      ((mlist simp) 4 9 10 13 16 19 20 21 24 25)
 365      #+cmucl
 366      ((mlist simp) 4 9 10 13 16 20 21 24 25))
 367      "rtest_antid"
 368      "rtest_bffac"
 369      "rtest_diff_form"
 370      "rtest_grobner"
 371      ((mlist simp) "rtest_finance"
 372       ;; Tested with acl 10.1
 373       ;; ECL newer than 15.3.7 doesn't fail any more
 374       #+(or gcl allegro ecl)
 375       ((mlist simp) 9 10 11))
 376      "rtest_fft"
 377      "rtest_rfft"
 378      ((mlist simp) "rtest_decfp"
 379       #+gcl ((mlist simp) 1 2 3 4))
 380      "rtest_wrstcse"
 381      ;; ACL 10.1 cannot load stringproc as it has no (get-encoding) function.
 382      #-(or ecl abcl)
 383      "rtest_draw"
 384      ((mlist simp) "rtest_engineering_format"
 385       #+sbcl
 386       ((mlist simp) 6)
 387       #+abcl
 388       ((mlist simp) 6)
 389       ;; ECL > 15.3.7 doesn't fail here any more
 390       #+ecl
 391       ((mlist simp) 6 8 10 12 14)
 392       #+ccl
 393       ((mlist simp) 6 8 10 12)
 394       ;; Tested with acl 10.1
 395       #+allegro
 396       ((mlist simp) 1 6 8 10 12 14))
 397      )
 398     )