3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
10 #***********************************************************************
13 set testdir [file dirname $argv0]
14 source $testdir/tester.tcl
17 #-------------------------------------------------------------------------
18 # test_suite NAME OPTIONS
20 # where available options are:
22 # -description TITLE (default "")
23 # -initialize SCRIPT (default "")
24 # -shutdown SCRIPT (default "")
25 # -presql SQL (default "")
26 # -files LIST-OF-FILES (default $::ALLTESTS)
27 # -prefix NAME (default "$::NAME.")
28 # -dbconfig SCRIPT (default "")
30 proc test_suite {name args} {
32 set default(-shutdown) ""
33 set default(-initialize) ""
34 set default(-presql) ""
35 set default(-description) "no description supplied (fixme)"
36 set default(-files) ""
37 set default(-prefix) "${name}."
38 set default(-dbconfig) ""
40 array set options [array get default]
41 if {[llength $args]%2} {
42 error "uneven number of options/switches passed to test_suite"
45 set o [array names options ${k}*]
46 if {[llength $o]>1} { error "ambiguous option: $k" }
47 if {[llength $o]==0} { error "unknown option: $k" }
48 set options([lindex $o 0]) $v
51 set ::testspec($name) [array get options]
52 lappend ::testsuitelist $name
55 #-------------------------------------------------------------------------
58 proc test_set {args} {
61 if {[string match -* $a]} {
63 -include { set isExclude 0 }
64 -exclude { set isExclude 1 }
66 error "Unknown switch: $a"
69 } elseif {$isExclude == 0} {
70 foreach f $a { set t($f) 1 }
72 foreach f $a { array unset t $f }
73 foreach f $a { array unset t */$f }
77 return [array names t]
80 #-------------------------------------------------------------------------
81 # Set up the following global list variables containing the names of
82 # various test scripts:
88 foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
89 foreach f [glob -nocomplain $testdir/../ext/rtree/*.test] {
93 if {$::tcl_platform(platform)!="unix"} {
94 set alltests [test_set $alltests -exclude crash.test crash2.test]
96 set alltests [test_set $alltests -exclude {
97 all.test async.test quick.test veryquick.test
98 memleak.test permutations.test soak.test fts3.test
99 mallocAll.test rtree.test
102 set allquicktests [test_set $alltests -exclude {
103 async2.test async3.test backup_ioerr.test corrupt.test
104 corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test
105 crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test
106 fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test
107 misc7.test mutex2.test notify2.test onefile.test pagerfault2.test
108 savepoint4.test savepoint6.test select9.test
109 speed1.test speed1p.test speed2.test speed3.test speed4.test
110 speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
111 thread003.test thread004.test thread005.test trans2.test vacuum3.test
112 incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
113 vtab_err.test walslow.test walcrash.test
114 walthread.test rtree3.test
116 if {[info exists ::env(QUICKTEST_INCLUDE)]} {
117 set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
120 #############################################################################
124 #-------------------------------------------------------------------------
125 # Define the generic test suites:
131 lappend ::testsuitelist xxx
133 test_suite "veryquick" -prefix "" -description {
134 "Very" quick test suite. Runs in less than 5 minutes on a workstation.
135 This test suite is the same as the "quick" tests, except that some files
136 that test malloc and IO errors are omitted.
138 test_set $allquicktests -exclude *malloc* *ioerr* *fault*
141 test_suite "valgrind" -prefix "" -description {
142 Run the "veryquick" test suite with a couple of multi-process tests (that
143 fail under valgrind) omitted.
145 test_set $allquicktests -exclude *malloc* *ioerr* *fault*
149 unset -nocomplain ::G(valgrind)
152 test_suite "quick" -prefix "" -description {
153 Quick test suite. Runs in around 10 minutes on a workstation.
155 test_set $allquicktests
158 test_suite "full" -prefix "" -description {
159 Full test suite. Takes a long time.
163 unset -nocomplain ::G(isquick)
166 test_suite "threads" -prefix "" -description {
167 All multi-threaded tests.
169 notify2.test thread001.test thread002.test thread003.test
170 thread004.test thread005.test walthread.test
173 test_suite "fts3" -prefix "" -description {
174 All FTS3 tests except fts3rnd.test.
176 fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
177 fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
178 fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
179 fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
180 fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test
181 fts3near.test fts3query.test fts3shared.test fts3snippet.test
183 fts3fault.test fts3malloc.test fts3matchinfo.test
185 fts3aux1.test fts3comp1.test
189 lappend ::testsuitelist xxx
190 #-------------------------------------------------------------------------
191 # Define the coverage related test suites:
195 test_suite "coverage-wal" -description {
196 Coverage tests for file wal.c.
198 wal.test wal2.test wal3.test walmode.test
199 walbak.test walhook.test walcrash2.test walcksum.test
200 walfault.test walbig.test walnoshm.test
204 test_suite "coverage-pager" -description {
205 Coverage tests for file pager.c.
207 pager1.test pager2.test pagerfault.test pagerfault2.test
208 walfault.test walbak.test journal2.test tkt-9d68c883.test
212 lappend ::testsuitelist xxx
213 #-------------------------------------------------------------------------
214 # Define the permutation test suites:
217 # Run some tests using pre-allocated page and scratch blocks.
219 test_suite "memsubsys1" -description {
220 Tests using pre-allocated page and scratch blocks
222 test_set $::allquicktests -exclude ioerr5.test malloc5.test
226 sqlite3_config_pagecache 4096 24
227 sqlite3_config_scratch 25000 1
229 autoinstall_test_functions
233 sqlite3_config_pagecache 0 0
234 sqlite3_config_scratch 0 0
236 autoinstall_test_functions
239 # Run some tests using pre-allocated page and scratch blocks. This time
240 # the allocations are too small to use in most cases.
242 # Both ioerr5.test and malloc5.test are excluded because they test the
243 # sqlite3_soft_heap_limit() and sqlite3_release_memory() functionality.
244 # This functionality is disabled if a pre-allocated page block is provided.
246 test_suite "memsubsys2" -description {
247 Tests using small pre-allocated page and scratch blocks
249 test_set $::allquicktests -exclude ioerr5.test malloc5.test
253 sqlite3_config_pagecache 512 5
254 sqlite3_config_scratch 1000 1
256 autoinstall_test_functions
260 sqlite3_config_pagecache 0 0
261 sqlite3_config_scratch 0 0
263 autoinstall_test_functions
266 # Run all tests with the lookaside allocator disabled.
268 test_suite "nolookaside" -description {
269 OOM tests with lookaside disabled
273 sqlite3_config_lookaside 0 0
275 autoinstall_test_functions
279 sqlite3_config_lookaside 100 500
281 autoinstall_test_functions
282 } -files $::allquicktests
284 # Run some tests in SQLITE_CONFIG_SINGLETHREAD mode.
286 test_suite "singlethread" -description {
287 Tests run in SQLITE_CONFIG_SINGLETHREAD mode
291 catch {sqlite3_config singlethread}
293 autoinstall_test_functions
295 delete.test delete2.test insert.test rollback.test select1.test
296 select2.test trans.test update.test vacuum.test types.test
297 types2.test types3.test
301 catch {sqlite3_config serialized}
303 autoinstall_test_functions
306 test_suite "nomutex" -description {
307 Tests run with the SQLITE_OPEN_MULTITHREADED flag passed to sqlite3_open().
309 rename sqlite3 sqlite3_nomutex
310 proc sqlite3 {args} {
311 if {[string range [lindex $args 0] 0 0] ne "-"} {
312 lappend args -fullmutex 0 -nomutex 1
314 uplevel [concat sqlite3_nomutex $args]
317 delete.test delete2.test insert.test rollback.test select1.test
318 select2.test trans.test update.test vacuum.test types.test
319 types2.test types3.test
322 rename sqlite3_nomutex sqlite3
325 # Run some tests in SQLITE_CONFIG_MULTITHREAD mode.
327 test_suite "multithread" -description {
328 Tests run in SQLITE_CONFIG_MULTITHREAD mode
332 catch {sqlite3_config multithread}
334 autoinstall_test_functions
336 delete.test delete2.test insert.test rollback.test select1.test
337 select2.test trans.test update.test vacuum.test types.test
338 types2.test types3.test
342 catch {sqlite3_config serialized}
344 autoinstall_test_functions
347 # Run some tests in SQLITE_OPEN_FULLMUTEX mode.
349 test_suite "fullmutex" -description {
350 Tests run in SQLITE_OPEN_FULLMUTEX mode
352 rename sqlite3 sqlite3_fullmutex
353 proc sqlite3 {args} {
354 if {[string range [lindex $args 0] 0 0] ne "-"} {
355 lappend args -nomutex 0 -fullmutex 1
357 uplevel [concat sqlite3_fullmutex $args]
360 delete.test delete2.test insert.test rollback.test select1.test
361 select2.test trans.test update.test vacuum.test types.test
362 types2.test types3.test
365 rename sqlite3_fullmutex sqlite3
368 # Run some tests using the "onefile" demo.
370 test_suite "onefile" -description {
371 Run some tests using the "test_onefile.c" demo
373 rename sqlite3 sqlite3_onefile
374 proc sqlite3 {args} {
375 if {[string range [lindex $args 0] 0 0] ne "-"} {
378 uplevel [concat sqlite3_onefile $args]
381 conflict.test insert.test insert2.test insert3.test
382 rollback.test select1.test select2.test select3.test
385 rename sqlite3_onefile sqlite3
388 # Run some tests using UTF-16 databases.
390 test_suite "utf16" -description {
391 Run tests using UTF-16 databases
393 pragma encoding = 'UTF-16'
395 alter.test alter3.test
396 auth.test bind.test blob.test capi2.test capi3.test collate1.test
397 collate2.test collate3.test collate4.test collate5.test collate6.test
398 conflict.test date.test delete.test expr.test fkey1.test func.test
399 hook.test index.test insert2.test insert.test interrupt.test in.test
400 intpkey.test ioerr.test join2.test join.test lastinsert.test
401 laststmtchanges.test limit.test lock2.test lock.test main.test
402 memdb.test minmax.test misc1.test misc2.test misc3.test notnull.test
403 null.test progress.test quote.test rowid.test select1.test select2.test
404 select3.test select4.test select5.test select6.test sort.test
405 subselect.test tableapi.test table.test temptable.test
406 trace.test trigger1.test trigger2.test trigger3.test
407 trigger4.test types2.test types.test unique.test update.test
408 vacuum.test view.test where.test
411 # Run some tests in exclusive locking mode.
413 test_suite "exclusive" -description {
414 Run tests in exclusive locking mode.
416 pragma locking_mode = 'exclusive'
418 rollback.test select1.test select2.test
419 malloc.test ioerr.test
422 # Run some tests in exclusive locking mode with truncated journals.
424 test_suite "exclusive-truncate" -description {
425 Run tests in exclusive locking mode and truncate journal mode.
427 pragma locking_mode = 'exclusive';
428 pragma journal_mode = TRUNCATE;
430 delete.test delete2.test insert.test rollback.test select1.test
431 select2.test update.test malloc.test ioerr.test
434 # Run some tests in persistent journal mode.
436 test_suite "persistent_journal" -description {
437 Run tests in persistent-journal mode.
439 pragma journal_mode = persist
441 delete.test delete2.test insert.test rollback.test select1.test
442 select2.test trans.test update.test vacuum.test
445 # Run some tests in truncating journal mode.
447 test_suite "truncate_journal" -description {
448 Run tests in persistent-journal mode.
450 pragma journal_mode = truncate
452 delete.test delete2.test insert.test rollback.test select1.test
453 select2.test trans.test update.test vacuum.test
454 malloc.test ioerr.test
457 # Run some error tests in persistent journal mode.
459 test_suite "persistent_journal_error" -description {
460 Run malloc.test and ioerr.test in persistent-journal mode.
462 pragma journal_mode = persist
464 malloc.test ioerr.test
467 # Run some tests in no journal mode.
469 test_suite "no_journal" -description {
470 Run tests in no-journal mode.
472 pragma journal_mode = persist
474 delete.test delete2.test insert.test rollback.test select1.test
475 select2.test trans.test update.test vacuum.test
478 # Run some error tests in no journal mode.
480 test_suite "no_journal_error" -description {
481 Run malloc.test and ioerr.test in no-journal mode.
483 pragma journal_mode = persist
485 malloc.test ioerr.test
488 # Run some crash-tests in autovacuum mode.
490 test_suite "autovacuum_crash" -description {
491 Run crash.test in autovacuum mode.
493 pragma auto_vacuum = 1
496 # Run some ioerr-tests in autovacuum mode.
498 test_suite "autovacuum_ioerr" -description {
499 Run ioerr.test in autovacuum mode.
501 pragma auto_vacuum = 1
504 # Run tests with an in-memory journal.
506 test_suite "inmemory_journal" -description {
507 Run tests with an in-memory journal file.
509 pragma journal_mode = 'memory'
510 } -files [test_set $::allquicktests -exclude {
511 # Exclude all tests that simulate IO errors.
512 autovacuum_ioerr2.test incrvacuum_ioerr.test ioerr.test
513 ioerr.test ioerr2.test ioerr3.test ioerr4.test ioerr5.test
514 vacuum3.test incrblob_err.test diskfull.test backup_ioerr.test
515 e_fts3.test fts3cov.test fts3malloc.test fts3rnd.test
518 # Exclude test scripts that use tcl IO to access journal files or count
519 # the number of fsync() calls.
520 pager.test exclusive.test jrnlmode.test sync.test misc1.test
521 journal1.test conflict.test crash8.test tkt3457.test io.test
524 pager1.test async4.test corrupt.test filefmt.test pager2.test
525 corrupt5.test corruptA.test pageropt.test
527 # Exclude stmt.test, which expects sub-journals to use temporary files.
530 # WAL mode is different.
535 test_suite "memsys3" -description {
536 Run tests using the allocator in mem3.c.
537 } -files [test_set $::allquicktests -exclude {
538 autovacuum.test delete3.test manydb.test
539 bigrow.test incrblob2.test memdb.test
540 bitvec.test index2.test memsubsys1.test
541 capi3c.test ioerr.test memsubsys2.test
542 capi3.test join3.test pagesize.test
543 collate5.test limit.test backup_ioerr.test
547 sqlite3_reset_auto_extension
549 sqlite3_config_heap 25000000 0
550 sqlite3_config_lookaside 0 0
552 # If both memsys3 and memsys5 are enabled in the build, the call to
553 # [sqlite3_config_heap] will initialize the system to use memsys5.
554 # The following overrides this preference and installs the memsys3
556 sqlite3_install_memsys3
558 install_malloc_faultsim 1
560 autoinstall_test_functions
564 sqlite3_config_heap 0 0
565 sqlite3_config_lookaside 100 500
566 install_malloc_faultsim 1
568 autoinstall_test_functions
573 test_suite "memsys5" -description {
574 Run tests using the allocator in mem5.c.
575 } -files [test_set $::allquicktests -exclude {
576 autovacuum.test delete3.test manydb.test
577 bigrow.test incrblob2.test memdb.test
578 bitvec.test index2.test memsubsys1.test
579 capi3c.test ioerr.test memsubsys2.test
580 capi3.test join3.test pagesize.test
581 collate5.test limit.test zeroblob.test
585 sqlite3_config_heap 25000000 64
586 sqlite3_config_lookaside 0 0
587 install_malloc_faultsim 1
589 autoinstall_test_functions
593 sqlite3_config_heap 0 0
594 sqlite3_config_lookaside 100 500
595 install_malloc_faultsim 1
597 autoinstall_test_functions
600 test_suite "memsys5-2" -description {
601 Run tests using the allocator in mem5.c in a different configuration.
607 sqlite3_config_memstatus 0
608 sqlite3_config_heap 40000000 16
609 sqlite3_config_lookaside 0 0
610 install_malloc_faultsim 1
612 autoinstall_test_functions
616 sqlite3_config_heap 0 0
617 sqlite3_config_lookaside 100 500
618 install_malloc_faultsim 1
620 autoinstall_test_functions
624 ifcapable threadsafe {
625 test_suite "no_mutex_try" -description {
626 The sqlite3_mutex_try() interface always fails
628 test_set $::allquicktests -exclude mutex1.test mutex2.test
632 install_mutex_counters 1
633 set ::disable_mutex_try 1
635 autoinstall_test_functions
639 install_mutex_counters 0
641 autoinstall_test_functions
645 # run_tests "crash_safe_append" -description {
646 # Run crash.test with persistent journals on a SAFE_APPEND file-system.
648 # rename crashsql sa_crashsql
649 # proc crashsql {args} {
650 # set options [lrange $args 0 [expr {[llength $args]-2}]]
651 # lappend options -char safe_append
652 # set sql [lindex $args end]
654 # PRAGMA journal_mode=persistent;
657 # set fd [open test.db-journal w]
658 # puts $fd [string repeat 1234567890 100000]
660 # eval sa_crashsql $options
664 # rename sa_crashsql crashsql
665 # } -files crash.test
667 test_suite "safe_append" -description {
668 Run some tests on a SAFE_APPEND file-system.
670 rename sqlite3 sqlite3_safeappend
671 proc sqlite3 {args} {
672 if {[string range [lindex $args 0] 0 0] ne "-"} {
673 lappend args -vfs devsym
675 uplevel [concat sqlite3_safeappend $args]
677 sqlite3_simulate_device -char safe_append
680 rename sqlite3_shutdown sqlite3
682 test_set $::allquicktests shared_err.test -exclude async3.test
685 # The set of tests to run on the alternative-pcache
686 set perm-alt-pcache-testset {
689 delete.test delete2.test
691 insert.test insert2.test
694 select1.test select2.test
699 foreach discard_rate {0 10 50 90 100} {
700 test_suite "pcache${discard_rate}" -description "
701 Alternative pcache implementation with ${discard_rate}% random discard
705 sqlite3_config_alt_pcache 1 $discard_rate 1
707 autoinstall_test_functions
711 sqlite3_config_alt_pcache 0 0 0
712 sqlite3_config_lookaside 100 500
713 install_malloc_faultsim 1
715 autoinstall_test_functions
716 } -files ${perm-alt-pcache-testset}
719 test_suite "journaltest" -description {
720 Check that pages are synced before being written (test_journal.c).
723 register_jt_vfs -default ""
726 } -files [test_set $::allquicktests -exclude {
727 wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test
728 async4.test bigfile.test
731 if {[info commands register_demovfs] != ""} {
732 test_suite "demovfs" -description {
733 Check that the demovfs (code in test_demovfs.c) more or less works.
739 insert.test insert2.test insert3.test rollback.test
740 select1.test select2.test select3.test
744 test_suite "wal" -description {
745 Run tests with journal_mode=WAL
747 set ::G(savepoint6_iterations) 100
749 unset -nocomplain ::G(savepoint6_iterations)
751 savepoint.test savepoint2.test savepoint6.test
752 trans.test avtrans.test
754 fts3aa.test fts3ab.test fts3ac.test fts3ad.test
755 fts3ae.test fts3af.test fts3ag.test fts3ah.test
756 fts3ai.test fts3aj.test fts3ak.test fts3al.test
757 fts3am.test fts3an.test fts3ao.test fts3b.test
758 fts3c.test fts3d.test fts3e.test fts3query.test
761 test_suite "rtree" -description {
762 All R-tree related tests. Provides coverage of source file rtree.c.
763 } -files [glob -nocomplain $::testdir/../ext/rtree/*.test]
765 test_suite "no_optimization" -description {
766 Run test scripts with optimizations disabled using the
767 sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) interface.
769 where.test where2.test where3.test where4.test where5.test
770 where6.test where7.test where8.test where9.test
771 whereA.test whereB.test wherelimit.test
772 select1.test select2.test select3.test select4.test select5.test
773 select7.test select8.test selectA.test selectC.test
775 optimization_control $::dbhandle all 0
779 #############################################################################
781 # run_tests NAME OPTIONS
783 # where available options are:
789 # -files LIST-OF-FILES
792 proc run_tests {name args} {
793 array set options $args
795 set ::G(perm:name) $name
796 set ::G(perm:prefix) $options(-prefix)
797 set ::G(perm:presql) $options(-presql)
799 set ::G(perm:dbconfig) $options(-dbconfig)
801 uplevel $options(-initialize)
803 foreach file [lsort $options(-files)] {
804 if {[file tail $file] == $file} { set file [file join $::testdir $file] }
805 slave_test_file $file
808 uplevel $options(-shutdown)
811 unset ::G(perm:prefix)
812 unset ::G(perm:presql)
813 unset ::G(perm:dbconfig)
816 proc run_test_suite {name} {
817 if {[info exists ::testspec($name)]==0} {
818 error "No such test suite: $name"
820 uplevel run_tests $name $::testspec($name)
824 puts "Usage: $::argv0 TESTSUITE ?TESTFILE?"
826 puts "Available test-suites are:"
827 foreach k $::testsuitelist {
828 if {[info exists ::testspec($k)]==0} {
829 puts " ----------------------------------------"
832 array set o $::testspec($k)
833 puts "Test suite: \"$k\""
834 set d [string trim $o(-description)]
835 set d [regsub {\n *} $d "\n "]
843 if {[info script] == $argv0} {
845 if {[llength $argv]==0} {
848 set suite [lindex $argv 0]
849 if {[info exists ::testspec($suite)]==0} help
851 if {[llength $argv]>1} { set extra [list -files [lrange $argv 1 end]] }
852 eval run_tests $suite $::testspec($suite) $extra