| blob | c43999d2da04c7be895a1250a6302fd3062309bd |
1 # 2009 December 03
2 #
3 # May you do good and not evil.
4 # May you find forgiveness for yourself and forgive others.
5 # May you share freely, never taking more than you give.
6 #
7 #***********************************************************************
8 #
9 # The tests in this file are structural coverage tests for FTS3.
10 #
12 set testdir [file dirname $argv0]
13 source $testdir/tester.tcl
15 # If this build does not include FTS3, skip the tests in this file.
16 #
17 ifcapable !fts3 { finish_test ; return }
18 source $testdir/fts3_common.tcl
19 source $testdir/malloc_common.tcl
21 set DO_MALLOC_TEST 0
22 set testprefix fts3cov
24 #--------------------------------------------------------------------------
25 # When it first needs to read a block from the %_segments table, the FTS3
26 # module compiles an SQL statement for that purpose. The statement is
27 # stored and reused each subsequent time a block is read. This test case
28 # tests the effects of an OOM error occuring while compiling the statement.
29 #
30 # Similarly, when FTS3 first needs to scan through a set of segment leaves
31 # to find a set of documents that matches a term, it allocates a string
32 # containing the text of the required SQL, and compiles one or more
33 # statements to traverse the leaves. This test case tests that OOM errors
34 # that occur while allocating this string and statement are handled correctly
35 # also.
36 #
37 do_test fts3cov-1.1 {
38 execsql {
39 CREATE VIRTUAL TABLE t1 USING fts3(x);
40 INSERT INTO t1(t1) VALUES('nodesize=24');
41 BEGIN;
42 INSERT INTO t1 VALUES('Is the night chilly and dark?');
43 INSERT INTO t1 VALUES('The night is chilly, but not dark.');
44 INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
45 INSERT INTO t1 VALUES('It covers but not hides the sky.');
46 COMMIT;
47 SELECT count(*)>0 FROM t1_segments;
48 }
49 } {1}
51 set DO_MALLOC_TEST 1
52 do_restart_select_test fts3cov-1.2 {
53 SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
54 } {1 2}
55 set DO_MALLOC_TEST 0
57 #--------------------------------------------------------------------------
58 # When querying the full-text index, if an expected internal node block is
59 # missing from the %_segments table, or if a NULL value is stored in the
60 # %_segments table instead of a binary blob, database corruption should be
61 # reported.
62 #
63 # Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
64 # segment b-tree that uses the %_segments table to store internal nodes.
65 #
66 do_test fts3cov-2.1 {
67 execsql {
68 INSERT INTO t1(t1) VALUES('nodesize=24');
69 BEGIN;
70 INSERT INTO t1 VALUES('The moon is behind, and at the full;');
71 INSERT INTO t1 VALUES('And yet she looks both small and dull.');
72 INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
73 INSERT INTO t1 VALUES('''T is a month before the month of May,');
74 INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
75 INSERT INTO t1 VALUES('The lovely lady, Christabel,');
76 INSERT INTO t1 VALUES('Whom her father loves so well,');
77 INSERT INTO t1 VALUES('What makes her in the wood so late,');
78 INSERT INTO t1 VALUES('A furlong from the castle gate?');
79 INSERT INTO t1 VALUES('She had dreams all yesternight');
80 INSERT INTO t1 VALUES('Of her own betrothed knight;');
81 INSERT INTO t1 VALUES('And she in the midnight wood will pray');
82 INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
83 COMMIT;
84 }
85 execsql {
86 INSERT INTO t1(t1) VALUES('optimize');
87 SELECT substr(hex(root), 1, 2) FROM t1_segdir;
88 }
89 } {03}
91 # Test the "missing entry" case:
92 do_test fts3cov-2.2 {
93 set root [db one {SELECT root FROM t1_segdir}]
94 read_fts3varint [string range $root 1 end] left_child
95 execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
96 } {}
97 do_error_test fts3cov-2.3 {
98 SELECT * FROM t1 WHERE t1 MATCH 'c*'
99 } {SQL logic error or missing database}
101 # Test the "replaced with NULL" case:
102 do_test fts3cov-2.4 {
103 execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
104 } {}
105 do_error_test fts3cov-2.5 {
106 SELECT * FROM t1 WHERE t1 MATCH 'cloud'
107 } {SQL logic error or missing database}
109 #--------------------------------------------------------------------------
110 # The following tests are to test the effects of OOM errors while storing
111 # terms in the pending-hash table. Specifically, while creating doclist
112 # blobs to store in the table. More specifically, to test OOM errors while
113 # appending column numbers to doclists. For example, if a doclist consists
114 # of:
115 #
116 # <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
117 #
118 # The following tests check that malloc errors encountered while appending
119 # the "0x01 <column N>" data to the dynamically growable blob used to
120 # accumulate the doclist in memory are handled correctly.
121 #
122 do_test fts3cov-3.1 {
123 set cols [list]
124 set vals [list]
125 for {set i 0} {$i < 120} {incr i} {
126 lappend cols "col$i"
127 lappend vals "'word'"
128 }
129 execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
130 } {}
131 set DO_MALLOC_TEST 1
132 do_write_test fts3cov-3.2 t2_content "
133 INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
134 "
135 do_write_test fts3cov-3.3 t2_content "
136 INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
137 "
138 do_write_test fts3cov-3.4 t2_content "
139 INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
140 "
142 #-------------------------------------------------------------------------
143 # If too much data accumulates in the pending-terms hash table, it is
144 # flushed to the database automatically, even if the transaction has not
145 # finished. The following tests check the effects of encountering an OOM
146 # while doing this.
147 #
148 do_test fts3cov-4.1 {
149 execsql {
150 CREATE VIRTUAL TABLE t3 USING fts3(x);
151 INSERT INTO t3(t3) VALUES('nodesize=24');
152 INSERT INTO t3(t3) VALUES('maxpending=100');
153 }
154 } {}
155 set DO_MALLOC_TEST 1
156 do_write_test fts3cov-4.2 t3_content {
157 INSERT INTO t3(docid, x)
158 SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
159 SELECT 3, 'And comforted fair Geraldine:' UNION ALL
160 SELECT 4, '''O well, bright dame, may you command' UNION ALL
161 SELECT 5, 'The service of Sir Leoline;' UNION ALL
162 SELECT 2, 'And gladly our stout chivalry' UNION ALL
163 SELECT 7, 'Will he send forth, and friends withal,' UNION ALL
164 SELECT 8, 'To guide and guard you safe and free' UNION ALL
165 SELECT 6, 'Home to your noble father''s hall.'''
166 }
168 #-------------------------------------------------------------------------
169 # When building the internal tree structure for each segment b-tree, FTS3
170 # assumes that the content of each internal node will be less than
171 # $nodesize bytes, where $nodesize is the advisory node size. If this turns
172 # out to be untrue, then an extra buffer must be malloc'd for each term.
173 # This test case tests these paths and the effects of said mallocs failing
174 # by inserting insert a document with some fairly large terms into a
175 # full-text table with a very small node-size.
176 #
177 # Test this handling of large terms in three contexts:
178 #
179 # 1. When flushing the pending-terms table.
180 # 2. When optimizing the data structures using the INSERT syntax.
181 # 2. When optimizing the data structures using the deprecated SELECT syntax.
182 #
183 do_test fts3cov-5.1 {
184 execsql {
185 CREATE VIRTUAL TABLE t4 USING fts3(x);
186 INSERT INTO t4(t4) VALUES('nodesize=24');
187 }
188 } {}
189 set DO_MALLOC_TEST 1
191 # Test when flushing pending-terms table.
192 do_write_test fts3cov-5.2 t4_content {
193 INSERT INTO t4
194 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
195 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL
196 SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL
197 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL
198 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
199 SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
200 }
202 # Test when optimizing via INSERT.
203 do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}
204 do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }
206 # Test when optimizing via SELECT.
207 do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}
208 do_write_test fts3cov-5.6 t4_segments {
209 SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)
210 EXCEPT SELECT 'Index optimized'
211 }
213 #-------------------------------------------------------------------------
214 # When merging all segments at a given level to create a single segment
215 # at level+1, FTS3 runs a query of the form:
216 #
217 # SELECT count(*) FROM %_segdir WHERE level = ?
218 #
219 # The query is compiled the first time this operation is required and
220 # reused thereafter. This test aims to test the effects of an OOM while
221 # preparing and executing this query for the first time.
222 #
223 # Then, keep inserting rows into the table so that the effects of an OOM
224 # while re-executing the same query can also be tested.
225 #
226 do_test fts3cov-6.1 {
227 execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }
228 for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
229 execsql { SELECT count(*) FROM t5_segdir }
230 } {16}
232 # First time.
233 db close
234 sqlite3 db test.db
235 do_write_test fts3cov-6.2 t5_content {
236 INSERT INTO t5 VALUES('segment number 16!');
237 }
239 # Second time.
240 do_test fts3cov-6.3 {
241 for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
242 execsql { SELECT count(*) FROM t5_segdir }
243 } {17}
244 do_write_test fts3cov-6.4 t5_content {
245 INSERT INTO t5 VALUES('segment number 16!');
246 }
248 #-------------------------------------------------------------------------
249 # Update the docid of a row. Test this in two scenarios:
250 #
251 # 1. When the row being updated is the only row in the table.
252 # 2. When it is not.
253 #
254 # The two cases above take different paths because in case 1 all data
255 # structures can simply be emptied before inserting the new row record.
256 # In case 2, the data structures actually have to be updated.
257 #
258 do_test fts3cov-7.1 {
259 execsql {
260 CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);
261 INSERT INTO t7 VALUES('A', 'B', 'C');
262 UPDATE t7 SET docid = 5;
263 SELECT docid, * FROM t7;
264 }
265 } {5 A B C}
266 do_test fts3cov-7.2 {
267 execsql {
268 INSERT INTO t7 VALUES('D', 'E', 'F');
269 UPDATE t7 SET docid = 1 WHERE docid = 6;
270 SELECT docid, * FROM t7;
271 }
272 } {1 D E F 5 A B C}
274 #-------------------------------------------------------------------------
275 # If a set of documents are modified within a transaction, the
276 # pending-terms table must be flushed each time a document with a docid
277 # less than or equal to the previous docid is modified.
278 #
279 # This test checks the effects of an OOM error occuring when the
280 # pending-terms table is flushed for this reason as part of a DELETE
281 # statement.
282 #
283 do_malloc_test fts3cov-8 -sqlprep {
284 BEGIN;
285 CREATE VIRTUAL TABLE t8 USING fts3;
286 INSERT INTO t8 VALUES('the output of each batch run');
287 INSERT INTO t8 VALUES('(possibly a day''s work)');
288 INSERT INTO t8 VALUES('was written to two separate disks');
289 COMMIT;
290 } -sqlbody {
291 BEGIN;
292 DELETE FROM t8 WHERE rowid = 3;
293 DELETE FROM t8 WHERE rowid = 2;
294 DELETE FROM t8 WHERE rowid = 1;
295 COMMIT;
296 }
298 #-------------------------------------------------------------------------
299 # Test some branches in the code that handles "special" inserts like:
300 #
301 # INSERT INTO t1(t1) VALUES('optimize');
302 #
303 # Also test that an optimize (INSERT method) works on an empty table.
304 #
305 set DO_MALLOC_TEST 0
306 do_test fts3cov-9.1 {
307 execsql { CREATE VIRTUAL TABLE xx USING fts3 }
308 } {}
309 do_error_test fts3cov-9.2 {
310 INSERT INTO xx(xx) VALUES('optimise'); -- British spelling
311 } {SQL logic error or missing database}
312 do_error_test fts3cov-9.3 {
313 INSERT INTO xx(xx) VALUES('short');
314 } {SQL logic error or missing database}
315 do_error_test fts3cov-9.4 {
316 INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');
317 } {SQL logic error or missing database}
318 do_test fts3cov-9.5 {
319 execsql { INSERT INTO xx(xx) VALUES('optimize') }
320 } {}
322 #-------------------------------------------------------------------------
323 # Test that a table can be optimized in the middle of a transaction when
324 # the pending-terms table is non-empty. This case involves some extra
325 # branches because data must be read not only from the database, but
326 # also from the pending-terms table.
327 #
328 do_malloc_test fts3cov-10 -sqlprep {
329 CREATE VIRTUAL TABLE t10 USING fts3;
330 INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');
331 BEGIN;
332 INSERT INTO t10 VALUES('You have to get the right balance between');
333 } -sqlbody {
334 INSERT INTO t10(t10) VALUES('optimize');
335 }
337 #-------------------------------------------------------------------------
338 # Test a full-text query for a term that was once in the index, but is
339 # no longer.
340 #
341 do_test fts3cov-11.1 {
342 execsql {
343 CREATE VIRTUAL TABLE xx USING fts3;
344 INSERT INTO xx VALUES('one two three');
345 INSERT INTO xx VALUES('four five six');
346 DELETE FROM xx WHERE docid = 1;
347 }
348 execsql { SELECT * FROM xx WHERE xx MATCH 'two' }
349 } {}
352 do_malloc_test fts3cov-12 -sqlprep {
353 CREATE VIRTUAL TABLE t12 USING fts3;
354 INSERT INTO t12 VALUES('is one of the two togther');
355 BEGIN;
356 INSERT INTO t12 VALUES('one which was appropriate at the time');
357 } -sqlbody {
358 SELECT * FROM t12 WHERE t12 MATCH 'one'
359 }
361 do_malloc_test fts3cov-13 -sqlprep {
362 PRAGMA encoding = 'UTF-16';
363 CREATE VIRTUAL TABLE t13 USING fts3;
364 INSERT INTO t13 VALUES('two scalar functions');
365 INSERT INTO t13 VALUES('scalar two functions');
366 INSERT INTO t13 VALUES('functions scalar two');
367 } -sqlbody {
368 SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';
369 SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';
370 SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';
371 }
373 do_execsql_test 14.0 {
374 CREATE VIRTUAL TABLE t14 USING fts4(a, b);
375 INSERT INTO t14 VALUES('one two three', 'one three four');
376 INSERT INTO t14 VALUES('a b c', 'd e a');
377 }
378 do_execsql_test 14.1 {
379 SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'
380 } {1}
381 do_execsql_test 14.2 {
382 SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'
383 } {}
384 do_execsql_test 14.3 {
385 SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'
386 } {2}
387 do_execsql_test 14.5 {
388 SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'
389 } {}
390 do_catchsql_test 14.6 {
391 SELECT rowid FROM t14 WHERE rowid MATCH 'one'
392 } {1 {unable to use function MATCH in the requested context}}
393 do_catchsql_test 14.7 {
394 SELECT rowid FROM t14 WHERE docid MATCH 'one'
395 } {1 {unable to use function MATCH in the requested context}}
397 do_execsql_test 15.0 {
398 CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);
399 INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');
400 INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');
401 }
402 do_execsql_test 15.1 {
403 SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
404 } {1 2}
406 # Test a corruption case.
407 #
408 do_execsql_test 16.1 {
409 CREATE VIRTUAL TABLE t16 USING fts4;
410 INSERT INTO t16 VALUES('theoretical work to examine the relationship');
411 INSERT INTO t16 VALUES('solution of our problems on the invisible');
412 DELETE FROM t16_content WHERE rowid = 2;
413 }
414 do_catchsql_test 16.2 {
415 SELECT * FROM t16 WHERE t16 MATCH 'invisible'
416 } {1 {database disk image is malformed}}
418 # And another corruption test case.
419 #
420 do_execsql_test 17.1 {
421 CREATE VIRTUAL TABLE t17 USING fts4;
422 INSERT INTO t17(content) VALUES('one one one');
423 UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'
424 } {}
425 do_catchsql_test 17.2 {
426 SELECT * FROM t17 WHERE t17 MATCH 'one'
427 } {1 {database disk image is malformed}}
432 finish_test