433 lines
15 KiB
Plaintext
433 lines
15 KiB
Plaintext
# 2009 December 03
|
|
#
|
|
# May you do good and not evil.
|
|
# May you find forgiveness for yourself and forgive others.
|
|
# May you share freely, never taking more than you give.
|
|
#
|
|
#***********************************************************************
|
|
#
|
|
# The tests in this file are structural coverage tests for FTS3.
|
|
#
|
|
|
|
set testdir [file dirname $argv0]
|
|
source $testdir/tester.tcl
|
|
|
|
# If this build does not include FTS3, skip the tests in this file.
|
|
#
|
|
ifcapable !fts3 { finish_test ; return }
|
|
source $testdir/fts3_common.tcl
|
|
source $testdir/malloc_common.tcl
|
|
|
|
set DO_MALLOC_TEST 0
|
|
set testprefix fts3cov
|
|
|
|
#--------------------------------------------------------------------------
|
|
# When it first needs to read a block from the %_segments table, the FTS3
|
|
# module compiles an SQL statement for that purpose. The statement is
|
|
# stored and reused each subsequent time a block is read. This test case
|
|
# tests the effects of an OOM error occuring while compiling the statement.
|
|
#
|
|
# Similarly, when FTS3 first needs to scan through a set of segment leaves
|
|
# to find a set of documents that matches a term, it allocates a string
|
|
# containing the text of the required SQL, and compiles one or more
|
|
# statements to traverse the leaves. This test case tests that OOM errors
|
|
# that occur while allocating this string and statement are handled correctly
|
|
# also.
|
|
#
|
|
do_test fts3cov-1.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE t1 USING fts3(x);
|
|
INSERT INTO t1(t1) VALUES('nodesize=24');
|
|
BEGIN;
|
|
INSERT INTO t1 VALUES('Is the night chilly and dark?');
|
|
INSERT INTO t1 VALUES('The night is chilly, but not dark.');
|
|
INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
|
|
INSERT INTO t1 VALUES('It covers but not hides the sky.');
|
|
COMMIT;
|
|
SELECT count(*)>0 FROM t1_segments;
|
|
}
|
|
} {1}
|
|
|
|
set DO_MALLOC_TEST 1
|
|
do_restart_select_test fts3cov-1.2 {
|
|
SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
|
|
} {1 2}
|
|
set DO_MALLOC_TEST 0
|
|
|
|
#--------------------------------------------------------------------------
|
|
# When querying the full-text index, if an expected internal node block is
|
|
# missing from the %_segments table, or if a NULL value is stored in the
|
|
# %_segments table instead of a binary blob, database corruption should be
|
|
# reported.
|
|
#
|
|
# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
|
|
# segment b-tree that uses the %_segments table to store internal nodes.
|
|
#
|
|
do_test fts3cov-2.1 {
|
|
execsql {
|
|
INSERT INTO t1(t1) VALUES('nodesize=24');
|
|
BEGIN;
|
|
INSERT INTO t1 VALUES('The moon is behind, and at the full;');
|
|
INSERT INTO t1 VALUES('And yet she looks both small and dull.');
|
|
INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
|
|
INSERT INTO t1 VALUES('''T is a month before the month of May,');
|
|
INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
|
|
INSERT INTO t1 VALUES('The lovely lady, Christabel,');
|
|
INSERT INTO t1 VALUES('Whom her father loves so well,');
|
|
INSERT INTO t1 VALUES('What makes her in the wood so late,');
|
|
INSERT INTO t1 VALUES('A furlong from the castle gate?');
|
|
INSERT INTO t1 VALUES('She had dreams all yesternight');
|
|
INSERT INTO t1 VALUES('Of her own betrothed knight;');
|
|
INSERT INTO t1 VALUES('And she in the midnight wood will pray');
|
|
INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
|
|
COMMIT;
|
|
}
|
|
execsql {
|
|
INSERT INTO t1(t1) VALUES('optimize');
|
|
SELECT substr(hex(root), 1, 2) FROM t1_segdir;
|
|
}
|
|
} {03}
|
|
|
|
# Test the "missing entry" case:
|
|
do_test fts3cov-2.2 {
|
|
set root [db one {SELECT root FROM t1_segdir}]
|
|
read_fts3varint [string range $root 1 end] left_child
|
|
execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
|
|
} {}
|
|
do_error_test fts3cov-2.3 {
|
|
SELECT * FROM t1 WHERE t1 MATCH 'c*'
|
|
} {SQL logic error or missing database}
|
|
|
|
# Test the "replaced with NULL" case:
|
|
do_test fts3cov-2.4 {
|
|
execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
|
|
} {}
|
|
do_error_test fts3cov-2.5 {
|
|
SELECT * FROM t1 WHERE t1 MATCH 'cloud'
|
|
} {SQL logic error or missing database}
|
|
|
|
#--------------------------------------------------------------------------
|
|
# The following tests are to test the effects of OOM errors while storing
|
|
# terms in the pending-hash table. Specifically, while creating doclist
|
|
# blobs to store in the table. More specifically, to test OOM errors while
|
|
# appending column numbers to doclists. For example, if a doclist consists
|
|
# of:
|
|
#
|
|
# <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
|
|
#
|
|
# The following tests check that malloc errors encountered while appending
|
|
# the "0x01 <column N>" data to the dynamically growable blob used to
|
|
# accumulate the doclist in memory are handled correctly.
|
|
#
|
|
do_test fts3cov-3.1 {
|
|
set cols [list]
|
|
set vals [list]
|
|
for {set i 0} {$i < 120} {incr i} {
|
|
lappend cols "col$i"
|
|
lappend vals "'word'"
|
|
}
|
|
execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
|
|
} {}
|
|
set DO_MALLOC_TEST 1
|
|
do_write_test fts3cov-3.2 t2_content "
|
|
INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
|
|
"
|
|
do_write_test fts3cov-3.3 t2_content "
|
|
INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
|
|
"
|
|
do_write_test fts3cov-3.4 t2_content "
|
|
INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
|
|
"
|
|
|
|
#-------------------------------------------------------------------------
|
|
# If too much data accumulates in the pending-terms hash table, it is
|
|
# flushed to the database automatically, even if the transaction has not
|
|
# finished. The following tests check the effects of encountering an OOM
|
|
# while doing this.
|
|
#
|
|
do_test fts3cov-4.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE t3 USING fts3(x);
|
|
INSERT INTO t3(t3) VALUES('nodesize=24');
|
|
INSERT INTO t3(t3) VALUES('maxpending=100');
|
|
}
|
|
} {}
|
|
set DO_MALLOC_TEST 1
|
|
do_write_test fts3cov-4.2 t3_content {
|
|
INSERT INTO t3(docid, x)
|
|
SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
|
|
SELECT 3, 'And comforted fair Geraldine:' UNION ALL
|
|
SELECT 4, '''O well, bright dame, may you command' UNION ALL
|
|
SELECT 5, 'The service of Sir Leoline;' UNION ALL
|
|
SELECT 2, 'And gladly our stout chivalry' UNION ALL
|
|
SELECT 7, 'Will he send forth, and friends withal,' UNION ALL
|
|
SELECT 8, 'To guide and guard you safe and free' UNION ALL
|
|
SELECT 6, 'Home to your noble father''s hall.'''
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# When building the internal tree structure for each segment b-tree, FTS3
|
|
# assumes that the content of each internal node will be less than
|
|
# $nodesize bytes, where $nodesize is the advisory node size. If this turns
|
|
# out to be untrue, then an extra buffer must be malloc'd for each term.
|
|
# This test case tests these paths and the effects of said mallocs failing
|
|
# by inserting insert a document with some fairly large terms into a
|
|
# full-text table with a very small node-size.
|
|
#
|
|
# Test this handling of large terms in three contexts:
|
|
#
|
|
# 1. When flushing the pending-terms table.
|
|
# 2. When optimizing the data structures using the INSERT syntax.
|
|
# 2. When optimizing the data structures using the deprecated SELECT syntax.
|
|
#
|
|
do_test fts3cov-5.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE t4 USING fts3(x);
|
|
INSERT INTO t4(t4) VALUES('nodesize=24');
|
|
}
|
|
} {}
|
|
set DO_MALLOC_TEST 1
|
|
|
|
# Test when flushing pending-terms table.
|
|
do_write_test fts3cov-5.2 t4_content {
|
|
INSERT INTO t4
|
|
SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
|
|
SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL
|
|
SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL
|
|
SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL
|
|
SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
|
|
SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
|
|
}
|
|
|
|
# Test when optimizing via INSERT.
|
|
do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}
|
|
do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }
|
|
|
|
# Test when optimizing via SELECT.
|
|
do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}
|
|
do_write_test fts3cov-5.6 t4_segments {
|
|
SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)
|
|
EXCEPT SELECT 'Index optimized'
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# When merging all segments at a given level to create a single segment
|
|
# at level+1, FTS3 runs a query of the form:
|
|
#
|
|
# SELECT count(*) FROM %_segdir WHERE level = ?
|
|
#
|
|
# The query is compiled the first time this operation is required and
|
|
# reused thereafter. This test aims to test the effects of an OOM while
|
|
# preparing and executing this query for the first time.
|
|
#
|
|
# Then, keep inserting rows into the table so that the effects of an OOM
|
|
# while re-executing the same query can also be tested.
|
|
#
|
|
do_test fts3cov-6.1 {
|
|
execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }
|
|
for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
|
|
execsql { SELECT count(*) FROM t5_segdir }
|
|
} {16}
|
|
|
|
# First time.
|
|
db close
|
|
sqlite3 db test.db
|
|
do_write_test fts3cov-6.2 t5_content {
|
|
INSERT INTO t5 VALUES('segment number 16!');
|
|
}
|
|
|
|
# Second time.
|
|
do_test fts3cov-6.3 {
|
|
for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
|
|
execsql { SELECT count(*) FROM t5_segdir }
|
|
} {17}
|
|
do_write_test fts3cov-6.4 t5_content {
|
|
INSERT INTO t5 VALUES('segment number 16!');
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Update the docid of a row. Test this in two scenarios:
|
|
#
|
|
# 1. When the row being updated is the only row in the table.
|
|
# 2. When it is not.
|
|
#
|
|
# The two cases above take different paths because in case 1 all data
|
|
# structures can simply be emptied before inserting the new row record.
|
|
# In case 2, the data structures actually have to be updated.
|
|
#
|
|
do_test fts3cov-7.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);
|
|
INSERT INTO t7 VALUES('A', 'B', 'C');
|
|
UPDATE t7 SET docid = 5;
|
|
SELECT docid, * FROM t7;
|
|
}
|
|
} {5 A B C}
|
|
do_test fts3cov-7.2 {
|
|
execsql {
|
|
INSERT INTO t7 VALUES('D', 'E', 'F');
|
|
UPDATE t7 SET docid = 1 WHERE docid = 6;
|
|
SELECT docid, * FROM t7;
|
|
}
|
|
} {1 D E F 5 A B C}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# If a set of documents are modified within a transaction, the
|
|
# pending-terms table must be flushed each time a document with a docid
|
|
# less than or equal to the previous docid is modified.
|
|
#
|
|
# This test checks the effects of an OOM error occuring when the
|
|
# pending-terms table is flushed for this reason as part of a DELETE
|
|
# statement.
|
|
#
|
|
do_malloc_test fts3cov-8 -sqlprep {
|
|
BEGIN;
|
|
CREATE VIRTUAL TABLE t8 USING fts3;
|
|
INSERT INTO t8 VALUES('the output of each batch run');
|
|
INSERT INTO t8 VALUES('(possibly a day''s work)');
|
|
INSERT INTO t8 VALUES('was written to two separate disks');
|
|
COMMIT;
|
|
} -sqlbody {
|
|
BEGIN;
|
|
DELETE FROM t8 WHERE rowid = 3;
|
|
DELETE FROM t8 WHERE rowid = 2;
|
|
DELETE FROM t8 WHERE rowid = 1;
|
|
COMMIT;
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Test some branches in the code that handles "special" inserts like:
|
|
#
|
|
# INSERT INTO t1(t1) VALUES('optimize');
|
|
#
|
|
# Also test that an optimize (INSERT method) works on an empty table.
|
|
#
|
|
set DO_MALLOC_TEST 0
|
|
do_test fts3cov-9.1 {
|
|
execsql { CREATE VIRTUAL TABLE xx USING fts3 }
|
|
} {}
|
|
do_error_test fts3cov-9.2 {
|
|
INSERT INTO xx(xx) VALUES('optimise'); -- British spelling
|
|
} {SQL logic error or missing database}
|
|
do_error_test fts3cov-9.3 {
|
|
INSERT INTO xx(xx) VALUES('short');
|
|
} {SQL logic error or missing database}
|
|
do_error_test fts3cov-9.4 {
|
|
INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');
|
|
} {SQL logic error or missing database}
|
|
do_test fts3cov-9.5 {
|
|
execsql { INSERT INTO xx(xx) VALUES('optimize') }
|
|
} {}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Test that a table can be optimized in the middle of a transaction when
|
|
# the pending-terms table is non-empty. This case involves some extra
|
|
# branches because data must be read not only from the database, but
|
|
# also from the pending-terms table.
|
|
#
|
|
do_malloc_test fts3cov-10 -sqlprep {
|
|
CREATE VIRTUAL TABLE t10 USING fts3;
|
|
INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');
|
|
BEGIN;
|
|
INSERT INTO t10 VALUES('You have to get the right balance between');
|
|
} -sqlbody {
|
|
INSERT INTO t10(t10) VALUES('optimize');
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Test a full-text query for a term that was once in the index, but is
|
|
# no longer.
|
|
#
|
|
do_test fts3cov-11.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE xx USING fts3;
|
|
INSERT INTO xx VALUES('one two three');
|
|
INSERT INTO xx VALUES('four five six');
|
|
DELETE FROM xx WHERE docid = 1;
|
|
}
|
|
execsql { SELECT * FROM xx WHERE xx MATCH 'two' }
|
|
} {}
|
|
|
|
|
|
do_malloc_test fts3cov-12 -sqlprep {
|
|
CREATE VIRTUAL TABLE t12 USING fts3;
|
|
INSERT INTO t12 VALUES('is one of the two togther');
|
|
BEGIN;
|
|
INSERT INTO t12 VALUES('one which was appropriate at the time');
|
|
} -sqlbody {
|
|
SELECT * FROM t12 WHERE t12 MATCH 'one'
|
|
}
|
|
|
|
do_malloc_test fts3cov-13 -sqlprep {
|
|
PRAGMA encoding = 'UTF-16';
|
|
CREATE VIRTUAL TABLE t13 USING fts3;
|
|
INSERT INTO t13 VALUES('two scalar functions');
|
|
INSERT INTO t13 VALUES('scalar two functions');
|
|
INSERT INTO t13 VALUES('functions scalar two');
|
|
} -sqlbody {
|
|
SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';
|
|
SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';
|
|
SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';
|
|
}
|
|
|
|
do_execsql_test 14.0 {
|
|
CREATE VIRTUAL TABLE t14 USING fts4(a, b);
|
|
INSERT INTO t14 VALUES('one two three', 'one three four');
|
|
INSERT INTO t14 VALUES('a b c', 'd e a');
|
|
}
|
|
do_execsql_test 14.1 {
|
|
SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'
|
|
} {1}
|
|
do_execsql_test 14.2 {
|
|
SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'
|
|
} {}
|
|
do_execsql_test 14.3 {
|
|
SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'
|
|
} {2}
|
|
do_execsql_test 14.5 {
|
|
SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'
|
|
} {}
|
|
do_catchsql_test 14.6 {
|
|
SELECT rowid FROM t14 WHERE rowid MATCH 'one'
|
|
} {1 {unable to use function MATCH in the requested context}}
|
|
do_catchsql_test 14.7 {
|
|
SELECT rowid FROM t14 WHERE docid MATCH 'one'
|
|
} {1 {unable to use function MATCH in the requested context}}
|
|
|
|
do_execsql_test 15.0 {
|
|
CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);
|
|
INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');
|
|
INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');
|
|
}
|
|
do_execsql_test 15.1 {
|
|
SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
|
|
} {1 2}
|
|
|
|
# Test a corruption case.
|
|
#
|
|
do_execsql_test 16.1 {
|
|
CREATE VIRTUAL TABLE t16 USING fts4;
|
|
INSERT INTO t16 VALUES('theoretical work to examine the relationship');
|
|
INSERT INTO t16 VALUES('solution of our problems on the invisible');
|
|
DELETE FROM t16_content WHERE rowid = 2;
|
|
}
|
|
do_catchsql_test 16.2 {
|
|
SELECT * FROM t16 WHERE t16 MATCH 'invisible'
|
|
} {1 {database disk image is malformed}}
|
|
|
|
# And another corruption test case.
|
|
#
|
|
do_execsql_test 17.1 {
|
|
CREATE VIRTUAL TABLE t17 USING fts4;
|
|
INSERT INTO t17(content) VALUES('one one one');
|
|
UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'
|
|
} {}
|
|
do_catchsql_test 17.2 {
|
|
SELECT * FROM t17 WHERE t17 MATCH 'one'
|
|
} {1 {database disk image is malformed}}
|
|
|
|
|
|
|
|
|
|
finish_test
|