1# 2015 Apr 24
2#
3# The author disclaims copyright to this source code.  In place of
4# a legal notice, here is a blessing:
5#
6#    May you do good and not evil.
7#    May you find forgiveness for yourself and forgive others.
8#    May you share freely, never taking more than you give.
9#
10#***********************************************************************
11#
12# This file tests that FTS5 handles corrupt databases (i.e. internal
13# inconsistencies in the backing tables) correctly. In this case
14# "correctly" means without crashing.
15#
16
17source [file join [file dirname [info script]] fts5_common.tcl]
18set testprefix fts5corrupt2
19
20# If SQLITE_ENABLE_FTS5 is defined, omit this file.
21ifcapable !fts5 {
22  finish_test
23  return
24}
25sqlite3_fts5_may_be_corrupt 1
26
27# Create a simple FTS5 table containing 100 documents. Each document
28# contains 10 terms, each of which start with the character "x".
29#
30expr srand(0)
31db func rnddoc fts5_rnddoc
32do_execsql_test 1.0 {
33  CREATE VIRTUAL TABLE t1 USING fts5(x);
34  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
35  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
36  INSERT INTO t1 SELECT rnddoc(10) FROM ii;
37}
38set mask [expr 31 << 31]
39
40if 0 {
41
42# Test 1:
43#
44#   For each page in the t1_data table, open a transaction and DELETE
45#   the t1_data entry. Then run:
46#
47#     * an integrity-check, and
48#     * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'"
49#
50#   and check that the corruption is detected in both cases. The
51#   rollback the transaction.
52#
53# Test 2:
54#
55#   Same thing, except instead of deleting a row from t1_data, replace its
56#   blob content with integer value 14.
57#
58foreach {tno stmt} {
59  1 { DELETE FROM t1_data WHERE rowid=$rowid }
60  2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
61} {
62  set tn 0
63  foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
64    incr tn
65    #if {$tn!=224} continue
66
67    do_test 1.$tno.$tn.1.$rowid {
68      execsql { BEGIN }
69      execsql $stmt
70      catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
71    } {1 {database disk image is malformed}}
72
73    if {($rowid & $mask)==0} {
74      # Node is a leaf node, not a b-tree node.
75      do_catchsql_test 1.$tno.$tn.2.$rowid {
76        SELECT rowid FROM t1 WHERE t1 MATCH 'x*'
77      } {1 {database disk image is malformed}}
78    }
79
80    do_execsql_test 1.$tno.$tn.3.$rowid {
81      ROLLBACK;
82      INSERT INTO t1(t1) VALUES('integrity-check');
83    } {}
84  }
85}
86
87}
88
89# Using the same database as the 1.* tests.
90#
91# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
92# of the fts index. For test $i, truncate the rightmost leafpage to $i
93# bytes. Then test both the integrity-check detects the corruption.
94#
95# Also tested is that "MATCH 'x*'" does not crash and sometimes reports
96# corruption. It may not report the db as corrupt because truncating the
97# final leaf to some sizes may create a valid leaf page.
98#
99set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
100set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
101set all [db eval {SELECT rowid FROM t1}]
102sqlite3_db_config db DEFENSIVE 0
103for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
104  do_execsql_test 2.$i.1 {
105    BEGIN;
106      UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
107  }
108
109  do_catchsql_test 2.$i.2 {
110    INSERT INTO t1(t1) VALUES('integrity-check');
111  } {1 {database disk image is malformed}}
112
113  do_test 2.$i.3 {
114    set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}]
115    expr {
116        $res=="1 {database disk image is malformed}"
117     || $res=="0 {$all}"
118    }
119  } 1
120
121  do_execsql_test 2.$i.4 {
122    ROLLBACK;
123    INSERT INTO t1(t1) VALUES('integrity-check');
124  } {}
125}
126
127#-------------------------------------------------------------------------
128# Test that corruption in leaf page headers is detected by queries that use
129# doclist-indexes.
130#
131set doc "A B C D E F G H I J "
132do_execsql_test 3.0 {
133  CREATE VIRTUAL TABLE x3 USING fts5(tt);
134  INSERT INTO x3(x3, rank) VALUES('pgsz', 32);
135  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000)
136  INSERT INTO x3
137  SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
138}
139
140foreach {tn hdr} {
141  1 "\x00\x00\x00\x00"
142  2 "\xFF\xFF\xFF\xFF"
143  3 "\x44\x45"
144} {
145  set tn2 0
146  set nCorrupt 0
147  set nCorrupt2 0
148  foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
149    if {$rowid & $mask} continue
150    incr tn2
151    do_test 3.$tn.$tn2.1 {
152      execsql BEGIN
153
154      set fd [db incrblob main x3_data block $rowid]
155      fconfigure $fd -encoding binary -translation binary
156      set existing [read $fd [string length $hdr]]
157      seek $fd 0
158      puts -nonewline $fd $hdr
159      close $fd
160
161      set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}]
162      if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
163      set {} 1
164    } {1}
165
166    if {($tn2 % 10)==0 && $existing != $hdr} {
167      do_test 3.$tn.$tn2.2 {
168        catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
169      } {1 {database disk image is malformed}}
170    }
171
172    execsql ROLLBACK
173  }
174
175  do_test 3.$tn.x { expr $nCorrupt>0 } 1
176}
177
178#--------------------------------------------------------------------
179#
180set doc "A B C D E F G H I J "
181do_execsql_test 4.0 {
182  CREATE VIRTUAL TABLE x4 USING fts5(tt);
183  INSERT INTO x4(x4, rank) VALUES('pgsz', 32);
184  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
185  INSERT INTO x4
186  SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
187}
188
189foreach {tn nCut} {
190  1 1
191  2 10
192} {
193  set tn2 0
194  set nCorrupt 0
195  foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] {
196    if {$rowid & $mask} continue
197    incr tn2
198    do_test 4.$tn.$tn2 {
199      execsql {
200        BEGIN;
201          UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut)
202          WHERE id = $rowid;
203      }
204
205      set res [catchsql {
206        SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC
207      }]
208      if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
209      set {} 1
210    } {1}
211
212    execsql ROLLBACK
213  }
214
215  # do_test 4.$tn.x { expr $nCorrupt>0 } 1
216}
217
218set doc [string repeat "A B C " 1000]
219do_execsql_test 5.0 {
220  CREATE VIRTUAL TABLE x5 USING fts5(tt);
221  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
222  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
223  INSERT INTO x5 SELECT $doc FROM ii;
224}
225
226foreach {tn hdr} {
227  1 "\x00\x01"
228} {
229  set tn2 0
230  set nCorrupt 0
231  foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] {
232    if {$rowid & $mask} continue
233    incr tn2
234    do_test 5.$tn.$tn2 {
235      execsql BEGIN
236
237      set fd [db incrblob main x5_data block $rowid]
238      fconfigure $fd -encoding binary -translation binary
239      puts -nonewline $fd $hdr
240      close $fd
241
242      catchsql { INSERT INTO x5(x5) VALUES('integrity-check') }
243      set {} {}
244    } {}
245
246    execsql ROLLBACK
247  }
248}
249
250#--------------------------------------------------------------------
251reset_db
252sqlite3_db_config db DEFENSIVE 0
253do_execsql_test 6.1 {
254  CREATE VIRTUAL TABLE x5 USING fts5(tt);
255  INSERT INTO x5 VALUES('a');
256  INSERT INTO x5 VALUES('a a');
257  INSERT INTO x5 VALUES('a a a');
258  INSERT INTO x5 VALUES('a a a a');
259
260  UPDATE x5_docsize SET sz = X'' WHERE id=3;
261}
262proc colsize {cmd i} {
263  $cmd xColumnSize $i
264}
265sqlite3_fts5_create_function db colsize colsize
266
267do_catchsql_test 6.2 {
268  SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a'
269} {1 SQLITE_CORRUPT_VTAB}
270
271
272sqlite3_fts5_may_be_corrupt 0
273finish_test
274