1# 2007 May 10 2# 3# The author disclaims copyright to this source code. In place of 4# a legal notice, here is a blessing: 5# 6# May you do good and not evil. 7# May you find forgiveness for yourself and forgive others. 8# May you share freely, never taking more than you give. 9# 10#*********************************************************************** 11# This file implements regression tests for SQLite library. The 12# focus of this file is generating semi-random strings of SQL 13# (a.k.a. "fuzz") and sending it into the parser to try to generate 14# errors. 15# 16# $Id: fuzz.test,v 1.6 2007/05/11 10:10:33 danielk1977 Exp $ 17 18set testdir [file dirname $argv0] 19source $testdir/tester.tcl 20 21proc fuzz {TemplateList} { 22 set n [llength $TemplateList] 23 set i [expr {int(rand()*$n)}] 24 return [uplevel 1 subst -novar [list [lindex $TemplateList $i]]] 25} 26 27# Fuzzy generation primitives: 28# 29# Literal 30# UnaryOp 31# BinaryOp 32# Expr 33# Table 34# Select 35# Insert 36# 37 38# Returns a string representing an SQL literal. 39# 40proc Literal {} { 41 set TemplateList { 42 456 0 -456 1 -1 43 2147483648 2147483647 2147483649 -2147483647 -2147483648 -2147483649 44 'The' 'first' 'experiments' 'in' 'hardware' 'fault' 'injection' 45 zeroblob(1000) 46 NULL 47 56.1 -56.1 48 123456789.1234567899 49 } 50 fuzz $TemplateList 51} 52 53# Returns a string containing an SQL unary operator (e.g. "+" or "NOT"). 54# 55proc UnaryOp {} { 56 set TemplateList {+ - NOT ~} 57 fuzz $TemplateList 58} 59 60# Returns a string containing an SQL binary operator (e.g. "*" or "/"). 61# 62proc BinaryOp {} { 63 set TemplateList { 64 || * / % + - << >> & | < <= > >= = == != <> AND OR 65 LIKE GLOB {NOT LIKE} 66 } 67 fuzz $TemplateList 68} 69 70# Return the complete text of an SQL expression. 71# 72set ::ExprDepth 0 73proc Expr { {c {}} } { 74 incr ::ExprDepth 75 76 set TemplateList [concat $c {[Literal]}] 77 if {$::ExprDepth < 25} { 78 lappend TemplateList \ 79 {[Expr $c] [BinaryOp] [Expr $c]} \ 80 {[UnaryOp] [Expr $c]} \ 81 {[Expr $c] ISNULL} \ 82 {[Expr $c] NOTNULL} \ 83 {CAST([Expr $c] AS blob)} \ 84 {CAST([Expr $c] AS text)} \ 85 {CAST([Expr $c] AS integer)} \ 86 {CAST([Expr $c] AS real)} \ 87 {CASE WHEN [Expr $c] THEN [Expr $c] ELSE [Expr $c] END} \ 88 {[Literal]} {[Literal]} {[Literal]} 89 } 90 if {$::SelectDepth < 10} { 91 lappend TemplateList \ 92 {([Select 1])} \ 93 {[Expr $c] IN ([Select 1])} \ 94 {[Expr $c] NOT IN ([Select 1])} \ 95 {EXISTS ([Select 1])} \ 96 } 97 set res [fuzz $TemplateList] 98 incr ::ExprDepth -1 99 return $res 100} 101 102# Return a valid table name. 103# 104set ::TableList [list] 105proc Table {} { 106 set TemplateList [concat sqlite_master $::TableList] 107 fuzz $TemplateList 108} 109 110# Return a SELECT statement. 111# 112set ::SelectDepth 0 113proc Select {{isExpr 0}} { 114 incr ::SelectDepth 115 set TemplateList { 116 {SELECT [Expr]} 117 {SELECT [Literal]} 118 } 119 if {$::SelectDepth < 5} { 120 lappend TemplateList \ 121 {SELECT [Expr] FROM ([Select])} \ 122 {SELECT [Expr] FROM [Table]} \ 123 124 if {0 == $isExpr} { 125 lappend TemplateList \ 126 {SELECT [Expr], [Expr] FROM ([Select]) ORDER BY [Expr]} \ 127 {SELECT * FROM ([Select]) ORDER BY [Expr]} \ 128 {SELECT * FROM [Table]} \ 129 {SELECT * FROM [Table] WHERE [Expr]} \ 130 } 131 } 132 set res [fuzz $TemplateList] 133 incr ::SelectDepth -1 134 set res 135} 136 137# Generate and return a fuzzy INSERT statement. 138# 139proc Insert {} { 140 set TemplateList { 141 {INSERT INTO [Table] VALUES([Expr], [Expr], [Expr]);} 142 {INSERT INTO [Table] VALUES([Expr], [Expr], [Expr], [Expr]);} 143 {INSERT INTO [Table] VALUES([Expr], [Expr]);} 144 } 145 fuzz $TemplateList 146} 147 148######################################################################## 149 150set ::log [open fuzzy.log w] 151 152# 153# Usage: do_fuzzy_test <testname> ?<options>? 154# 155# -template 156# -errorlist 157# 158proc do_fuzzy_test {testname args} { 159 set ::fuzzyopts(-errorlist) [list] 160 array set ::fuzzyopts $args 161 lappend ::fuzzyopts(-errorlist) {parser stack overflow} {ORDER BY column} 162 163 for {set ii 0} {$ii < 2000} {incr ii} { 164 do_test ${testname}.$ii { 165 set ::sql [subst $::fuzzyopts(-template)] 166 puts $::log $::sql 167 flush $::log 168 set rc [catch {execsql $::sql} msg] 169 set e 1 170 if {$rc} { 171 set e 0 172 foreach error $::fuzzyopts(-errorlist) { 173 if {0 == [string first $error $msg]} { 174 set e 1 175 break 176 } 177 } 178 } 179 if {$e == 0} { 180 puts "" 181 puts $::sql 182 puts $msg 183 } 184 set e 185 } {1} 186 } 187} 188 189#---------------------------------------------------------------- 190# These tests caused errors that were first caught by the tests 191# in this file. They are still here. 192do_test fuzz-1.1 { 193 execsql { 194 SELECT 'abc' LIKE X'ABCD'; 195 } 196} {0} 197do_test fuzz-1.2 { 198 execsql { 199 SELECT 'abc' LIKE zeroblob(10); 200 } 201} {0} 202do_test fuzz-1.3 { 203 execsql { 204 SELECT zeroblob(10) LIKE 'abc'; 205 } 206} {0} 207do_test fuzz-1.4 { 208 execsql { 209 SELECT (- -21) % NOT (456 LIKE zeroblob(10)); 210 } 211} {0} 212do_test fuzz-1.5 { 213 execsql { 214 SELECT (SELECT ( 215 SELECT (SELECT -2147483648) FROM (SELECT 1) ORDER BY 1 216 )) 217 } 218} {-2147483648} 219do_test fuzz-1.6 { 220 execsql { 221 SELECT 'abc', zeroblob(1) FROM (SELECT 1) ORDER BY 1 222 } 223} [execsql {SELECT 'abc', zeroblob(1)}] 224 225do_test fuzz-1.7 { 226 execsql { 227 SELECT ( SELECT zeroblob(1000) FROM ( 228 SELECT * FROM (SELECT 'first') ORDER BY NOT 'in') 229 ) 230 } 231} [execsql {SELECT zeroblob(1000)}] 232 233do_test fuzz-1.8 { 234 execsql { 235 SELECT CAST(zeroblob(1000) AS text); 236 } 237} {{}} 238 239#---------------------------------------------------------------- 240# Test some fuzzily generated expressions. 241# 242do_fuzzy_test fuzz-2 -template { SELECT [Expr] } 243 244do_test fuzz-3.1 { 245 execsql { 246 CREATE TABLE abc(a, b, c); 247 CREATE TABLE def(a, b, c); 248 CREATE TABLE ghi(a, b, c); 249 } 250} {} 251set ::TableList [list abc def ghi] 252set ::ColumnList [list a b c] 253 254#---------------------------------------------------------------- 255# Test some fuzzily generated SELECT statements. 256# 257do_fuzzy_test fuzz-3.2 -template {[Select]} 258 259#---------------------------------------------------------------- 260# Insert a small amount of data into the database and then run 261# some more generated SELECT statements. 262# 263do_test fuzz-4.1 { 264 execsql { 265 INSERT INTO abc VALUES(1, 2, 3); 266 INSERT INTO abc VALUES(4, 5, 6); 267 INSERT INTO abc VALUES(7, 8, 9); 268 INSERT INTO def VALUES(1, 2, 3); 269 INSERT INTO def VALUES(4, 5, 6); 270 INSERT INTO def VALUES(7, 8, 9); 271 INSERT INTO ghi VALUES(1, 2, 3); 272 INSERT INTO ghi VALUES(4, 5, 6); 273 INSERT INTO ghi VALUES(7, 8, 9); 274 CREATE INDEX abc_i ON abc(a, b, c); 275 CREATE INDEX def_i ON def(c, a, b); 276 CREATE INDEX ghi_i ON ghi(b, c, a); 277 } 278} {} 279do_fuzzy_test fuzz-4.2 -template {[Select]} 280 281#---------------------------------------------------------------- 282# Test some fuzzy INSERT statements: 283# 284do_test fuzz-5.1 {execsql BEGIN} {} 285do_fuzzy_test fuzz-5.2 -template {[Insert]} -errorlist table 286integrity_check fuzz-5.2.integrity 287do_test fuzz-5.3 {execsql COMMIT} {} 288integrity_check fuzz-5.4.integrity 289 290do_fuzzy_test fuzz-6.1 -template {[Select]} 291 292close $::log 293finish_test 294