1# 2007 June 21 2# 3# The author disclaims copyright to this source code. In place of 4# a legal notice, here is a blessing: 5# 6# May you do good and not evil. 7# May you find forgiveness for yourself and forgive others. 8# May you share freely, never taking more than you give. 9# 10#************************************************************************* 11# This file implements regression tests for SQLite library. The focus 12# of this script is testing the pluggable tokeniser feature of the 13# FTS3 module. 14# 15# $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $ 16# 17 18set testdir [file dirname $argv0] 19source $testdir/tester.tcl 20 21# If SQLITE_ENABLE_FTS3 is defined, omit this file. 22ifcapable !fts3 { 23 finish_test 24 return 25} 26 27set ::testprefix fts3atoken 28 29proc escape_string {str} { 30 set out "" 31 foreach char [split $str ""] { 32 scan $char %c i 33 if {$i<=127} { 34 append out $char 35 } else { 36 append out [format {\x%.4x} $i] 37 } 38 } 39 set out 40} 41 42#-------------------------------------------------------------------------- 43# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar 44# function fts3_tokenizer(). The procedure is as follows: 45# 46# 1: Verify that there is no such fts3 tokenizer as 'blah'. 47# 48# 2: Query for the built-in tokenizer 'simple'. Insert a copy of the 49# retrieved value as tokenizer 'blah'. 50# 51# 3: Test that the value returned for tokenizer 'blah' is now the 52# same as that retrieved for 'simple'. 53# 54# 4: Test that it is now possible to create an fts3 table using 55# tokenizer 'blah' (it was not possible in step 1). 56# 57# 5: Test that the table created to use tokenizer 'blah' is usable. 58# 59ifcapable fts3_tokenizer { 60 do_test fts3atoken-1.1 { 61 catchsql { 62 CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); 63 } 64 } {1 {unknown tokenizer: blah}} 65 do_test fts3atoken-1.2 { 66 execsql { 67 SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; 68 } 69 } {0} 70 do_test fts3atoken-1.3 { 71 execsql { 72 SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); 73 } 74 } {1} 75 do_test fts3atoken-1.4 { 76 catchsql { 77 CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); 78 } 79 } {0 {}} 80 do_test fts3atoken-1.5 { 81 execsql { 82 INSERT INTO t1(content) VALUES('There was movement at the station'); 83 INSERT INTO t1(content) VALUES('For the word has passed around'); 84 INSERT INTO t1(content) VALUES('That the colt from ol regret had got'); 85 SELECT content FROM t1 WHERE content MATCH 'movement' 86 } 87 } {{There was movement at the station}} 88} else { 89 do_catchsql_test 1.6 { 90 SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; 91 } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}} 92} 93 94#-------------------------------------------------------------------------- 95# Test cases fts3atoken-2.* test error cases in the scalar function based 96# API for getting and setting tokenizers. 97# 98do_test fts3atoken-2.1 { 99 catchsql { 100 SELECT fts3_tokenizer('nosuchtokenizer'); 101 } 102} {1 {unknown tokenizer: nosuchtokenizer}} 103 104#-------------------------------------------------------------------------- 105# Test cases fts3atoken-3.* test the three built-in tokenizers with a 106# simple input string via the built-in test function. This is as much 107# to test the test function as the tokenizer implementations. 108# 109do_test fts3atoken-3.1 { 110 execsql { 111 SELECT fts3_tokenizer_test('simple', 'I don''t see how'); 112 } 113} {{0 i I 1 don don 2 t t 3 see see 4 how how}} 114do_test fts3atoken-3.2 { 115 execsql { 116 SELECT fts3_tokenizer_test('porter', 'I don''t see how'); 117 } 118} {{0 i I 1 don don 2 t t 3 see see 4 how how}} 119ifcapable icu { 120 do_test fts3atoken-3.3 { 121 execsql { 122 SELECT fts3_tokenizer_test('icu', 'I don''t see how'); 123 } 124 } {{0 i I 1 don't don't 2 see see 3 how how}} 125} 126 127#-------------------------------------------------------------------------- 128# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this 129# tokenizer only has two modes - "thai" and "everybody else". Some other 130# Asian languages (Lao, Khmer etc.) require the same special treatment as 131# Thai, but ICU doesn't support them yet. 132# 133ifcapable icu { 134 135 proc do_icu_test {name locale input output} { 136 set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }] 137 do_test $name { 138 lindex $::out 0 139 } $output 140 } 141 142 do_icu_test fts3atoken-4.1 en_US {} {} 143 do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \ 144 0 test Test 1 cases cases 2 fts3 fts3 145 ] 146 147 # The following test shows that ICU is smart enough to recognise 148 # Thai chararacters, even when the locale is set to English/United 149 # States. 150 # 151 set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a" 152 set output "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 " 153 append output "1 \u0e19\u0e30 \u0e19\u0e30 " 154 append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a" 155 156 do_icu_test fts3atoken-4.3 th_TH $input $output 157 do_icu_test fts3atoken-4.4 en_US $input $output 158 159 # ICU handles an unknown locale by falling back to the default. 160 # So this is not an error. 161 do_icu_test fts3atoken-4.5 MiddleOfTheOcean $input $output 162 163 set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire" 164 append longtoken "AReallocInTheIcuTokenizerCode" 165 166 set input "short tokens then " 167 append input $longtoken 168 set output "0 short short " 169 append output "1 tokens tokens " 170 append output "2 then then " 171 append output "3 [string tolower $longtoken] $longtoken" 172 173 do_icu_test fts3atoken-4.6 MiddleOfTheOcean $input $output 174 do_icu_test fts3atoken-4.7 th_TH $input $output 175 do_icu_test fts3atoken-4.8 en_US $input $output 176 177 do_execsql_test 5.1 { 178 CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); 179 insert into x1 (name) values (NULL); 180 insert into x1 (name) values (NULL); 181 delete from x1; 182 } 183 184 proc cp_to_str {codepoint_list} { 185 set fmt [string repeat %c [llength $codepoint_list]] 186 eval [list format $fmt] $codepoint_list 187 } 188 189 do_test 5.2 { 190 set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}] 191 execsql { INSERT INTO x1 VALUES($str) } 192 } {} 193} 194 195do_test fts3atoken-internal { 196 execsql { SELECT fts3_tokenizer_internal_test() } 197} {ok} 198 199#------------------------------------------------------------------------- 200# Test empty tokenizer names. 201# 202do_catchsql_test 6.1.1 { 203 CREATE VIRTUAL TABLE t3 USING fts4(tokenize=""); 204} {1 {unknown tokenizer: }} 205do_catchsql_test 6.1.2 { 206 CREATE VIRTUAL TABLE t3 USING fts4(tokenize=); 207} {1 {unknown tokenizer: }} 208do_catchsql_test 6.1.3 { 209 CREATE VIRTUAL TABLE t3 USING fts4(tokenize=" "); 210} {1 {unknown tokenizer: }} 211 212do_catchsql_test 6.2.1 { 213 SELECT fts3_tokenizer(NULL); 214} {1 {unknown tokenizer: }} 215ifcapable fts3_tokenizer { 216 do_catchsql_test 6.2.2 { 217 SELECT fts3_tokenizer(NULL, X'1234567812345678'); 218 } {1 {argument type mismatch}} 219 do_catchsql_test 6.2.3 { 220 SELECT fts3_tokenizer(NULL, X'12345678'); 221 } {1 {argument type mismatch}} 222} 223 224 225finish_test 226