1# 2016 Jan 15 2# 3# The author disclaims copyright to this source code. In place of 4# a legal notice, here is a blessing: 5# 6# May you do good and not evil. 7# May you find forgiveness for yourself and forgive others. 8# May you share freely, never taking more than you give. 9# 10#************************************************************************* 11# 12 13source [file join [file dirname [info script]] fts5_common.tcl] 14ifcapable !fts5||!fts3 { finish_test ; return } 15set ::testprefix fts5tok2 16 17sqlite3_fts5_register_fts5tokenize db 18 19#------------------------------------------------------------------------- 20# Simple test cases. Using the default (ascii) tokenizer. 21# 22do_execsql_test 1.0 { 23 CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61); 24 CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61); 25} 26 27do_test 1.1 { 28 array unset -nocomplain A 29 30 for {set i 1} {$i < 65536} {incr i} { 31 set input [format "abc%cxyz" $i] 32 set expect [execsql { 33 SELECT input, token, start, end FROM t3 WHERE input=$input 34 }] 35 36 incr A([llength $expect]) 37 38 set res [execsql { 39 SELECT input, token, start, end FROM t5($input) 40 }] 41 if {$res != $expect} {error "failed at i=$i"} 42 } 43} {} 44 45do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {} 46 47finish_test 48