xref: /sqlite-3.40.0/ext/fts5/tool/loadfts5.tcl (revision bd5af9ea) 
1
2
3proc loadfile {f} {
4  set fd [open $f]
5  set data [read $fd]
6  close $fd
7  return $data
8}
9
10set ::nRow 0
11set ::nRowPerDot 1000
12
13proc load_hierachy {dir} {
14  foreach f [glob -nocomplain -dir $dir *] {
15    if {$::O(limit) && $::nRow>=$::O(limit)} break
16    if {[file isdir $f]} {
17      load_hierachy $f
18    } else {
19      db eval { INSERT INTO t1 VALUES($f, loadfile($f)) }
20      incr ::nRow
21
22      if {$::O(trans) && ($::nRow % $::O(trans))==0} {
23        db eval { COMMIT }
24        db eval { INSERT INTO t1(t1) VALUES('integrity-check') }
25        db eval { BEGIN }
26      }
27
28      if {($::nRow % $::nRowPerDot)==0} {
29        puts -nonewline .
30        if {($::nRow % (65*$::nRowPerDot))==0} { puts "" }
31        flush stdout
32      }
33
34    }
35  }
36}
37
38proc usage {} {
39  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE PATH"
40  puts stderr ""
41  puts stderr "Switches are:"
42  puts stderr "  -fts4        (use fts4 instead of fts5)"
43  puts stderr "  -fts5        (use fts5)"
44  puts stderr "  -porter      (use porter tokenizer)"
45  puts stderr "  -delete      (delete the database file before starting)"
46  puts stderr "  -limit N     (load no more than N documents)"
47  puts stderr "  -automerge N (set the automerge parameter to N)"
48  puts stderr "  -crisismerge N (set the crisismerge parameter to N)"
49  puts stderr "  -prefix PREFIX (comma separated prefix= argument)"
50  puts stderr "  -trans N     (commit after N inserts - 0 == never)"
51  puts stderr "  -hashsize N  (set the fts5 hashsize parameter to N)"
52  puts stderr "  -detail MODE (detail mode for fts5 tables)"
53  exit 1
54}
55
56set O(vtab)       fts5
57set O(tok)        ""
58set O(limit)      0
59set O(delete)     0
60set O(automerge)  -1
61set O(crisismerge)  -1
62set O(prefix)     ""
63set O(trans)      0
64set O(hashsize)   -1
65set O(detail)     full
66
67if {[llength $argv]<2} usage
68set nOpt [expr {[llength $argv]-2}]
69for {set i 0} {$i < $nOpt} {incr i} {
70  set arg [lindex $argv $i]
71  switch -- [lindex $argv $i] {
72    -fts4 {
73      set O(vtab) fts4
74    }
75
76    -fts5 {
77      set O(vtab) fts5
78    }
79
80    -porter {
81      set O(tok) ", tokenize=porter"
82    }
83
84    -delete {
85      set O(delete) 1
86    }
87
88    -limit {
89      if { [incr i]>=$nOpt } usage
90      set O(limit) [lindex $argv $i]
91    }
92
93    -trans {
94      if { [incr i]>=$nOpt } usage
95      set O(trans) [lindex $argv $i]
96    }
97
98    -automerge {
99      if { [incr i]>=$nOpt } usage
100      set O(automerge) [lindex $argv $i]
101    }
102
103    -crisismerge {
104      if { [incr i]>=$nOpt } usage
105      set O(crisismerge) [lindex $argv $i]
106    }
107
108    -prefix {
109      if { [incr i]>=$nOpt } usage
110      set O(prefix) [lindex $argv $i]
111    }
112
113    -hashsize {
114      if { [incr i]>=$nOpt } usage
115      set O(hashsize) [lindex $argv $i]
116    }
117
118    -detail {
119      if { [incr i]>=$nOpt } usage
120      set O(detail) [lindex $argv $i]
121    }
122
123    default {
124      usage
125    }
126  }
127}
128
129set dbfile [lindex $argv end-1]
130if {$O(delete)} { file delete -force $dbfile }
131sqlite3 db $dbfile
132catch { load_static_extension db fts5 }
133db func loadfile loadfile
134db eval "PRAGMA page_size=4096"
135
136db eval BEGIN
137  set pref ""
138  if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" }
139  if {$O(vtab)=="fts5"} {
140    append pref ", detail=$O(detail)"
141  }
142  catch {
143    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)"
144    db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);"
145  }
146
147  if {$O(hashsize)>=0} {
148    catch {
149      db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));"
150    }
151  }
152
153
154  if {$O(automerge)>=0} {
155    if {$O(vtab) == "fts5"} {
156      db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
157    } else {
158      db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) }
159    }
160  }
161  if {$O(crisismerge)>=0} {
162    if {$O(vtab) == "fts5"} {
163      db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))}
164    } else {
165    }
166  }
167  load_hierachy [lindex $argv end]
168db eval COMMIT
169puts ""
170
171
172
173