1cac028baSdrh /*
2cac028baSdrh ** 2016-12-28
3cac028baSdrh **
4cac028baSdrh ** The author disclaims copyright to this source code. In place of
5cac028baSdrh ** a legal notice, here is a blessing:
6cac028baSdrh **
7cac028baSdrh ** May you do good and not evil.
8cac028baSdrh ** May you find forgiveness for yourself and forgive others.
9cac028baSdrh ** May you share freely, never taking more than you give.
10cac028baSdrh **
11cac028baSdrh *************************************************************************
12cac028baSdrh **
13cac028baSdrh ** This file implements "key-value" performance test for SQLite. The
14cac028baSdrh ** purpose is to compare the speed of SQLite for accessing large BLOBs
15cac028baSdrh ** versus reading those same BLOB values out of individual files in the
16cac028baSdrh ** filesystem.
17cac028baSdrh **
182f917e05Sdrh ** Run "kvtest" with no arguments for on-line help, or see comments below.
192f917e05Sdrh **
202f917e05Sdrh ** HOW TO COMPILE:
212f917e05Sdrh **
222f917e05Sdrh ** (1) Gather this source file and a recent SQLite3 amalgamation with its
232f917e05Sdrh ** header into the working directory. You should have:
242f917e05Sdrh **
252f917e05Sdrh ** kvtest.c >--- this file
262f917e05Sdrh ** sqlite3.c \___ SQLite
272f917e05Sdrh ** sqlite3.h / amlagamation & header
282f917e05Sdrh **
292f917e05Sdrh ** (2) Run you compiler against the two C source code files.
302f917e05Sdrh **
312f917e05Sdrh ** (a) On linux or mac:
322f917e05Sdrh **
332f917e05Sdrh ** OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION"
342f917e05Sdrh ** gcc -Os -I. $OPTS kvtest.c sqlite3.c -o kvtest
352f917e05Sdrh **
362f917e05Sdrh ** The $OPTS options can be omitted. The $OPTS merely omit
372f917e05Sdrh ** the need to link against -ldl and -lpthread, or whatever
382f917e05Sdrh ** the equivalent libraries are called on your system.
392f917e05Sdrh **
402f917e05Sdrh ** (b) Windows with MSVC:
412f917e05Sdrh **
422f917e05Sdrh ** cl -I. kvtest.c sqlite3.c
432f917e05Sdrh **
442f917e05Sdrh ** USAGE:
452f917e05Sdrh **
462f917e05Sdrh ** (1) Create a test database by running "kvtest init" with appropriate
472f917e05Sdrh ** options. See the help message for available options.
482f917e05Sdrh **
492f917e05Sdrh ** (2) Construct the corresponding pile-of-files database on disk using
502f917e05Sdrh ** the "kvtest export" command.
512f917e05Sdrh **
522f917e05Sdrh ** (3) Run tests using "kvtest run" against either the SQLite database or
532f917e05Sdrh ** the pile-of-files database and with appropriate options.
542f917e05Sdrh **
552f917e05Sdrh ** For example:
562f917e05Sdrh **
572f917e05Sdrh ** ./kvtest init x1.db --count 100000 --size 10000
582f917e05Sdrh ** mkdir x1
592f917e05Sdrh ** ./kvtest export x1.db x1
602f917e05Sdrh ** ./kvtest run x1.db --count 10000 --max-id 1000000
612f917e05Sdrh ** ./kvtest run x1 --count 10000 --max-id 1000000
62cac028baSdrh */
63cac028baSdrh static const char zHelp[] =
64dfd0de82Sdan "Usage: kvtest COMMAND ARGS...\n"
65cac028baSdrh "\n"
66dfd0de82Sdan " kvtest init DBFILE --count N --size M --pagesize X\n"
67cac028baSdrh "\n"
68cac028baSdrh " Generate a new test database file named DBFILE containing N\n"
69cac028baSdrh " BLOBs each of size M bytes. The page size of the new database\n"
709dc7ad1dSdrh " file will be X. Additional options:\n"
719dc7ad1dSdrh "\n"
729dc7ad1dSdrh " --variance V Randomly vary M by plus or minus V\n"
73cac028baSdrh "\n"
741346fdadSdrh " kvtest export DBFILE DIRECTORY [--tree]\n"
75cac028baSdrh "\n"
76cac028baSdrh " Export all the blobs in the kv table of DBFILE into separate\n"
771346fdadSdrh " files in DIRECTORY. DIRECTORY is created if it does not previously\n"
781346fdadSdrh " exist. If the --tree option is used, then the blobs are written\n"
791346fdadSdrh " into a hierarchy of directories, using names like 00/00/00,\n"
801346fdadSdrh " 00/00/01, 00/00/02, and so forth. Without the --tree option, all\n"
811346fdadSdrh " files are in the top-level directory with names like 000000, 000001,\n"
821346fdadSdrh " 000002, and so forth.\n"
83cac028baSdrh "\n"
84a4327e65Sdrh " kvtest stat DBFILE [options]\n"
859dc7ad1dSdrh "\n"
86a4327e65Sdrh " Display summary information about DBFILE. Options:\n"
87a4327e65Sdrh "\n"
88a4327e65Sdrh " --vacuum Run VACUUM on the database file\n"
899dc7ad1dSdrh "\n"
90dfd0de82Sdan " kvtest run DBFILE [options]\n"
91cac028baSdrh "\n"
92cac028baSdrh " Run a performance test. DBFILE can be either the name of a\n"
93cac028baSdrh " database or a directory containing sample files. Options:\n"
94cac028baSdrh "\n"
9561c565f6Sdrh " --asc Read blobs in ascending order\n"
96cac028baSdrh " --blob-api Use the BLOB API\n"
9761c565f6Sdrh " --cache-size N Database cache size\n"
9861c565f6Sdrh " --count N Read N blobs\n"
99cac028baSdrh " --desc Read blobs in descending order\n"
100f1518907Sdrh " --fsync Synchronous file writes\n"
101f1518907Sdrh " --integrity-check Run \"PRAGMA integrity_check\" after test\n"
102cac028baSdrh " --max-id N Maximum blob key to use\n"
103d1b2566bSdrh " --mmap N Mmap as much as N bytes of DBFILE\n"
10416d62901Sdrh " --multitrans Each read or write in its own transaction\n"
10516d62901Sdrh " --nocheckpoint Omit the checkpoint on WAL mode writes\n"
106f1518907Sdrh " --nosync Set \"PRAGMA synchronous=OFF\"\n"
1072b15f6e1Sdrh " --jmode MODE Set MODE journal mode prior to starting\n"
10861c565f6Sdrh " --random Read blobs in a random order\n"
109cac028baSdrh " --start N Start reading with this blob key\n"
110befcd8adSdan " --stats Output operating stats before exiting\n"
11116d62901Sdrh " --update Do an overwrite test\n"
112cac028baSdrh ;
113cac028baSdrh
114cac028baSdrh /* Reference resources used */
115cac028baSdrh #include <stdio.h>
116cac028baSdrh #include <stdlib.h>
117cac028baSdrh #include <sys/types.h>
118cac028baSdrh #include <sys/stat.h>
119cac028baSdrh #include <assert.h>
120cac028baSdrh #include <string.h>
121cac028baSdrh #include "sqlite3.h"
122cac028baSdrh
123d7b9b97cSdrh #ifndef _WIN32
124d7b9b97cSdrh # include <unistd.h>
125d7b9b97cSdrh #else
126d7b9b97cSdrh /* Provide Windows equivalent for the needed parts of unistd.h */
1271346fdadSdrh # include <direct.h>
128d7b9b97cSdrh # include <io.h>
129d7b9b97cSdrh # define R_OK 2
130cae20d5cSdrh # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
131d7b9b97cSdrh # define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
132d7b9b97cSdrh # define access _access
133d7b9b97cSdrh #endif
134d7b9b97cSdrh
135ac8ba26eSmistachkin #if !defined(_MSC_VER)
136a2e71cedSdrh # include <stdint.h>
137ac8ba26eSmistachkin #endif
138a2e71cedSdrh
139a2e71cedSdrh /*
140a2e71cedSdrh ** The following macros are used to cast pointers to integers and
141a2e71cedSdrh ** integers to pointers. The way you do this varies from one compiler
142a2e71cedSdrh ** to the next, so we have developed the following set of #if statements
143a2e71cedSdrh ** to generate appropriate macros for a wide range of compilers.
144a2e71cedSdrh **
145a2e71cedSdrh ** The correct "ANSI" way to do this is to use the intptr_t type.
146a2e71cedSdrh ** Unfortunately, that typedef is not available on all compilers, or
147a2e71cedSdrh ** if it is available, it requires an #include of specific headers
148a2e71cedSdrh ** that vary from one machine to the next.
149a2e71cedSdrh **
150a2e71cedSdrh ** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on
151a2e71cedSdrh ** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)).
152a2e71cedSdrh ** So we have to define the macros in different ways depending on the
153a2e71cedSdrh ** compiler.
154a2e71cedSdrh */
155a2e71cedSdrh #if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */
156a2e71cedSdrh # define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X))
157a2e71cedSdrh # define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))
1585930f663Sdrh #else
159a2e71cedSdrh # define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X))
160a2e71cedSdrh # define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(intptr_t)(X))
161a2e71cedSdrh #endif
162d7b9b97cSdrh
163cac028baSdrh /*
164cac028baSdrh ** Show thqe help text and quit.
165cac028baSdrh */
showHelp(void)166cac028baSdrh static void showHelp(void){
167cac028baSdrh fprintf(stdout, "%s", zHelp);
168cac028baSdrh exit(1);
169cac028baSdrh }
170cac028baSdrh
171cac028baSdrh /*
172cac028baSdrh ** Show an error message an quit.
173cac028baSdrh */
fatalError(const char * zFormat,...)174cac028baSdrh static void fatalError(const char *zFormat, ...){
175cac028baSdrh va_list ap;
176cac028baSdrh fprintf(stdout, "ERROR: ");
177cac028baSdrh va_start(ap, zFormat);
178cac028baSdrh vfprintf(stdout, zFormat, ap);
179cac028baSdrh va_end(ap);
180cac028baSdrh fprintf(stdout, "\n");
181cac028baSdrh exit(1);
182cac028baSdrh }
183cac028baSdrh
184cac028baSdrh /*
185d1b2566bSdrh ** Return the value of a hexadecimal digit. Return -1 if the input
186d1b2566bSdrh ** is not a hex digit.
187d1b2566bSdrh */
hexDigitValue(char c)188d1b2566bSdrh static int hexDigitValue(char c){
189d1b2566bSdrh if( c>='0' && c<='9' ) return c - '0';
190d1b2566bSdrh if( c>='a' && c<='f' ) return c - 'a' + 10;
191d1b2566bSdrh if( c>='A' && c<='F' ) return c - 'A' + 10;
192d1b2566bSdrh return -1;
193d1b2566bSdrh }
194d1b2566bSdrh
195d1b2566bSdrh /*
196d1b2566bSdrh ** Interpret zArg as an integer value, possibly with suffixes.
197d1b2566bSdrh */
integerValue(const char * zArg)198d1b2566bSdrh static int integerValue(const char *zArg){
199d1b2566bSdrh int v = 0;
200d1b2566bSdrh static const struct { char *zSuffix; int iMult; } aMult[] = {
201d1b2566bSdrh { "KiB", 1024 },
202d1b2566bSdrh { "MiB", 1024*1024 },
203d1b2566bSdrh { "GiB", 1024*1024*1024 },
204d1b2566bSdrh { "KB", 1000 },
205d1b2566bSdrh { "MB", 1000000 },
206d1b2566bSdrh { "GB", 1000000000 },
207d1b2566bSdrh { "K", 1000 },
208d1b2566bSdrh { "M", 1000000 },
209d1b2566bSdrh { "G", 1000000000 },
210d1b2566bSdrh };
211d1b2566bSdrh int i;
212d1b2566bSdrh int isNeg = 0;
213d1b2566bSdrh if( zArg[0]=='-' ){
214d1b2566bSdrh isNeg = 1;
215d1b2566bSdrh zArg++;
216d1b2566bSdrh }else if( zArg[0]=='+' ){
217d1b2566bSdrh zArg++;
218d1b2566bSdrh }
219d1b2566bSdrh if( zArg[0]=='0' && zArg[1]=='x' ){
220d1b2566bSdrh int x;
221d1b2566bSdrh zArg += 2;
222d1b2566bSdrh while( (x = hexDigitValue(zArg[0]))>=0 ){
223d1b2566bSdrh v = (v<<4) + x;
224d1b2566bSdrh zArg++;
225d1b2566bSdrh }
226d1b2566bSdrh }else{
227d1b2566bSdrh while( zArg[0]>='0' && zArg[0]<='9' ){
228d1b2566bSdrh v = v*10 + zArg[0] - '0';
229d1b2566bSdrh zArg++;
230d1b2566bSdrh }
231d1b2566bSdrh }
232d1b2566bSdrh for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){
233d1b2566bSdrh if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
234d1b2566bSdrh v *= aMult[i].iMult;
235d1b2566bSdrh break;
236d1b2566bSdrh }
237d1b2566bSdrh }
238d1b2566bSdrh return isNeg? -v : v;
239d1b2566bSdrh }
240d1b2566bSdrh
241d1b2566bSdrh
242d1b2566bSdrh /*
243cac028baSdrh ** Check the filesystem object zPath. Determine what it is:
244cac028baSdrh **
2451346fdadSdrh ** PATH_DIR A single directory holding many files
2461346fdadSdrh ** PATH_TREE A directory hierarchy with files at the leaves
247cac028baSdrh ** PATH_DB An SQLite database
248cac028baSdrh ** PATH_NEXIST Does not exist
249cac028baSdrh ** PATH_OTHER Something else
2501346fdadSdrh **
2511346fdadSdrh ** PATH_DIR means all of the separate files are grouped together
2521346fdadSdrh ** into a single directory with names like 000000, 000001, 000002, and
2531346fdadSdrh ** so forth. PATH_TREE means there is a hierarchy of directories so
2541346fdadSdrh ** that no single directory has too many entries. The files have names
2551346fdadSdrh ** like 00/00/00, 00/00/01, 00/00/02 and so forth. The decision between
2561346fdadSdrh ** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
2571346fdadSdrh ** named "00" at the top-level.
258cac028baSdrh */
259cac028baSdrh #define PATH_DIR 1
2601346fdadSdrh #define PATH_TREE 2
2611346fdadSdrh #define PATH_DB 3
262cac028baSdrh #define PATH_NEXIST 0
263cac028baSdrh #define PATH_OTHER 99
pathType(const char * zPath)264cac028baSdrh static int pathType(const char *zPath){
265cac028baSdrh struct stat x;
266cac028baSdrh int rc;
267cac028baSdrh if( access(zPath,R_OK) ) return PATH_NEXIST;
268cac028baSdrh memset(&x, 0, sizeof(x));
269cac028baSdrh rc = stat(zPath, &x);
270cac028baSdrh if( rc<0 ) return PATH_OTHER;
2711346fdadSdrh if( S_ISDIR(x.st_mode) ){
2721346fdadSdrh char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
2731346fdadSdrh memset(&x, 0, sizeof(x));
2741346fdadSdrh rc = stat(zLayer1, &x);
2751346fdadSdrh sqlite3_free(zLayer1);
2761346fdadSdrh if( rc<0 ) return PATH_DIR;
2771346fdadSdrh if( S_ISDIR(x.st_mode) ) return PATH_TREE;
2781346fdadSdrh return PATH_DIR;
2791346fdadSdrh }
280cac028baSdrh if( (x.st_size%512)==0 ) return PATH_DB;
281cac028baSdrh return PATH_OTHER;
282cac028baSdrh }
283cac028baSdrh
284cac028baSdrh /*
285cae20d5cSdrh ** Return the size of a file in bytes. Or return -1 if the
286cae20d5cSdrh ** named object is not a regular file or does not exist.
287cae20d5cSdrh */
fileSize(const char * zPath)288cae20d5cSdrh static sqlite3_int64 fileSize(const char *zPath){
289cae20d5cSdrh struct stat x;
290cae20d5cSdrh int rc;
291cae20d5cSdrh memset(&x, 0, sizeof(x));
292cae20d5cSdrh rc = stat(zPath, &x);
293cae20d5cSdrh if( rc<0 ) return -1;
294cae20d5cSdrh if( !S_ISREG(x.st_mode) ) return -1;
295cae20d5cSdrh return x.st_size;
296cae20d5cSdrh }
297cae20d5cSdrh
298cae20d5cSdrh /*
299cac028baSdrh ** A Pseudo-random number generator with a fixed seed. Use this so
300cac028baSdrh ** that the same sequence of "random" numbers are generated on each
301cac028baSdrh ** run, for repeatability.
302cac028baSdrh */
randInt(void)303cac028baSdrh static unsigned int randInt(void){
304cac028baSdrh static unsigned int x = 0x333a13cd;
305cac028baSdrh static unsigned int y = 0xecb2adea;
306cac028baSdrh x = (x>>1) ^ ((1+~(x&1)) & 0xd0000001);
307cac028baSdrh y = y*1103515245 + 12345;
308cac028baSdrh return x^y;
309cac028baSdrh }
310cac028baSdrh
311cac028baSdrh /*
312cac028baSdrh ** Do database initialization.
313cac028baSdrh */
initMain(int argc,char ** argv)314cac028baSdrh static int initMain(int argc, char **argv){
315cac028baSdrh char *zDb;
316cac028baSdrh int i, rc;
317cac028baSdrh int nCount = 1000;
318cac028baSdrh int sz = 10000;
3199dc7ad1dSdrh int iVariance = 0;
320cac028baSdrh int pgsz = 4096;
321cac028baSdrh sqlite3 *db;
322cac028baSdrh char *zSql;
323cac028baSdrh char *zErrMsg = 0;
324cac028baSdrh
325cac028baSdrh assert( strcmp(argv[1],"init")==0 );
326cac028baSdrh assert( argc>=3 );
327cac028baSdrh zDb = argv[2];
328cac028baSdrh for(i=3; i<argc; i++){
329cac028baSdrh char *z = argv[i];
330cac028baSdrh if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
331cac028baSdrh if( z[1]=='-' ) z++;
332cac028baSdrh if( strcmp(z, "-count")==0 ){
333cac028baSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
334d1b2566bSdrh nCount = integerValue(argv[++i]);
335cac028baSdrh if( nCount<1 ) fatalError("the --count must be positive");
336cac028baSdrh continue;
337cac028baSdrh }
338cac028baSdrh if( strcmp(z, "-size")==0 ){
339cac028baSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
340d1b2566bSdrh sz = integerValue(argv[++i]);
341cac028baSdrh if( sz<1 ) fatalError("the --size must be positive");
342cac028baSdrh continue;
343cac028baSdrh }
3449dc7ad1dSdrh if( strcmp(z, "-variance")==0 ){
3459dc7ad1dSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
3469dc7ad1dSdrh iVariance = integerValue(argv[++i]);
3479dc7ad1dSdrh continue;
3489dc7ad1dSdrh }
349cac028baSdrh if( strcmp(z, "-pagesize")==0 ){
350cac028baSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
351d1b2566bSdrh pgsz = integerValue(argv[++i]);
352cac028baSdrh if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){
353cac028baSdrh fatalError("the --pagesize must be power of 2 between 512 and 65536");
354cac028baSdrh }
355cac028baSdrh continue;
356cac028baSdrh }
357cac028baSdrh fatalError("unknown option: \"%s\"", argv[i]);
358cac028baSdrh }
359cac028baSdrh rc = sqlite3_open(zDb, &db);
360cac028baSdrh if( rc ){
361cac028baSdrh fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
362cac028baSdrh }
363cac028baSdrh zSql = sqlite3_mprintf(
364cac028baSdrh "DROP TABLE IF EXISTS kv;\n"
365cac028baSdrh "PRAGMA page_size=%d;\n"
366cac028baSdrh "VACUUM;\n"
367cac028baSdrh "BEGIN;\n"
368cac028baSdrh "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n"
369cac028baSdrh "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)"
3709dc7ad1dSdrh " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n"
371cac028baSdrh "COMMIT;\n",
372809955b5Sdan pgsz, nCount, sz, iVariance+1
373cac028baSdrh );
374cac028baSdrh rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
375cac028baSdrh if( rc ) fatalError("database create failed: %s", zErrMsg);
376cac028baSdrh sqlite3_free(zSql);
377cac028baSdrh sqlite3_close(db);
378cac028baSdrh return 0;
379cac028baSdrh }
380cac028baSdrh
381cac028baSdrh /*
3829dc7ad1dSdrh ** Analyze an existing database file. Report its content.
3839dc7ad1dSdrh */
statMain(int argc,char ** argv)3849dc7ad1dSdrh static int statMain(int argc, char **argv){
3859dc7ad1dSdrh char *zDb;
3869dc7ad1dSdrh int i, rc;
3879dc7ad1dSdrh sqlite3 *db;
3889dc7ad1dSdrh char *zSql;
3899dc7ad1dSdrh sqlite3_stmt *pStmt;
390a4327e65Sdrh int doVacuum = 0;
3919dc7ad1dSdrh
3929dc7ad1dSdrh assert( strcmp(argv[1],"stat")==0 );
3939dc7ad1dSdrh assert( argc>=3 );
3949dc7ad1dSdrh zDb = argv[2];
3959dc7ad1dSdrh for(i=3; i<argc; i++){
3969dc7ad1dSdrh char *z = argv[i];
3979dc7ad1dSdrh if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
3989dc7ad1dSdrh if( z[1]=='-' ) z++;
399a4327e65Sdrh if( strcmp(z, "-vacuum")==0 ){
400a4327e65Sdrh doVacuum = 1;
401a4327e65Sdrh continue;
402a4327e65Sdrh }
4039dc7ad1dSdrh fatalError("unknown option: \"%s\"", argv[i]);
4049dc7ad1dSdrh }
4059dc7ad1dSdrh rc = sqlite3_open(zDb, &db);
4069dc7ad1dSdrh if( rc ){
4079dc7ad1dSdrh fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
4089dc7ad1dSdrh }
409a4327e65Sdrh if( doVacuum ){
410a4327e65Sdrh printf("Vacuuming...."); fflush(stdout);
411a4327e65Sdrh sqlite3_exec(db, "VACUUM", 0, 0, 0);
412a4327e65Sdrh printf(" done\n");
413a4327e65Sdrh }
4149dc7ad1dSdrh zSql = sqlite3_mprintf(
4159dc7ad1dSdrh "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"
4169dc7ad1dSdrh " FROM kv"
4179dc7ad1dSdrh );
4189dc7ad1dSdrh rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4199dc7ad1dSdrh if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4209dc7ad1dSdrh sqlite3_free(zSql);
4219dc7ad1dSdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
4229dc7ad1dSdrh printf("Number of entries: %8d\n", sqlite3_column_int(pStmt, 0));
4239dc7ad1dSdrh printf("Average value size: %8d\n", sqlite3_column_int(pStmt, 3));
4249dc7ad1dSdrh printf("Minimum value size: %8d\n", sqlite3_column_int(pStmt, 1));
4259dc7ad1dSdrh printf("Maximum value size: %8d\n", sqlite3_column_int(pStmt, 2));
4269dc7ad1dSdrh }else{
4279dc7ad1dSdrh printf("No rows\n");
4289dc7ad1dSdrh }
4299dc7ad1dSdrh sqlite3_finalize(pStmt);
4309dc7ad1dSdrh zSql = sqlite3_mprintf("PRAGMA page_size");
4319dc7ad1dSdrh rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4329dc7ad1dSdrh if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4339dc7ad1dSdrh sqlite3_free(zSql);
4349dc7ad1dSdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
4359dc7ad1dSdrh printf("Page-size: %8d\n", sqlite3_column_int(pStmt, 0));
4369dc7ad1dSdrh }
4379dc7ad1dSdrh sqlite3_finalize(pStmt);
4389dc7ad1dSdrh zSql = sqlite3_mprintf("PRAGMA page_count");
4399dc7ad1dSdrh rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4409dc7ad1dSdrh if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4419dc7ad1dSdrh sqlite3_free(zSql);
4429dc7ad1dSdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
4439dc7ad1dSdrh printf("Page-count: %8d\n", sqlite3_column_int(pStmt, 0));
4449dc7ad1dSdrh }
4459dc7ad1dSdrh sqlite3_finalize(pStmt);
446a4327e65Sdrh zSql = sqlite3_mprintf("PRAGMA freelist_count");
447a4327e65Sdrh rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
448a4327e65Sdrh if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
449a4327e65Sdrh sqlite3_free(zSql);
450a4327e65Sdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
451a4327e65Sdrh printf("Freelist-count: %8d\n", sqlite3_column_int(pStmt, 0));
452a4327e65Sdrh }
453a4327e65Sdrh sqlite3_finalize(pStmt);
454a4327e65Sdrh rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0);
455a4327e65Sdrh if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db));
456a4327e65Sdrh while( sqlite3_step(pStmt)==SQLITE_ROW ){
457a4327e65Sdrh printf("Integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
458a4327e65Sdrh }
459a4327e65Sdrh sqlite3_finalize(pStmt);
4609dc7ad1dSdrh sqlite3_close(db);
4619dc7ad1dSdrh return 0;
4629dc7ad1dSdrh }
4639dc7ad1dSdrh
4649dc7ad1dSdrh /*
465a2e71cedSdrh ** remember(V,PTR)
466a2e71cedSdrh **
467a2e71cedSdrh ** Return the integer value V. Also save the value of V in a
468a2e71cedSdrh ** C-language variable whose address is PTR.
469a2e71cedSdrh */
rememberFunc(sqlite3_context * pCtx,int argc,sqlite3_value ** argv)470a2e71cedSdrh static void rememberFunc(
471a2e71cedSdrh sqlite3_context *pCtx,
472a2e71cedSdrh int argc,
473a2e71cedSdrh sqlite3_value **argv
474a2e71cedSdrh ){
475a2e71cedSdrh sqlite3_int64 v;
476a2e71cedSdrh sqlite3_int64 ptr;
477a2e71cedSdrh assert( argc==2 );
478a2e71cedSdrh v = sqlite3_value_int64(argv[0]);
479a2e71cedSdrh ptr = sqlite3_value_int64(argv[1]);
480a2e71cedSdrh *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
481a2e71cedSdrh sqlite3_result_int64(pCtx, v);
482a2e71cedSdrh }
483a2e71cedSdrh
484a2e71cedSdrh /*
4851346fdadSdrh ** Make sure a directory named zDir exists.
4861346fdadSdrh */
kvtest_mkdir(const char * zDir)4871346fdadSdrh static void kvtest_mkdir(const char *zDir){
4881346fdadSdrh #if defined(_WIN32)
4891346fdadSdrh (void)mkdir(zDir);
4901346fdadSdrh #else
4911346fdadSdrh (void)mkdir(zDir, 0755);
4921346fdadSdrh #endif
4931346fdadSdrh }
4941346fdadSdrh
4951346fdadSdrh /*
496cac028baSdrh ** Export the kv table to individual files in the filesystem
497cac028baSdrh */
exportMain(int argc,char ** argv)498cac028baSdrh static int exportMain(int argc, char **argv){
499cac028baSdrh char *zDb;
500cac028baSdrh char *zDir;
501cac028baSdrh sqlite3 *db;
5021346fdadSdrh sqlite3_stmt *pStmt;
503cac028baSdrh int rc;
5041346fdadSdrh int ePathType;
5051346fdadSdrh int nFN;
5061346fdadSdrh char *zFN;
5071346fdadSdrh char *zTail;
5081346fdadSdrh size_t nWrote;
5091346fdadSdrh int i;
510cac028baSdrh
511cac028baSdrh assert( strcmp(argv[1],"export")==0 );
512cac028baSdrh assert( argc>=3 );
5131346fdadSdrh if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
514cac028baSdrh zDb = argv[2];
515cac028baSdrh zDir = argv[3];
5161346fdadSdrh kvtest_mkdir(zDir);
5171346fdadSdrh for(i=4; i<argc; i++){
5181346fdadSdrh const char *z = argv[i];
5191346fdadSdrh if( z[0]=='-' && z[1]=='-' ) z++;
5201346fdadSdrh if( strcmp(z,"-tree")==0 ){
5211346fdadSdrh zFN = sqlite3_mprintf("%s/00", zDir);
5221346fdadSdrh kvtest_mkdir(zFN);
5231346fdadSdrh sqlite3_free(zFN);
5241346fdadSdrh continue;
5251346fdadSdrh }
5261346fdadSdrh fatalError("unknown argument: \"%s\"\n", argv[i]);
5271346fdadSdrh }
5281346fdadSdrh ePathType = pathType(zDir);
5291346fdadSdrh if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
530cac028baSdrh fatalError("object \"%s\" is not a directory", zDir);
531cac028baSdrh }
532cac028baSdrh rc = sqlite3_open(zDb, &db);
533cac028baSdrh if( rc ){
534cac028baSdrh fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
535cac028baSdrh }
5361346fdadSdrh rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
5371346fdadSdrh if( rc ){
5381346fdadSdrh fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
5391346fdadSdrh }
5401346fdadSdrh nFN = (int)strlen(zDir);
5411346fdadSdrh zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
5421346fdadSdrh if( zFN==0 ){
5431346fdadSdrh fatalError("malloc failed\n");
5441346fdadSdrh }
5451346fdadSdrh zTail = zFN + nFN + 1;
5461346fdadSdrh while( sqlite3_step(pStmt)==SQLITE_ROW ){
5471346fdadSdrh int iKey = sqlite3_column_int(pStmt, 0);
548cf1393dfSdrh sqlite3_int64 nData = sqlite3_column_bytes(pStmt, 1);
5491346fdadSdrh const void *pData = sqlite3_column_blob(pStmt, 1);
5501346fdadSdrh FILE *out;
5511346fdadSdrh if( ePathType==PATH_DIR ){
5521346fdadSdrh sqlite3_snprintf(20, zTail, "%06d", iKey);
5531346fdadSdrh }else{
5541346fdadSdrh sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
5551346fdadSdrh kvtest_mkdir(zFN);
5561346fdadSdrh sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
5571346fdadSdrh kvtest_mkdir(zFN);
5581346fdadSdrh sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
5591346fdadSdrh iKey/10000, (iKey/100)%100, iKey%100);
5601346fdadSdrh }
5611346fdadSdrh out = fopen(zFN, "wb");
562ac8ba26eSmistachkin nWrote = fwrite(pData, 1, (size_t)nData, out);
5631346fdadSdrh fclose(out);
5641346fdadSdrh printf("\r%s ", zTail); fflush(stdout);
565928ba149Sdrh if( nWrote!=(size_t)nData ){
5661346fdadSdrh fatalError("Wrote only %d of %d bytes to %s\n",
5671346fdadSdrh (int)nWrote, nData, zFN);
5681346fdadSdrh }
5691346fdadSdrh }
5701346fdadSdrh sqlite3_finalize(pStmt);
571cac028baSdrh sqlite3_close(db);
5721346fdadSdrh sqlite3_free(zFN);
5736739c692Sdrh printf("\n");
574cac028baSdrh return 0;
575cac028baSdrh }
576cac028baSdrh
577cac028baSdrh /*
578cac028baSdrh ** Read the content of file zName into memory obtained from sqlite3_malloc64()
579cac028baSdrh ** and return a pointer to the buffer. The caller is responsible for freeing
580cac028baSdrh ** the memory.
581cac028baSdrh **
582cac028baSdrh ** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
583cac028baSdrh ** read.
584cac028baSdrh **
585cac028baSdrh ** For convenience, a nul-terminator byte is always appended to the data read
586cac028baSdrh ** from the file before the buffer is returned. This byte is not included in
587cac028baSdrh ** the final value of (*pnByte), if applicable.
588cac028baSdrh **
589cac028baSdrh ** NULL is returned if any error is encountered. The final value of *pnByte
590cac028baSdrh ** is undefined in this case.
591cac028baSdrh */
readFile(const char * zName,sqlite3_int64 * pnByte)592cf1393dfSdrh static unsigned char *readFile(const char *zName, sqlite3_int64 *pnByte){
593cae20d5cSdrh FILE *in; /* FILE from which to read content of zName */
594cae20d5cSdrh sqlite3_int64 nIn; /* Size of zName in bytes */
595cae20d5cSdrh size_t nRead; /* Number of bytes actually read */
596cae20d5cSdrh unsigned char *pBuf; /* Content read from disk */
597cae20d5cSdrh
598cae20d5cSdrh nIn = fileSize(zName);
599cae20d5cSdrh if( nIn<0 ) return 0;
600cae20d5cSdrh in = fopen(zName, "rb");
601cac028baSdrh if( in==0 ) return 0;
602cae20d5cSdrh pBuf = sqlite3_malloc64( nIn );
603cac028baSdrh if( pBuf==0 ) return 0;
604b67e1754Sdrh nRead = fread(pBuf, (size_t)nIn, 1, in);
605cac028baSdrh fclose(in);
606cac028baSdrh if( nRead!=1 ){
607cac028baSdrh sqlite3_free(pBuf);
608cac028baSdrh return 0;
609cac028baSdrh }
610cf1393dfSdrh if( pnByte ) *pnByte = nIn;
611cac028baSdrh return pBuf;
612cac028baSdrh }
613cac028baSdrh
614cac028baSdrh /*
615a2e71cedSdrh ** Overwrite a file with randomness. Do not change the size of the
616a2e71cedSdrh ** file.
617a2e71cedSdrh */
updateFile(const char * zName,sqlite3_int64 * pnByte,int doFsync)618cf1393dfSdrh static void updateFile(const char *zName, sqlite3_int64 *pnByte, int doFsync){
619a2e71cedSdrh FILE *out; /* FILE from which to read content of zName */
620a2e71cedSdrh sqlite3_int64 sz; /* Size of zName in bytes */
621a2e71cedSdrh size_t nWritten; /* Number of bytes actually read */
622a2e71cedSdrh unsigned char *pBuf; /* Content to store on disk */
623f1518907Sdrh const char *zMode = "wb"; /* Mode for fopen() */
624a2e71cedSdrh
625a2e71cedSdrh sz = fileSize(zName);
626a2e71cedSdrh if( sz<0 ){
627a2e71cedSdrh fatalError("No such file: \"%s\"", zName);
628a2e71cedSdrh }
629cf1393dfSdrh *pnByte = sz;
630a2e71cedSdrh if( sz==0 ) return;
631a2e71cedSdrh pBuf = sqlite3_malloc64( sz );
632a2e71cedSdrh if( pBuf==0 ){
633a2e71cedSdrh fatalError("Cannot allocate %lld bytes\n", sz);
634a2e71cedSdrh }
635a2e71cedSdrh sqlite3_randomness((int)sz, pBuf);
636f1518907Sdrh #if defined(_WIN32)
637f1518907Sdrh if( doFsync ) zMode = "wbc";
638f1518907Sdrh #endif
639f1518907Sdrh out = fopen(zName, zMode);
640a2e71cedSdrh if( out==0 ){
641a2e71cedSdrh fatalError("Cannot open \"%s\" for writing\n", zName);
642a2e71cedSdrh }
643a2e71cedSdrh nWritten = fwrite(pBuf, 1, (size_t)sz, out);
644f1518907Sdrh if( doFsync ){
645f1518907Sdrh #if defined(_WIN32)
646f1518907Sdrh fflush(out);
647f1518907Sdrh #else
648f1518907Sdrh fsync(fileno(out));
649f1518907Sdrh #endif
650f1518907Sdrh }
651a2e71cedSdrh fclose(out);
652a2e71cedSdrh if( nWritten!=(size_t)sz ){
653a2e71cedSdrh fatalError("Wrote only %d of %d bytes to \"%s\"\n",
654a2e71cedSdrh (int)nWritten, (int)sz, zName);
655a2e71cedSdrh }
656a2e71cedSdrh sqlite3_free(pBuf);
657a2e71cedSdrh }
658a2e71cedSdrh
659a2e71cedSdrh /*
660cac028baSdrh ** Return the current time in milliseconds since the beginning of
661cac028baSdrh ** the Julian epoch.
662cac028baSdrh */
timeOfDay(void)663cac028baSdrh static sqlite3_int64 timeOfDay(void){
664cac028baSdrh static sqlite3_vfs *clockVfs = 0;
665cac028baSdrh sqlite3_int64 t;
666cac028baSdrh if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
667cac028baSdrh if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
668cac028baSdrh clockVfs->xCurrentTimeInt64(clockVfs, &t);
669cac028baSdrh }else{
670cac028baSdrh double r;
671cac028baSdrh clockVfs->xCurrentTime(clockVfs, &r);
672cac028baSdrh t = (sqlite3_int64)(r*86400000.0);
673cac028baSdrh }
674cac028baSdrh return t;
675cac028baSdrh }
676cac028baSdrh
677befcd8adSdan #ifdef __linux__
678befcd8adSdan /*
679befcd8adSdan ** Attempt to display I/O stats on Linux using /proc/PID/io
680befcd8adSdan */
displayLinuxIoStats(FILE * out)681befcd8adSdan static void displayLinuxIoStats(FILE *out){
682befcd8adSdan FILE *in;
683befcd8adSdan char z[200];
684befcd8adSdan sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
685befcd8adSdan in = fopen(z, "rb");
686befcd8adSdan if( in==0 ) return;
687befcd8adSdan while( fgets(z, sizeof(z), in)!=0 ){
688befcd8adSdan static const struct {
689befcd8adSdan const char *zPattern;
690befcd8adSdan const char *zDesc;
691befcd8adSdan } aTrans[] = {
692befcd8adSdan { "rchar: ", "Bytes received by read():" },
693befcd8adSdan { "wchar: ", "Bytes sent to write():" },
694befcd8adSdan { "syscr: ", "Read() system calls:" },
695befcd8adSdan { "syscw: ", "Write() system calls:" },
696befcd8adSdan { "read_bytes: ", "Bytes read from storage:" },
697befcd8adSdan { "write_bytes: ", "Bytes written to storage:" },
698befcd8adSdan { "cancelled_write_bytes: ", "Cancelled write bytes:" },
699befcd8adSdan };
700befcd8adSdan int i;
701befcd8adSdan for(i=0; i<sizeof(aTrans)/sizeof(aTrans[0]); i++){
702befcd8adSdan int n = (int)strlen(aTrans[i].zPattern);
703befcd8adSdan if( strncmp(aTrans[i].zPattern, z, n)==0 ){
704befcd8adSdan fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);
705befcd8adSdan break;
706befcd8adSdan }
707befcd8adSdan }
708befcd8adSdan }
709befcd8adSdan fclose(in);
710befcd8adSdan }
711befcd8adSdan #endif
712befcd8adSdan
713befcd8adSdan /*
714befcd8adSdan ** Display memory stats.
715befcd8adSdan */
display_stats(sqlite3 * db,int bReset)716befcd8adSdan static int display_stats(
717befcd8adSdan sqlite3 *db, /* Database to query */
718befcd8adSdan int bReset /* True to reset SQLite stats */
719befcd8adSdan ){
720befcd8adSdan int iCur;
721befcd8adSdan int iHiwtr;
722befcd8adSdan FILE *out = stdout;
723befcd8adSdan
724befcd8adSdan fprintf(out, "\n");
725befcd8adSdan
726befcd8adSdan iHiwtr = iCur = -1;
727befcd8adSdan sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset);
728befcd8adSdan fprintf(out,
729befcd8adSdan "Memory Used: %d (max %d) bytes\n",
730befcd8adSdan iCur, iHiwtr);
731befcd8adSdan iHiwtr = iCur = -1;
732befcd8adSdan sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset);
733befcd8adSdan fprintf(out, "Number of Outstanding Allocations: %d (max %d)\n",
734befcd8adSdan iCur, iHiwtr);
735befcd8adSdan iHiwtr = iCur = -1;
736befcd8adSdan sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset);
737befcd8adSdan fprintf(out,
738befcd8adSdan "Number of Pcache Pages Used: %d (max %d) pages\n",
739befcd8adSdan iCur, iHiwtr);
740befcd8adSdan iHiwtr = iCur = -1;
741befcd8adSdan sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset);
742befcd8adSdan fprintf(out,
743befcd8adSdan "Number of Pcache Overflow Bytes: %d (max %d) bytes\n",
744befcd8adSdan iCur, iHiwtr);
745befcd8adSdan iHiwtr = iCur = -1;
746befcd8adSdan sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset);
747befcd8adSdan fprintf(out, "Largest Allocation: %d bytes\n",
748befcd8adSdan iHiwtr);
749befcd8adSdan iHiwtr = iCur = -1;
750befcd8adSdan sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset);
751befcd8adSdan fprintf(out, "Largest Pcache Allocation: %d bytes\n",
752befcd8adSdan iHiwtr);
753befcd8adSdan
754befcd8adSdan iHiwtr = iCur = -1;
755befcd8adSdan sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);
756befcd8adSdan fprintf(out, "Pager Heap Usage: %d bytes\n",
757befcd8adSdan iCur);
758befcd8adSdan iHiwtr = iCur = -1;
759befcd8adSdan sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);
760befcd8adSdan fprintf(out, "Page cache hits: %d\n", iCur);
761befcd8adSdan iHiwtr = iCur = -1;
762befcd8adSdan sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);
763befcd8adSdan fprintf(out, "Page cache misses: %d\n", iCur);
764befcd8adSdan iHiwtr = iCur = -1;
765befcd8adSdan sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);
766befcd8adSdan fprintf(out, "Page cache writes: %d\n", iCur);
767befcd8adSdan iHiwtr = iCur = -1;
768befcd8adSdan
769befcd8adSdan #ifdef __linux__
770befcd8adSdan displayLinuxIoStats(out);
771befcd8adSdan #endif
772befcd8adSdan
773befcd8adSdan return 0;
774befcd8adSdan }
775befcd8adSdan
776cac028baSdrh /* Blob access order */
777cac028baSdrh #define ORDER_ASC 1
778cac028baSdrh #define ORDER_DESC 2
779cac028baSdrh #define ORDER_RANDOM 3
780cac028baSdrh
781d1b2566bSdrh
782cac028baSdrh /*
783cac028baSdrh ** Run a performance test
784cac028baSdrh */
runMain(int argc,char ** argv)785cac028baSdrh static int runMain(int argc, char **argv){
786cac028baSdrh int eType; /* Is zDb a database or a directory? */
787cac028baSdrh char *zDb; /* Database or directory name */
788cac028baSdrh int i; /* Loop counter */
789cac028baSdrh int rc; /* Return code from SQLite calls */
790cac028baSdrh int nCount = 1000; /* Number of blob fetch operations */
791cac028baSdrh int nExtra = 0; /* Extra cycles */
792cac028baSdrh int iKey = 1; /* Next blob key */
7932b15f6e1Sdrh int iMax = 0; /* Largest allowed key */
79461c565f6Sdrh int iPagesize = 0; /* Database page size */
79561c565f6Sdrh int iCache = 1000; /* Database cache size in kibibytes */
796cac028baSdrh int bBlobApi = 0; /* Use the incremental blob I/O API */
797befcd8adSdan int bStats = 0; /* Print stats before exiting */
798cac028baSdrh int eOrder = ORDER_ASC; /* Access order */
799a2e71cedSdrh int isUpdateTest = 0; /* Do in-place updates rather than reads */
800a2e71cedSdrh int doIntegrityCk = 0; /* Run PRAGMA integrity_check after the test */
801a2e71cedSdrh int noSync = 0; /* Disable synchronous mode */
802f1518907Sdrh int doFsync = 0; /* Update disk files synchronously */
80316d62901Sdrh int doMultiTrans = 0; /* Each operation in its own transaction */
80416d62901Sdrh int noCheckpoint = 0; /* Omit the checkpoint in WAL mode */
805cac028baSdrh sqlite3 *db = 0; /* Database connection */
806cac028baSdrh sqlite3_stmt *pStmt = 0; /* Prepared statement for SQL access */
807cac028baSdrh sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */
808cac028baSdrh sqlite3_int64 tmStart; /* Start time */
809cac028baSdrh sqlite3_int64 tmElapsed; /* Elapsed time */
810bace3240Sdrh int mmapSize = 0; /* --mmap N argument */
811cf1393dfSdrh sqlite3_int64 nData = 0; /* Bytes of data */
812cac028baSdrh sqlite3_int64 nTotal = 0; /* Total data read */
813eacb4412Sdrh unsigned char *pData = 0; /* Content of the blob */
814cf1393dfSdrh sqlite3_int64 nAlloc = 0; /* Space allocated for pData[] */
8152b15f6e1Sdrh const char *zJMode = 0; /* Journal mode */
816cac028baSdrh
817cac028baSdrh
818cac028baSdrh assert( strcmp(argv[1],"run")==0 );
819cac028baSdrh assert( argc>=3 );
820cac028baSdrh zDb = argv[2];
821cac028baSdrh eType = pathType(zDb);
822cac028baSdrh if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
823cac028baSdrh if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
824cac028baSdrh for(i=3; i<argc; i++){
825cac028baSdrh char *z = argv[i];
826cac028baSdrh if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
827cac028baSdrh if( z[1]=='-' ) z++;
82816d62901Sdrh if( strcmp(z, "-asc")==0 ){
82916d62901Sdrh eOrder = ORDER_ASC;
83016d62901Sdrh continue;
83116d62901Sdrh }
83216d62901Sdrh if( strcmp(z, "-blob-api")==0 ){
83316d62901Sdrh bBlobApi = 1;
83416d62901Sdrh continue;
83516d62901Sdrh }
83616d62901Sdrh if( strcmp(z, "-cache-size")==0 ){
83716d62901Sdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
83816d62901Sdrh iCache = integerValue(argv[++i]);
83916d62901Sdrh continue;
84016d62901Sdrh }
841cac028baSdrh if( strcmp(z, "-count")==0 ){
842cac028baSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
843d1b2566bSdrh nCount = integerValue(argv[++i]);
844cac028baSdrh if( nCount<1 ) fatalError("the --count must be positive");
845cac028baSdrh continue;
846cac028baSdrh }
84716d62901Sdrh if( strcmp(z, "-desc")==0 ){
84816d62901Sdrh eOrder = ORDER_DESC;
84916d62901Sdrh continue;
85016d62901Sdrh }
85116d62901Sdrh if( strcmp(z, "-fsync")==0 ){
85216d62901Sdrh doFsync = 1;
85316d62901Sdrh continue;
85416d62901Sdrh }
85516d62901Sdrh if( strcmp(z, "-integrity-check")==0 ){
85616d62901Sdrh doIntegrityCk = 1;
85716d62901Sdrh continue;
85816d62901Sdrh }
85916d62901Sdrh if( strcmp(z, "-jmode")==0 ){
86016d62901Sdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
86116d62901Sdrh zJMode = argv[++i];
86216d62901Sdrh continue;
86316d62901Sdrh }
864bace3240Sdrh if( strcmp(z, "-mmap")==0 ){
865bace3240Sdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
866d1b2566bSdrh mmapSize = integerValue(argv[++i]);
867bace3240Sdrh if( nCount<0 ) fatalError("the --mmap must be non-negative");
868bace3240Sdrh continue;
869bace3240Sdrh }
870cac028baSdrh if( strcmp(z, "-max-id")==0 ){
871cac028baSdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
872d1b2566bSdrh iMax = integerValue(argv[++i]);
873cac028baSdrh continue;
874cac028baSdrh }
87516d62901Sdrh if( strcmp(z, "-multitrans")==0 ){
87616d62901Sdrh doMultiTrans = 1;
877cac028baSdrh continue;
878cac028baSdrh }
87916d62901Sdrh if( strcmp(z, "-nocheckpoint")==0 ){
88016d62901Sdrh noCheckpoint = 1;
88161c565f6Sdrh continue;
88261c565f6Sdrh }
88316d62901Sdrh if( strcmp(z, "-nosync")==0 ){
88416d62901Sdrh noSync = 1;
8852b15f6e1Sdrh continue;
8862b15f6e1Sdrh }
887cac028baSdrh if( strcmp(z, "-random")==0 ){
888cac028baSdrh eOrder = ORDER_RANDOM;
889cac028baSdrh continue;
890cac028baSdrh }
89116d62901Sdrh if( strcmp(z, "-start")==0 ){
89216d62901Sdrh if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
89316d62901Sdrh iKey = integerValue(argv[++i]);
89416d62901Sdrh if( iKey<1 ) fatalError("the --start must be positive");
895cac028baSdrh continue;
896cac028baSdrh }
897befcd8adSdan if( strcmp(z, "-stats")==0 ){
898befcd8adSdan bStats = 1;
899befcd8adSdan continue;
900befcd8adSdan }
901a2e71cedSdrh if( strcmp(z, "-update")==0 ){
902a2e71cedSdrh isUpdateTest = 1;
903a2e71cedSdrh continue;
904a2e71cedSdrh }
905cac028baSdrh fatalError("unknown option: \"%s\"", argv[i]);
906cac028baSdrh }
907a2e71cedSdrh if( eType==PATH_DB ){
908a2e71cedSdrh /* Recover any prior crashes prior to starting the timer */
909a2e71cedSdrh sqlite3_open(zDb, &db);
910*067b92baSdrh sqlite3_exec(db, "SELECT rowid FROM sqlite_schema LIMIT 1", 0, 0, 0);
911a2e71cedSdrh sqlite3_close(db);
9125930f663Sdrh db = 0;
913a2e71cedSdrh }
914cac028baSdrh tmStart = timeOfDay();
915cac028baSdrh if( eType==PATH_DB ){
91661c565f6Sdrh char *zSql;
917cac028baSdrh rc = sqlite3_open(zDb, &db);
918cac028baSdrh if( rc ){
919cac028baSdrh fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
920cac028baSdrh }
921bace3240Sdrh zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize);
922bace3240Sdrh sqlite3_exec(db, zSql, 0, 0, 0);
923a2e71cedSdrh sqlite3_free(zSql);
92461c565f6Sdrh zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache);
92561c565f6Sdrh sqlite3_exec(db, zSql, 0, 0, 0);
92661c565f6Sdrh sqlite3_free(zSql);
927a2e71cedSdrh if( noSync ){
928a2e71cedSdrh sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0);
929a2e71cedSdrh }
93061c565f6Sdrh pStmt = 0;
93161c565f6Sdrh sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0);
93261c565f6Sdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
93361c565f6Sdrh iPagesize = sqlite3_column_int(pStmt, 0);
93461c565f6Sdrh }
93561c565f6Sdrh sqlite3_finalize(pStmt);
93661c565f6Sdrh sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0);
93761c565f6Sdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
93861c565f6Sdrh iCache = sqlite3_column_int(pStmt, 0);
93961c565f6Sdrh }else{
94061c565f6Sdrh iCache = 0;
94161c565f6Sdrh }
94261c565f6Sdrh sqlite3_finalize(pStmt);
94361c565f6Sdrh pStmt = 0;
9442b15f6e1Sdrh if( zJMode ){
9452b15f6e1Sdrh zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);
9462b15f6e1Sdrh sqlite3_exec(db, zSql, 0, 0, 0);
9472b15f6e1Sdrh sqlite3_free(zSql);
94816d62901Sdrh if( noCheckpoint ){
94916d62901Sdrh sqlite3_exec(db, "PRAGMA wal_autocheckpoint=0", 0, 0, 0);
95016d62901Sdrh }
9512b15f6e1Sdrh }
9522b15f6e1Sdrh sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);
9532b15f6e1Sdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
9542b15f6e1Sdrh zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
9552b15f6e1Sdrh }else{
9562b15f6e1Sdrh zJMode = "???";
9572b15f6e1Sdrh }
9582b15f6e1Sdrh sqlite3_finalize(pStmt);
9592b15f6e1Sdrh if( iMax<=0 ){
9602b15f6e1Sdrh sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);
9612b15f6e1Sdrh if( sqlite3_step(pStmt)==SQLITE_ROW ){
9622b15f6e1Sdrh iMax = sqlite3_column_int(pStmt, 0);
9632b15f6e1Sdrh }
9642b15f6e1Sdrh sqlite3_finalize(pStmt);
9652b15f6e1Sdrh }
9662b15f6e1Sdrh pStmt = 0;
96716d62901Sdrh if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);
968cac028baSdrh }
9692b15f6e1Sdrh if( iMax<=0 ) iMax = 1000;
970cac028baSdrh for(i=0; i<nCount; i++){
9711346fdadSdrh if( eType==PATH_DIR || eType==PATH_TREE ){
9721346fdadSdrh /* CASE 1: Reading or writing blobs out of separate files */
973cac028baSdrh char *zKey;
9741346fdadSdrh if( eType==PATH_DIR ){
975cac028baSdrh zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
9761346fdadSdrh }else{
9771346fdadSdrh zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
9781346fdadSdrh (iKey/100)%100, iKey%100);
9791346fdadSdrh }
980cac028baSdrh nData = 0;
981a2e71cedSdrh if( isUpdateTest ){
982f1518907Sdrh updateFile(zKey, &nData, doFsync);
983a2e71cedSdrh }else{
984cac028baSdrh pData = readFile(zKey, &nData);
985cac028baSdrh sqlite3_free(pData);
986a2e71cedSdrh }
987a2e71cedSdrh sqlite3_free(zKey);
988cac028baSdrh }else if( bBlobApi ){
989cac028baSdrh /* CASE 2: Reading from database using the incremental BLOB I/O API */
990cac028baSdrh if( pBlob==0 ){
991a2e71cedSdrh rc = sqlite3_blob_open(db, "main", "kv", "v", iKey,
992a2e71cedSdrh isUpdateTest, &pBlob);
993cac028baSdrh if( rc ){
994cac028baSdrh fatalError("could not open sqlite3_blob handle: %s",
995cac028baSdrh sqlite3_errmsg(db));
996cac028baSdrh }
997cac028baSdrh }else{
998cac028baSdrh rc = sqlite3_blob_reopen(pBlob, iKey);
999cac028baSdrh }
1000cac028baSdrh if( rc==SQLITE_OK ){
1001cac028baSdrh nData = sqlite3_blob_bytes(pBlob);
1002184d902dSdrh if( nAlloc<nData+1 ){
1003184d902dSdrh nAlloc = nData+100;
1004cf1393dfSdrh pData = sqlite3_realloc64(pData, nAlloc);
1005184d902dSdrh }
1006cac028baSdrh if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
1007a2e71cedSdrh if( isUpdateTest ){
1008a2e71cedSdrh sqlite3_randomness((int)nData, pData);
1009cf1393dfSdrh rc = sqlite3_blob_write(pBlob, pData, (int)nData, 0);
1010a2e71cedSdrh if( rc!=SQLITE_OK ){
1011a2e71cedSdrh fatalError("could not write the blob at %d: %s", iKey,
1012a2e71cedSdrh sqlite3_errmsg(db));
1013a2e71cedSdrh }
1014a2e71cedSdrh }else{
1015cf1393dfSdrh rc = sqlite3_blob_read(pBlob, pData, (int)nData, 0);
1016cac028baSdrh if( rc!=SQLITE_OK ){
1017cac028baSdrh fatalError("could not read the blob at %d: %s", iKey,
1018cac028baSdrh sqlite3_errmsg(db));
1019cac028baSdrh }
1020cac028baSdrh }
1021a2e71cedSdrh }
1022cac028baSdrh }else{
1023cac028baSdrh /* CASE 3: Reading from database using SQL */
1024cac028baSdrh if( pStmt==0 ){
1025a2e71cedSdrh if( isUpdateTest ){
1026a2e71cedSdrh sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,
1027a2e71cedSdrh rememberFunc, 0, 0);
1028a2e71cedSdrh
1029a2e71cedSdrh rc = sqlite3_prepare_v2(db,
1030a2e71cedSdrh "UPDATE kv SET v=randomblob(remember(length(v),?2))"
1031a2e71cedSdrh " WHERE k=?1", -1, &pStmt, 0);
1032a2e71cedSdrh sqlite3_bind_int64(pStmt, 2, SQLITE_PTR_TO_INT(&nData));
1033a2e71cedSdrh }else{
1034cac028baSdrh rc = sqlite3_prepare_v2(db,
1035cac028baSdrh "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0);
1036a2e71cedSdrh }
1037cac028baSdrh if( rc ){
1038cac028baSdrh fatalError("cannot prepare query: %s", sqlite3_errmsg(db));
1039cac028baSdrh }
1040cac028baSdrh }else{
1041cac028baSdrh sqlite3_reset(pStmt);
1042cac028baSdrh }
1043cac028baSdrh sqlite3_bind_int(pStmt, 1, iKey);
1044a2e71cedSdrh nData = 0;
1045cac028baSdrh rc = sqlite3_step(pStmt);
1046cac028baSdrh if( rc==SQLITE_ROW ){
1047cac028baSdrh nData = sqlite3_column_bytes(pStmt, 0);
1048cac028baSdrh pData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
1049cac028baSdrh }
1050cac028baSdrh }
1051cac028baSdrh if( eOrder==ORDER_ASC ){
1052cac028baSdrh iKey++;
1053cac028baSdrh if( iKey>iMax ) iKey = 1;
1054cac028baSdrh }else if( eOrder==ORDER_DESC ){
1055cac028baSdrh iKey--;
1056cac028baSdrh if( iKey<=0 ) iKey = iMax;
1057cac028baSdrh }else{
1058cac028baSdrh iKey = (randInt()%iMax)+1;
1059cac028baSdrh }
1060cac028baSdrh nTotal += nData;
1061cac028baSdrh if( nData==0 ){ nCount++; nExtra++; }
1062cac028baSdrh }
1063184d902dSdrh if( nAlloc ) sqlite3_free(pData);
1064cac028baSdrh if( pStmt ) sqlite3_finalize(pStmt);
1065cac028baSdrh if( pBlob ) sqlite3_blob_close(pBlob);
1066befcd8adSdan if( bStats ){
1067befcd8adSdan display_stats(db, 0);
1068befcd8adSdan }
1069a2e71cedSdrh if( db ){
107016d62901Sdrh if( !doMultiTrans ) sqlite3_exec(db, "COMMIT", 0, 0, 0);
107116d62901Sdrh if( !noCheckpoint ){
1072a2e71cedSdrh sqlite3_close(db);
107316d62901Sdrh db = 0;
107416d62901Sdrh }
1075a2e71cedSdrh }
1076cac028baSdrh tmElapsed = timeOfDay() - tmStart;
107716d62901Sdrh if( db && noCheckpoint ){
107816d62901Sdrh sqlite3_close(db);
107916d62901Sdrh db = 0;
108016d62901Sdrh }
1081cac028baSdrh if( nExtra ){
1082cac028baSdrh printf("%d cycles due to %d misses\n", nCount, nExtra);
1083cac028baSdrh }
108461c565f6Sdrh if( eType==PATH_DB ){
108561c565f6Sdrh printf("SQLite version: %s\n", sqlite3_libversion());
1086a2e71cedSdrh if( doIntegrityCk ){
1087a2e71cedSdrh sqlite3_open(zDb, &db);
1088a2e71cedSdrh sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);
1089a2e71cedSdrh while( sqlite3_step(pStmt)==SQLITE_ROW ){
1090a2e71cedSdrh printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
1091a2e71cedSdrh }
1092a2e71cedSdrh sqlite3_finalize(pStmt);
1093a2e71cedSdrh sqlite3_close(db);
10945930f663Sdrh db = 0;
1095a2e71cedSdrh }
109661c565f6Sdrh }
10976739c692Sdrh printf("--count %d --max-id %d", nCount-nExtra, iMax);
109861c565f6Sdrh switch( eOrder ){
109961c565f6Sdrh case ORDER_RANDOM: printf(" --random\n"); break;
110061c565f6Sdrh case ORDER_DESC: printf(" --desc\n"); break;
110161c565f6Sdrh default: printf(" --asc\n"); break;
110261c565f6Sdrh }
11032b15f6e1Sdrh if( eType==PATH_DB ){
11042b15f6e1Sdrh printf("--cache-size %d --jmode %s\n", iCache, zJMode);
11052b15f6e1Sdrh printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : "");
1106a2e71cedSdrh if( noSync ) printf("--nosync\n");
11072b15f6e1Sdrh }
110861c565f6Sdrh if( iPagesize ) printf("Database page size: %d\n", iPagesize);
1109cac028baSdrh printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
1110a2e71cedSdrh if( isUpdateTest ){
1111a2e71cedSdrh printf("Microseconds per BLOB write: %.3f\n", tmElapsed*1000.0/nCount);
1112a2e71cedSdrh printf("Content write rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
1113a2e71cedSdrh }else{
1114cac028baSdrh printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
1115cac028baSdrh printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
1116a2e71cedSdrh }
1117cac028baSdrh return 0;
1118cac028baSdrh }
1119cac028baSdrh
1120cac028baSdrh
main(int argc,char ** argv)1121cac028baSdrh int main(int argc, char **argv){
1122cac028baSdrh if( argc<3 ) showHelp();
1123cac028baSdrh if( strcmp(argv[1],"init")==0 ){
1124cac028baSdrh return initMain(argc, argv);
1125cac028baSdrh }
1126cac028baSdrh if( strcmp(argv[1],"export")==0 ){
1127cac028baSdrh return exportMain(argc, argv);
1128cac028baSdrh }
1129cac028baSdrh if( strcmp(argv[1],"run")==0 ){
1130cac028baSdrh return runMain(argc, argv);
1131cac028baSdrh }
11329dc7ad1dSdrh if( strcmp(argv[1],"stat")==0 ){
11339dc7ad1dSdrh return statMain(argc, argv);
11349dc7ad1dSdrh }
1135cac028baSdrh showHelp();
1136cac028baSdrh return 0;
1137cac028baSdrh }
1138