xref: /sqlite-3.40.0/test/kvtest.c (revision 067b92ba)
1cac028baSdrh /*
2cac028baSdrh ** 2016-12-28
3cac028baSdrh **
4cac028baSdrh ** The author disclaims copyright to this source code.  In place of
5cac028baSdrh ** a legal notice, here is a blessing:
6cac028baSdrh **
7cac028baSdrh **    May you do good and not evil.
8cac028baSdrh **    May you find forgiveness for yourself and forgive others.
9cac028baSdrh **    May you share freely, never taking more than you give.
10cac028baSdrh **
11cac028baSdrh *************************************************************************
12cac028baSdrh **
13cac028baSdrh ** This file implements "key-value" performance test for SQLite.  The
14cac028baSdrh ** purpose is to compare the speed of SQLite for accessing large BLOBs
15cac028baSdrh ** versus reading those same BLOB values out of individual files in the
16cac028baSdrh ** filesystem.
17cac028baSdrh **
182f917e05Sdrh ** Run "kvtest" with no arguments for on-line help, or see comments below.
192f917e05Sdrh **
202f917e05Sdrh ** HOW TO COMPILE:
212f917e05Sdrh **
222f917e05Sdrh ** (1) Gather this source file and a recent SQLite3 amalgamation with its
232f917e05Sdrh **     header into the working directory.  You should have:
242f917e05Sdrh **
252f917e05Sdrh **          kvtest.c       >--- this file
262f917e05Sdrh **          sqlite3.c      \___ SQLite
272f917e05Sdrh **          sqlite3.h      /    amlagamation & header
282f917e05Sdrh **
292f917e05Sdrh ** (2) Run you compiler against the two C source code files.
302f917e05Sdrh **
312f917e05Sdrh **    (a) On linux or mac:
322f917e05Sdrh **
332f917e05Sdrh **        OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION"
342f917e05Sdrh **        gcc -Os -I. $OPTS kvtest.c sqlite3.c -o kvtest
352f917e05Sdrh **
362f917e05Sdrh **             The $OPTS options can be omitted.  The $OPTS merely omit
372f917e05Sdrh **             the need to link against -ldl and -lpthread, or whatever
382f917e05Sdrh **             the equivalent libraries are called on your system.
392f917e05Sdrh **
402f917e05Sdrh **    (b) Windows with MSVC:
412f917e05Sdrh **
422f917e05Sdrh **        cl -I. kvtest.c sqlite3.c
432f917e05Sdrh **
442f917e05Sdrh ** USAGE:
452f917e05Sdrh **
462f917e05Sdrh ** (1) Create a test database by running "kvtest init" with appropriate
472f917e05Sdrh **     options.  See the help message for available options.
482f917e05Sdrh **
492f917e05Sdrh ** (2) Construct the corresponding pile-of-files database on disk using
502f917e05Sdrh **     the "kvtest export" command.
512f917e05Sdrh **
522f917e05Sdrh ** (3) Run tests using "kvtest run" against either the SQLite database or
532f917e05Sdrh **     the pile-of-files database and with appropriate options.
542f917e05Sdrh **
552f917e05Sdrh ** For example:
562f917e05Sdrh **
572f917e05Sdrh **       ./kvtest init x1.db --count 100000 --size 10000
582f917e05Sdrh **       mkdir x1
592f917e05Sdrh **       ./kvtest export x1.db x1
602f917e05Sdrh **       ./kvtest run x1.db --count 10000 --max-id 1000000
612f917e05Sdrh **       ./kvtest run x1 --count 10000 --max-id 1000000
62cac028baSdrh */
63cac028baSdrh static const char zHelp[] =
64dfd0de82Sdan "Usage: kvtest COMMAND ARGS...\n"
65cac028baSdrh "\n"
66dfd0de82Sdan "   kvtest init DBFILE --count N --size M --pagesize X\n"
67cac028baSdrh "\n"
68cac028baSdrh "        Generate a new test database file named DBFILE containing N\n"
69cac028baSdrh "        BLOBs each of size M bytes.  The page size of the new database\n"
709dc7ad1dSdrh "        file will be X.  Additional options:\n"
719dc7ad1dSdrh "\n"
729dc7ad1dSdrh "           --variance V           Randomly vary M by plus or minus V\n"
73cac028baSdrh "\n"
741346fdadSdrh "   kvtest export DBFILE DIRECTORY [--tree]\n"
75cac028baSdrh "\n"
76cac028baSdrh "        Export all the blobs in the kv table of DBFILE into separate\n"
771346fdadSdrh "        files in DIRECTORY.  DIRECTORY is created if it does not previously\n"
781346fdadSdrh "        exist.  If the --tree option is used, then the blobs are written\n"
791346fdadSdrh "        into a hierarchy of directories, using names like 00/00/00,\n"
801346fdadSdrh "        00/00/01, 00/00/02, and so forth.  Without the --tree option, all\n"
811346fdadSdrh "        files are in the top-level directory with names like 000000, 000001,\n"
821346fdadSdrh "        000002, and so forth.\n"
83cac028baSdrh "\n"
84a4327e65Sdrh "   kvtest stat DBFILE [options]\n"
859dc7ad1dSdrh "\n"
86a4327e65Sdrh "        Display summary information about DBFILE.  Options:\n"
87a4327e65Sdrh "\n"
88a4327e65Sdrh "           --vacuum               Run VACUUM on the database file\n"
899dc7ad1dSdrh "\n"
90dfd0de82Sdan "   kvtest run DBFILE [options]\n"
91cac028baSdrh "\n"
92cac028baSdrh "        Run a performance test.  DBFILE can be either the name of a\n"
93cac028baSdrh "        database or a directory containing sample files.  Options:\n"
94cac028baSdrh "\n"
9561c565f6Sdrh "           --asc                  Read blobs in ascending order\n"
96cac028baSdrh "           --blob-api             Use the BLOB API\n"
9761c565f6Sdrh "           --cache-size N         Database cache size\n"
9861c565f6Sdrh "           --count N              Read N blobs\n"
99cac028baSdrh "           --desc                 Read blobs in descending order\n"
100f1518907Sdrh "           --fsync                Synchronous file writes\n"
101f1518907Sdrh "           --integrity-check      Run \"PRAGMA integrity_check\" after test\n"
102cac028baSdrh "           --max-id N             Maximum blob key to use\n"
103d1b2566bSdrh "           --mmap N               Mmap as much as N bytes of DBFILE\n"
10416d62901Sdrh "           --multitrans           Each read or write in its own transaction\n"
10516d62901Sdrh "           --nocheckpoint         Omit the checkpoint on WAL mode writes\n"
106f1518907Sdrh "           --nosync               Set \"PRAGMA synchronous=OFF\"\n"
1072b15f6e1Sdrh "           --jmode MODE           Set MODE journal mode prior to starting\n"
10861c565f6Sdrh "           --random               Read blobs in a random order\n"
109cac028baSdrh "           --start N              Start reading with this blob key\n"
110befcd8adSdan "           --stats                Output operating stats before exiting\n"
11116d62901Sdrh "           --update               Do an overwrite test\n"
112cac028baSdrh ;
113cac028baSdrh 
114cac028baSdrh /* Reference resources used */
115cac028baSdrh #include <stdio.h>
116cac028baSdrh #include <stdlib.h>
117cac028baSdrh #include <sys/types.h>
118cac028baSdrh #include <sys/stat.h>
119cac028baSdrh #include <assert.h>
120cac028baSdrh #include <string.h>
121cac028baSdrh #include "sqlite3.h"
122cac028baSdrh 
123d7b9b97cSdrh #ifndef _WIN32
124d7b9b97cSdrh # include <unistd.h>
125d7b9b97cSdrh #else
126d7b9b97cSdrh   /* Provide Windows equivalent for the needed parts of unistd.h */
1271346fdadSdrh # include <direct.h>
128d7b9b97cSdrh # include <io.h>
129d7b9b97cSdrh # define R_OK 2
130cae20d5cSdrh # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
131d7b9b97cSdrh # define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
132d7b9b97cSdrh # define access _access
133d7b9b97cSdrh #endif
134d7b9b97cSdrh 
135ac8ba26eSmistachkin #if !defined(_MSC_VER)
136a2e71cedSdrh # include <stdint.h>
137ac8ba26eSmistachkin #endif
138a2e71cedSdrh 
139a2e71cedSdrh /*
140a2e71cedSdrh ** The following macros are used to cast pointers to integers and
141a2e71cedSdrh ** integers to pointers.  The way you do this varies from one compiler
142a2e71cedSdrh ** to the next, so we have developed the following set of #if statements
143a2e71cedSdrh ** to generate appropriate macros for a wide range of compilers.
144a2e71cedSdrh **
145a2e71cedSdrh ** The correct "ANSI" way to do this is to use the intptr_t type.
146a2e71cedSdrh ** Unfortunately, that typedef is not available on all compilers, or
147a2e71cedSdrh ** if it is available, it requires an #include of specific headers
148a2e71cedSdrh ** that vary from one machine to the next.
149a2e71cedSdrh **
150a2e71cedSdrh ** Ticket #3860:  The llvm-gcc-4.2 compiler from Apple chokes on
151a2e71cedSdrh ** the ((void*)&((char*)0)[X]) construct.  But MSVC chokes on ((void*)(X)).
152a2e71cedSdrh ** So we have to define the macros in different ways depending on the
153a2e71cedSdrh ** compiler.
154a2e71cedSdrh */
155a2e71cedSdrh #if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
156a2e71cedSdrh # define SQLITE_INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
157a2e71cedSdrh # define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))
1585930f663Sdrh #else
159a2e71cedSdrh # define SQLITE_INT_TO_PTR(X)  ((void*)(intptr_t)(X))
160a2e71cedSdrh # define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(intptr_t)(X))
161a2e71cedSdrh #endif
162d7b9b97cSdrh 
163cac028baSdrh /*
164cac028baSdrh ** Show thqe help text and quit.
165cac028baSdrh */
showHelp(void)166cac028baSdrh static void showHelp(void){
167cac028baSdrh   fprintf(stdout, "%s", zHelp);
168cac028baSdrh   exit(1);
169cac028baSdrh }
170cac028baSdrh 
171cac028baSdrh /*
172cac028baSdrh ** Show an error message an quit.
173cac028baSdrh */
fatalError(const char * zFormat,...)174cac028baSdrh static void fatalError(const char *zFormat, ...){
175cac028baSdrh   va_list ap;
176cac028baSdrh   fprintf(stdout, "ERROR: ");
177cac028baSdrh   va_start(ap, zFormat);
178cac028baSdrh   vfprintf(stdout, zFormat, ap);
179cac028baSdrh   va_end(ap);
180cac028baSdrh   fprintf(stdout, "\n");
181cac028baSdrh   exit(1);
182cac028baSdrh }
183cac028baSdrh 
184cac028baSdrh /*
185d1b2566bSdrh ** Return the value of a hexadecimal digit.  Return -1 if the input
186d1b2566bSdrh ** is not a hex digit.
187d1b2566bSdrh */
hexDigitValue(char c)188d1b2566bSdrh static int hexDigitValue(char c){
189d1b2566bSdrh   if( c>='0' && c<='9' ) return c - '0';
190d1b2566bSdrh   if( c>='a' && c<='f' ) return c - 'a' + 10;
191d1b2566bSdrh   if( c>='A' && c<='F' ) return c - 'A' + 10;
192d1b2566bSdrh   return -1;
193d1b2566bSdrh }
194d1b2566bSdrh 
195d1b2566bSdrh /*
196d1b2566bSdrh ** Interpret zArg as an integer value, possibly with suffixes.
197d1b2566bSdrh */
integerValue(const char * zArg)198d1b2566bSdrh static int integerValue(const char *zArg){
199d1b2566bSdrh   int v = 0;
200d1b2566bSdrh   static const struct { char *zSuffix; int iMult; } aMult[] = {
201d1b2566bSdrh     { "KiB", 1024 },
202d1b2566bSdrh     { "MiB", 1024*1024 },
203d1b2566bSdrh     { "GiB", 1024*1024*1024 },
204d1b2566bSdrh     { "KB",  1000 },
205d1b2566bSdrh     { "MB",  1000000 },
206d1b2566bSdrh     { "GB",  1000000000 },
207d1b2566bSdrh     { "K",   1000 },
208d1b2566bSdrh     { "M",   1000000 },
209d1b2566bSdrh     { "G",   1000000000 },
210d1b2566bSdrh   };
211d1b2566bSdrh   int i;
212d1b2566bSdrh   int isNeg = 0;
213d1b2566bSdrh   if( zArg[0]=='-' ){
214d1b2566bSdrh     isNeg = 1;
215d1b2566bSdrh     zArg++;
216d1b2566bSdrh   }else if( zArg[0]=='+' ){
217d1b2566bSdrh     zArg++;
218d1b2566bSdrh   }
219d1b2566bSdrh   if( zArg[0]=='0' && zArg[1]=='x' ){
220d1b2566bSdrh     int x;
221d1b2566bSdrh     zArg += 2;
222d1b2566bSdrh     while( (x = hexDigitValue(zArg[0]))>=0 ){
223d1b2566bSdrh       v = (v<<4) + x;
224d1b2566bSdrh       zArg++;
225d1b2566bSdrh     }
226d1b2566bSdrh   }else{
227d1b2566bSdrh     while( zArg[0]>='0' && zArg[0]<='9' ){
228d1b2566bSdrh       v = v*10 + zArg[0] - '0';
229d1b2566bSdrh       zArg++;
230d1b2566bSdrh     }
231d1b2566bSdrh   }
232d1b2566bSdrh   for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){
233d1b2566bSdrh     if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
234d1b2566bSdrh       v *= aMult[i].iMult;
235d1b2566bSdrh       break;
236d1b2566bSdrh     }
237d1b2566bSdrh   }
238d1b2566bSdrh   return isNeg? -v : v;
239d1b2566bSdrh }
240d1b2566bSdrh 
241d1b2566bSdrh 
242d1b2566bSdrh /*
243cac028baSdrh ** Check the filesystem object zPath.  Determine what it is:
244cac028baSdrh **
2451346fdadSdrh **    PATH_DIR     A single directory holding many files
2461346fdadSdrh **    PATH_TREE    A directory hierarchy with files at the leaves
247cac028baSdrh **    PATH_DB      An SQLite database
248cac028baSdrh **    PATH_NEXIST  Does not exist
249cac028baSdrh **    PATH_OTHER   Something else
2501346fdadSdrh **
2511346fdadSdrh ** PATH_DIR means all of the separate files are grouped together
2521346fdadSdrh ** into a single directory with names like 000000, 000001, 000002, and
2531346fdadSdrh ** so forth.  PATH_TREE means there is a hierarchy of directories so
2541346fdadSdrh ** that no single directory has too many entries.  The files have names
2551346fdadSdrh ** like 00/00/00, 00/00/01, 00/00/02 and so forth.  The decision between
2561346fdadSdrh ** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
2571346fdadSdrh ** named "00" at the top-level.
258cac028baSdrh */
259cac028baSdrh #define PATH_DIR     1
2601346fdadSdrh #define PATH_TREE    2
2611346fdadSdrh #define PATH_DB      3
262cac028baSdrh #define PATH_NEXIST  0
263cac028baSdrh #define PATH_OTHER   99
pathType(const char * zPath)264cac028baSdrh static int pathType(const char *zPath){
265cac028baSdrh   struct stat x;
266cac028baSdrh   int rc;
267cac028baSdrh   if( access(zPath,R_OK) ) return PATH_NEXIST;
268cac028baSdrh   memset(&x, 0, sizeof(x));
269cac028baSdrh   rc = stat(zPath, &x);
270cac028baSdrh   if( rc<0 ) return PATH_OTHER;
2711346fdadSdrh   if( S_ISDIR(x.st_mode) ){
2721346fdadSdrh     char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
2731346fdadSdrh     memset(&x, 0, sizeof(x));
2741346fdadSdrh     rc = stat(zLayer1, &x);
2751346fdadSdrh     sqlite3_free(zLayer1);
2761346fdadSdrh     if( rc<0 ) return PATH_DIR;
2771346fdadSdrh     if( S_ISDIR(x.st_mode) ) return PATH_TREE;
2781346fdadSdrh     return PATH_DIR;
2791346fdadSdrh   }
280cac028baSdrh   if( (x.st_size%512)==0 ) return PATH_DB;
281cac028baSdrh   return PATH_OTHER;
282cac028baSdrh }
283cac028baSdrh 
284cac028baSdrh /*
285cae20d5cSdrh ** Return the size of a file in bytes.  Or return -1 if the
286cae20d5cSdrh ** named object is not a regular file or does not exist.
287cae20d5cSdrh */
fileSize(const char * zPath)288cae20d5cSdrh static sqlite3_int64 fileSize(const char *zPath){
289cae20d5cSdrh   struct stat x;
290cae20d5cSdrh   int rc;
291cae20d5cSdrh   memset(&x, 0, sizeof(x));
292cae20d5cSdrh   rc = stat(zPath, &x);
293cae20d5cSdrh   if( rc<0 ) return -1;
294cae20d5cSdrh   if( !S_ISREG(x.st_mode) ) return -1;
295cae20d5cSdrh   return x.st_size;
296cae20d5cSdrh }
297cae20d5cSdrh 
298cae20d5cSdrh /*
299cac028baSdrh ** A Pseudo-random number generator with a fixed seed.  Use this so
300cac028baSdrh ** that the same sequence of "random" numbers are generated on each
301cac028baSdrh ** run, for repeatability.
302cac028baSdrh */
randInt(void)303cac028baSdrh static unsigned int randInt(void){
304cac028baSdrh   static unsigned int x = 0x333a13cd;
305cac028baSdrh   static unsigned int y = 0xecb2adea;
306cac028baSdrh   x = (x>>1) ^ ((1+~(x&1)) & 0xd0000001);
307cac028baSdrh   y = y*1103515245 + 12345;
308cac028baSdrh   return x^y;
309cac028baSdrh }
310cac028baSdrh 
311cac028baSdrh /*
312cac028baSdrh ** Do database initialization.
313cac028baSdrh */
initMain(int argc,char ** argv)314cac028baSdrh static int initMain(int argc, char **argv){
315cac028baSdrh   char *zDb;
316cac028baSdrh   int i, rc;
317cac028baSdrh   int nCount = 1000;
318cac028baSdrh   int sz = 10000;
3199dc7ad1dSdrh   int iVariance = 0;
320cac028baSdrh   int pgsz = 4096;
321cac028baSdrh   sqlite3 *db;
322cac028baSdrh   char *zSql;
323cac028baSdrh   char *zErrMsg = 0;
324cac028baSdrh 
325cac028baSdrh   assert( strcmp(argv[1],"init")==0 );
326cac028baSdrh   assert( argc>=3 );
327cac028baSdrh   zDb = argv[2];
328cac028baSdrh   for(i=3; i<argc; i++){
329cac028baSdrh     char *z = argv[i];
330cac028baSdrh     if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
331cac028baSdrh     if( z[1]=='-' ) z++;
332cac028baSdrh     if( strcmp(z, "-count")==0 ){
333cac028baSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
334d1b2566bSdrh       nCount = integerValue(argv[++i]);
335cac028baSdrh       if( nCount<1 ) fatalError("the --count must be positive");
336cac028baSdrh       continue;
337cac028baSdrh     }
338cac028baSdrh     if( strcmp(z, "-size")==0 ){
339cac028baSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
340d1b2566bSdrh       sz = integerValue(argv[++i]);
341cac028baSdrh       if( sz<1 ) fatalError("the --size must be positive");
342cac028baSdrh       continue;
343cac028baSdrh     }
3449dc7ad1dSdrh     if( strcmp(z, "-variance")==0 ){
3459dc7ad1dSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
3469dc7ad1dSdrh       iVariance = integerValue(argv[++i]);
3479dc7ad1dSdrh       continue;
3489dc7ad1dSdrh     }
349cac028baSdrh     if( strcmp(z, "-pagesize")==0 ){
350cac028baSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
351d1b2566bSdrh       pgsz = integerValue(argv[++i]);
352cac028baSdrh       if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){
353cac028baSdrh         fatalError("the --pagesize must be power of 2 between 512 and 65536");
354cac028baSdrh       }
355cac028baSdrh       continue;
356cac028baSdrh     }
357cac028baSdrh     fatalError("unknown option: \"%s\"", argv[i]);
358cac028baSdrh   }
359cac028baSdrh   rc = sqlite3_open(zDb, &db);
360cac028baSdrh   if( rc ){
361cac028baSdrh     fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
362cac028baSdrh   }
363cac028baSdrh   zSql = sqlite3_mprintf(
364cac028baSdrh     "DROP TABLE IF EXISTS kv;\n"
365cac028baSdrh     "PRAGMA page_size=%d;\n"
366cac028baSdrh     "VACUUM;\n"
367cac028baSdrh     "BEGIN;\n"
368cac028baSdrh     "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n"
369cac028baSdrh     "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)"
3709dc7ad1dSdrh     " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n"
371cac028baSdrh     "COMMIT;\n",
372809955b5Sdan     pgsz, nCount, sz, iVariance+1
373cac028baSdrh   );
374cac028baSdrh   rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
375cac028baSdrh   if( rc ) fatalError("database create failed: %s", zErrMsg);
376cac028baSdrh   sqlite3_free(zSql);
377cac028baSdrh   sqlite3_close(db);
378cac028baSdrh   return 0;
379cac028baSdrh }
380cac028baSdrh 
381cac028baSdrh /*
3829dc7ad1dSdrh ** Analyze an existing database file.  Report its content.
3839dc7ad1dSdrh */
statMain(int argc,char ** argv)3849dc7ad1dSdrh static int statMain(int argc, char **argv){
3859dc7ad1dSdrh   char *zDb;
3869dc7ad1dSdrh   int i, rc;
3879dc7ad1dSdrh   sqlite3 *db;
3889dc7ad1dSdrh   char *zSql;
3899dc7ad1dSdrh   sqlite3_stmt *pStmt;
390a4327e65Sdrh   int doVacuum = 0;
3919dc7ad1dSdrh 
3929dc7ad1dSdrh   assert( strcmp(argv[1],"stat")==0 );
3939dc7ad1dSdrh   assert( argc>=3 );
3949dc7ad1dSdrh   zDb = argv[2];
3959dc7ad1dSdrh   for(i=3; i<argc; i++){
3969dc7ad1dSdrh     char *z = argv[i];
3979dc7ad1dSdrh     if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
3989dc7ad1dSdrh     if( z[1]=='-' ) z++;
399a4327e65Sdrh     if( strcmp(z, "-vacuum")==0 ){
400a4327e65Sdrh       doVacuum = 1;
401a4327e65Sdrh       continue;
402a4327e65Sdrh     }
4039dc7ad1dSdrh     fatalError("unknown option: \"%s\"", argv[i]);
4049dc7ad1dSdrh   }
4059dc7ad1dSdrh   rc = sqlite3_open(zDb, &db);
4069dc7ad1dSdrh   if( rc ){
4079dc7ad1dSdrh     fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
4089dc7ad1dSdrh   }
409a4327e65Sdrh   if( doVacuum ){
410a4327e65Sdrh     printf("Vacuuming...."); fflush(stdout);
411a4327e65Sdrh     sqlite3_exec(db, "VACUUM", 0, 0, 0);
412a4327e65Sdrh     printf("       done\n");
413a4327e65Sdrh   }
4149dc7ad1dSdrh   zSql = sqlite3_mprintf(
4159dc7ad1dSdrh     "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"
4169dc7ad1dSdrh     "  FROM kv"
4179dc7ad1dSdrh   );
4189dc7ad1dSdrh   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4199dc7ad1dSdrh   if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4209dc7ad1dSdrh   sqlite3_free(zSql);
4219dc7ad1dSdrh   if( sqlite3_step(pStmt)==SQLITE_ROW ){
4229dc7ad1dSdrh     printf("Number of entries:  %8d\n", sqlite3_column_int(pStmt, 0));
4239dc7ad1dSdrh     printf("Average value size: %8d\n", sqlite3_column_int(pStmt, 3));
4249dc7ad1dSdrh     printf("Minimum value size: %8d\n", sqlite3_column_int(pStmt, 1));
4259dc7ad1dSdrh     printf("Maximum value size: %8d\n", sqlite3_column_int(pStmt, 2));
4269dc7ad1dSdrh   }else{
4279dc7ad1dSdrh     printf("No rows\n");
4289dc7ad1dSdrh   }
4299dc7ad1dSdrh   sqlite3_finalize(pStmt);
4309dc7ad1dSdrh   zSql = sqlite3_mprintf("PRAGMA page_size");
4319dc7ad1dSdrh   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4329dc7ad1dSdrh   if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4339dc7ad1dSdrh   sqlite3_free(zSql);
4349dc7ad1dSdrh   if( sqlite3_step(pStmt)==SQLITE_ROW ){
4359dc7ad1dSdrh     printf("Page-size:          %8d\n", sqlite3_column_int(pStmt, 0));
4369dc7ad1dSdrh   }
4379dc7ad1dSdrh   sqlite3_finalize(pStmt);
4389dc7ad1dSdrh   zSql = sqlite3_mprintf("PRAGMA page_count");
4399dc7ad1dSdrh   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
4409dc7ad1dSdrh   if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
4419dc7ad1dSdrh   sqlite3_free(zSql);
4429dc7ad1dSdrh   if( sqlite3_step(pStmt)==SQLITE_ROW ){
4439dc7ad1dSdrh     printf("Page-count:         %8d\n", sqlite3_column_int(pStmt, 0));
4449dc7ad1dSdrh   }
4459dc7ad1dSdrh   sqlite3_finalize(pStmt);
446a4327e65Sdrh   zSql = sqlite3_mprintf("PRAGMA freelist_count");
447a4327e65Sdrh   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
448a4327e65Sdrh   if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
449a4327e65Sdrh   sqlite3_free(zSql);
450a4327e65Sdrh   if( sqlite3_step(pStmt)==SQLITE_ROW ){
451a4327e65Sdrh     printf("Freelist-count:     %8d\n", sqlite3_column_int(pStmt, 0));
452a4327e65Sdrh   }
453a4327e65Sdrh   sqlite3_finalize(pStmt);
454a4327e65Sdrh   rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0);
455a4327e65Sdrh   if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db));
456a4327e65Sdrh   while( sqlite3_step(pStmt)==SQLITE_ROW ){
457a4327e65Sdrh     printf("Integrity-check:    %s\n", sqlite3_column_text(pStmt, 0));
458a4327e65Sdrh   }
459a4327e65Sdrh   sqlite3_finalize(pStmt);
4609dc7ad1dSdrh   sqlite3_close(db);
4619dc7ad1dSdrh   return 0;
4629dc7ad1dSdrh }
4639dc7ad1dSdrh 
4649dc7ad1dSdrh /*
465a2e71cedSdrh **      remember(V,PTR)
466a2e71cedSdrh **
467a2e71cedSdrh ** Return the integer value V.  Also save the value of V in a
468a2e71cedSdrh ** C-language variable whose address is PTR.
469a2e71cedSdrh */
rememberFunc(sqlite3_context * pCtx,int argc,sqlite3_value ** argv)470a2e71cedSdrh static void rememberFunc(
471a2e71cedSdrh   sqlite3_context *pCtx,
472a2e71cedSdrh   int argc,
473a2e71cedSdrh   sqlite3_value **argv
474a2e71cedSdrh ){
475a2e71cedSdrh   sqlite3_int64 v;
476a2e71cedSdrh   sqlite3_int64 ptr;
477a2e71cedSdrh   assert( argc==2 );
478a2e71cedSdrh   v = sqlite3_value_int64(argv[0]);
479a2e71cedSdrh   ptr = sqlite3_value_int64(argv[1]);
480a2e71cedSdrh   *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
481a2e71cedSdrh   sqlite3_result_int64(pCtx, v);
482a2e71cedSdrh }
483a2e71cedSdrh 
484a2e71cedSdrh /*
4851346fdadSdrh ** Make sure a directory named zDir exists.
4861346fdadSdrh */
kvtest_mkdir(const char * zDir)4871346fdadSdrh static void kvtest_mkdir(const char *zDir){
4881346fdadSdrh #if defined(_WIN32)
4891346fdadSdrh   (void)mkdir(zDir);
4901346fdadSdrh #else
4911346fdadSdrh   (void)mkdir(zDir, 0755);
4921346fdadSdrh #endif
4931346fdadSdrh }
4941346fdadSdrh 
4951346fdadSdrh /*
496cac028baSdrh ** Export the kv table to individual files in the filesystem
497cac028baSdrh */
exportMain(int argc,char ** argv)498cac028baSdrh static int exportMain(int argc, char **argv){
499cac028baSdrh   char *zDb;
500cac028baSdrh   char *zDir;
501cac028baSdrh   sqlite3 *db;
5021346fdadSdrh   sqlite3_stmt *pStmt;
503cac028baSdrh   int rc;
5041346fdadSdrh   int ePathType;
5051346fdadSdrh   int nFN;
5061346fdadSdrh   char *zFN;
5071346fdadSdrh   char *zTail;
5081346fdadSdrh   size_t nWrote;
5091346fdadSdrh   int i;
510cac028baSdrh 
511cac028baSdrh   assert( strcmp(argv[1],"export")==0 );
512cac028baSdrh   assert( argc>=3 );
5131346fdadSdrh   if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
514cac028baSdrh   zDb = argv[2];
515cac028baSdrh   zDir = argv[3];
5161346fdadSdrh   kvtest_mkdir(zDir);
5171346fdadSdrh   for(i=4; i<argc; i++){
5181346fdadSdrh     const char *z = argv[i];
5191346fdadSdrh     if( z[0]=='-' && z[1]=='-' ) z++;
5201346fdadSdrh     if( strcmp(z,"-tree")==0 ){
5211346fdadSdrh       zFN = sqlite3_mprintf("%s/00", zDir);
5221346fdadSdrh       kvtest_mkdir(zFN);
5231346fdadSdrh       sqlite3_free(zFN);
5241346fdadSdrh       continue;
5251346fdadSdrh     }
5261346fdadSdrh     fatalError("unknown argument: \"%s\"\n", argv[i]);
5271346fdadSdrh   }
5281346fdadSdrh   ePathType = pathType(zDir);
5291346fdadSdrh   if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
530cac028baSdrh     fatalError("object \"%s\" is not a directory", zDir);
531cac028baSdrh   }
532cac028baSdrh   rc = sqlite3_open(zDb, &db);
533cac028baSdrh   if( rc ){
534cac028baSdrh     fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
535cac028baSdrh   }
5361346fdadSdrh   rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
5371346fdadSdrh   if( rc ){
5381346fdadSdrh     fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
5391346fdadSdrh   }
5401346fdadSdrh   nFN = (int)strlen(zDir);
5411346fdadSdrh   zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
5421346fdadSdrh   if( zFN==0 ){
5431346fdadSdrh     fatalError("malloc failed\n");
5441346fdadSdrh   }
5451346fdadSdrh   zTail = zFN + nFN + 1;
5461346fdadSdrh   while( sqlite3_step(pStmt)==SQLITE_ROW ){
5471346fdadSdrh     int iKey = sqlite3_column_int(pStmt, 0);
548cf1393dfSdrh     sqlite3_int64 nData = sqlite3_column_bytes(pStmt, 1);
5491346fdadSdrh     const void *pData = sqlite3_column_blob(pStmt, 1);
5501346fdadSdrh     FILE *out;
5511346fdadSdrh     if( ePathType==PATH_DIR ){
5521346fdadSdrh       sqlite3_snprintf(20, zTail, "%06d", iKey);
5531346fdadSdrh     }else{
5541346fdadSdrh       sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
5551346fdadSdrh       kvtest_mkdir(zFN);
5561346fdadSdrh       sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
5571346fdadSdrh       kvtest_mkdir(zFN);
5581346fdadSdrh       sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
5591346fdadSdrh                        iKey/10000, (iKey/100)%100, iKey%100);
5601346fdadSdrh     }
5611346fdadSdrh     out = fopen(zFN, "wb");
562ac8ba26eSmistachkin     nWrote = fwrite(pData, 1, (size_t)nData, out);
5631346fdadSdrh     fclose(out);
5641346fdadSdrh     printf("\r%s   ", zTail); fflush(stdout);
565928ba149Sdrh     if( nWrote!=(size_t)nData ){
5661346fdadSdrh       fatalError("Wrote only %d of %d bytes to %s\n",
5671346fdadSdrh                   (int)nWrote, nData, zFN);
5681346fdadSdrh     }
5691346fdadSdrh   }
5701346fdadSdrh   sqlite3_finalize(pStmt);
571cac028baSdrh   sqlite3_close(db);
5721346fdadSdrh   sqlite3_free(zFN);
5736739c692Sdrh   printf("\n");
574cac028baSdrh   return 0;
575cac028baSdrh }
576cac028baSdrh 
577cac028baSdrh /*
578cac028baSdrh ** Read the content of file zName into memory obtained from sqlite3_malloc64()
579cac028baSdrh ** and return a pointer to the buffer. The caller is responsible for freeing
580cac028baSdrh ** the memory.
581cac028baSdrh **
582cac028baSdrh ** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
583cac028baSdrh ** read.
584cac028baSdrh **
585cac028baSdrh ** For convenience, a nul-terminator byte is always appended to the data read
586cac028baSdrh ** from the file before the buffer is returned. This byte is not included in
587cac028baSdrh ** the final value of (*pnByte), if applicable.
588cac028baSdrh **
589cac028baSdrh ** NULL is returned if any error is encountered. The final value of *pnByte
590cac028baSdrh ** is undefined in this case.
591cac028baSdrh */
readFile(const char * zName,sqlite3_int64 * pnByte)592cf1393dfSdrh static unsigned char *readFile(const char *zName, sqlite3_int64 *pnByte){
593cae20d5cSdrh   FILE *in;               /* FILE from which to read content of zName */
594cae20d5cSdrh   sqlite3_int64 nIn;      /* Size of zName in bytes */
595cae20d5cSdrh   size_t nRead;           /* Number of bytes actually read */
596cae20d5cSdrh   unsigned char *pBuf;    /* Content read from disk */
597cae20d5cSdrh 
598cae20d5cSdrh   nIn = fileSize(zName);
599cae20d5cSdrh   if( nIn<0 ) return 0;
600cae20d5cSdrh   in = fopen(zName, "rb");
601cac028baSdrh   if( in==0 ) return 0;
602cae20d5cSdrh   pBuf = sqlite3_malloc64( nIn );
603cac028baSdrh   if( pBuf==0 ) return 0;
604b67e1754Sdrh   nRead = fread(pBuf, (size_t)nIn, 1, in);
605cac028baSdrh   fclose(in);
606cac028baSdrh   if( nRead!=1 ){
607cac028baSdrh     sqlite3_free(pBuf);
608cac028baSdrh     return 0;
609cac028baSdrh   }
610cf1393dfSdrh   if( pnByte ) *pnByte = nIn;
611cac028baSdrh   return pBuf;
612cac028baSdrh }
613cac028baSdrh 
614cac028baSdrh /*
615a2e71cedSdrh ** Overwrite a file with randomness.  Do not change the size of the
616a2e71cedSdrh ** file.
617a2e71cedSdrh */
updateFile(const char * zName,sqlite3_int64 * pnByte,int doFsync)618cf1393dfSdrh static void updateFile(const char *zName, sqlite3_int64 *pnByte, int doFsync){
619a2e71cedSdrh   FILE *out;              /* FILE from which to read content of zName */
620a2e71cedSdrh   sqlite3_int64 sz;       /* Size of zName in bytes */
621a2e71cedSdrh   size_t nWritten;        /* Number of bytes actually read */
622a2e71cedSdrh   unsigned char *pBuf;    /* Content to store on disk */
623f1518907Sdrh   const char *zMode = "wb";   /* Mode for fopen() */
624a2e71cedSdrh 
625a2e71cedSdrh   sz = fileSize(zName);
626a2e71cedSdrh   if( sz<0 ){
627a2e71cedSdrh     fatalError("No such file: \"%s\"", zName);
628a2e71cedSdrh   }
629cf1393dfSdrh   *pnByte = sz;
630a2e71cedSdrh   if( sz==0 ) return;
631a2e71cedSdrh   pBuf = sqlite3_malloc64( sz );
632a2e71cedSdrh   if( pBuf==0 ){
633a2e71cedSdrh     fatalError("Cannot allocate %lld bytes\n", sz);
634a2e71cedSdrh   }
635a2e71cedSdrh   sqlite3_randomness((int)sz, pBuf);
636f1518907Sdrh #if defined(_WIN32)
637f1518907Sdrh   if( doFsync ) zMode = "wbc";
638f1518907Sdrh #endif
639f1518907Sdrh   out = fopen(zName, zMode);
640a2e71cedSdrh   if( out==0 ){
641a2e71cedSdrh     fatalError("Cannot open \"%s\" for writing\n", zName);
642a2e71cedSdrh   }
643a2e71cedSdrh   nWritten = fwrite(pBuf, 1, (size_t)sz, out);
644f1518907Sdrh   if( doFsync ){
645f1518907Sdrh #if defined(_WIN32)
646f1518907Sdrh     fflush(out);
647f1518907Sdrh #else
648f1518907Sdrh     fsync(fileno(out));
649f1518907Sdrh #endif
650f1518907Sdrh   }
651a2e71cedSdrh   fclose(out);
652a2e71cedSdrh   if( nWritten!=(size_t)sz ){
653a2e71cedSdrh     fatalError("Wrote only %d of %d bytes to \"%s\"\n",
654a2e71cedSdrh                (int)nWritten, (int)sz, zName);
655a2e71cedSdrh   }
656a2e71cedSdrh   sqlite3_free(pBuf);
657a2e71cedSdrh }
658a2e71cedSdrh 
659a2e71cedSdrh /*
660cac028baSdrh ** Return the current time in milliseconds since the beginning of
661cac028baSdrh ** the Julian epoch.
662cac028baSdrh */
timeOfDay(void)663cac028baSdrh static sqlite3_int64 timeOfDay(void){
664cac028baSdrh   static sqlite3_vfs *clockVfs = 0;
665cac028baSdrh   sqlite3_int64 t;
666cac028baSdrh   if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
667cac028baSdrh   if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
668cac028baSdrh     clockVfs->xCurrentTimeInt64(clockVfs, &t);
669cac028baSdrh   }else{
670cac028baSdrh     double r;
671cac028baSdrh     clockVfs->xCurrentTime(clockVfs, &r);
672cac028baSdrh     t = (sqlite3_int64)(r*86400000.0);
673cac028baSdrh   }
674cac028baSdrh   return t;
675cac028baSdrh }
676cac028baSdrh 
677befcd8adSdan #ifdef __linux__
678befcd8adSdan /*
679befcd8adSdan ** Attempt to display I/O stats on Linux using /proc/PID/io
680befcd8adSdan */
displayLinuxIoStats(FILE * out)681befcd8adSdan static void displayLinuxIoStats(FILE *out){
682befcd8adSdan   FILE *in;
683befcd8adSdan   char z[200];
684befcd8adSdan   sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
685befcd8adSdan   in = fopen(z, "rb");
686befcd8adSdan   if( in==0 ) return;
687befcd8adSdan   while( fgets(z, sizeof(z), in)!=0 ){
688befcd8adSdan     static const struct {
689befcd8adSdan       const char *zPattern;
690befcd8adSdan       const char *zDesc;
691befcd8adSdan     } aTrans[] = {
692befcd8adSdan       { "rchar: ",                  "Bytes received by read():" },
693befcd8adSdan       { "wchar: ",                  "Bytes sent to write():"    },
694befcd8adSdan       { "syscr: ",                  "Read() system calls:"      },
695befcd8adSdan       { "syscw: ",                  "Write() system calls:"     },
696befcd8adSdan       { "read_bytes: ",             "Bytes read from storage:"  },
697befcd8adSdan       { "write_bytes: ",            "Bytes written to storage:" },
698befcd8adSdan       { "cancelled_write_bytes: ",  "Cancelled write bytes:"    },
699befcd8adSdan     };
700befcd8adSdan     int i;
701befcd8adSdan     for(i=0; i<sizeof(aTrans)/sizeof(aTrans[0]); i++){
702befcd8adSdan       int n = (int)strlen(aTrans[i].zPattern);
703befcd8adSdan       if( strncmp(aTrans[i].zPattern, z, n)==0 ){
704befcd8adSdan         fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);
705befcd8adSdan         break;
706befcd8adSdan       }
707befcd8adSdan     }
708befcd8adSdan   }
709befcd8adSdan   fclose(in);
710befcd8adSdan }
711befcd8adSdan #endif
712befcd8adSdan 
713befcd8adSdan /*
714befcd8adSdan ** Display memory stats.
715befcd8adSdan */
display_stats(sqlite3 * db,int bReset)716befcd8adSdan static int display_stats(
717befcd8adSdan   sqlite3 *db,                    /* Database to query */
718befcd8adSdan   int bReset                      /* True to reset SQLite stats */
719befcd8adSdan ){
720befcd8adSdan   int iCur;
721befcd8adSdan   int iHiwtr;
722befcd8adSdan   FILE *out = stdout;
723befcd8adSdan 
724befcd8adSdan   fprintf(out, "\n");
725befcd8adSdan 
726befcd8adSdan   iHiwtr = iCur = -1;
727befcd8adSdan   sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset);
728befcd8adSdan   fprintf(out,
729befcd8adSdan           "Memory Used:                         %d (max %d) bytes\n",
730befcd8adSdan           iCur, iHiwtr);
731befcd8adSdan   iHiwtr = iCur = -1;
732befcd8adSdan   sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset);
733befcd8adSdan   fprintf(out, "Number of Outstanding Allocations:   %d (max %d)\n",
734befcd8adSdan           iCur, iHiwtr);
735befcd8adSdan   iHiwtr = iCur = -1;
736befcd8adSdan   sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset);
737befcd8adSdan   fprintf(out,
738befcd8adSdan       "Number of Pcache Pages Used:         %d (max %d) pages\n",
739befcd8adSdan       iCur, iHiwtr);
740befcd8adSdan   iHiwtr = iCur = -1;
741befcd8adSdan   sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset);
742befcd8adSdan   fprintf(out,
743befcd8adSdan           "Number of Pcache Overflow Bytes:     %d (max %d) bytes\n",
744befcd8adSdan           iCur, iHiwtr);
745befcd8adSdan   iHiwtr = iCur = -1;
746befcd8adSdan   sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset);
747befcd8adSdan   fprintf(out, "Largest Allocation:                  %d bytes\n",
748befcd8adSdan           iHiwtr);
749befcd8adSdan   iHiwtr = iCur = -1;
750befcd8adSdan   sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset);
751befcd8adSdan   fprintf(out, "Largest Pcache Allocation:           %d bytes\n",
752befcd8adSdan           iHiwtr);
753befcd8adSdan 
754befcd8adSdan   iHiwtr = iCur = -1;
755befcd8adSdan   sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset);
756befcd8adSdan   fprintf(out, "Pager Heap Usage:                    %d bytes\n",
757befcd8adSdan       iCur);
758befcd8adSdan   iHiwtr = iCur = -1;
759befcd8adSdan   sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);
760befcd8adSdan   fprintf(out, "Page cache hits:                     %d\n", iCur);
761befcd8adSdan   iHiwtr = iCur = -1;
762befcd8adSdan   sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);
763befcd8adSdan   fprintf(out, "Page cache misses:                   %d\n", iCur);
764befcd8adSdan   iHiwtr = iCur = -1;
765befcd8adSdan   sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);
766befcd8adSdan   fprintf(out, "Page cache writes:                   %d\n", iCur);
767befcd8adSdan   iHiwtr = iCur = -1;
768befcd8adSdan 
769befcd8adSdan #ifdef __linux__
770befcd8adSdan   displayLinuxIoStats(out);
771befcd8adSdan #endif
772befcd8adSdan 
773befcd8adSdan   return 0;
774befcd8adSdan }
775befcd8adSdan 
776cac028baSdrh /* Blob access order */
777cac028baSdrh #define ORDER_ASC     1
778cac028baSdrh #define ORDER_DESC    2
779cac028baSdrh #define ORDER_RANDOM  3
780cac028baSdrh 
781d1b2566bSdrh 
782cac028baSdrh /*
783cac028baSdrh ** Run a performance test
784cac028baSdrh */
runMain(int argc,char ** argv)785cac028baSdrh static int runMain(int argc, char **argv){
786cac028baSdrh   int eType;                  /* Is zDb a database or a directory? */
787cac028baSdrh   char *zDb;                  /* Database or directory name */
788cac028baSdrh   int i;                      /* Loop counter */
789cac028baSdrh   int rc;                     /* Return code from SQLite calls */
790cac028baSdrh   int nCount = 1000;          /* Number of blob fetch operations */
791cac028baSdrh   int nExtra = 0;             /* Extra cycles */
792cac028baSdrh   int iKey = 1;               /* Next blob key */
7932b15f6e1Sdrh   int iMax = 0;               /* Largest allowed key */
79461c565f6Sdrh   int iPagesize = 0;          /* Database page size */
79561c565f6Sdrh   int iCache = 1000;          /* Database cache size in kibibytes */
796cac028baSdrh   int bBlobApi = 0;           /* Use the incremental blob I/O API */
797befcd8adSdan   int bStats = 0;             /* Print stats before exiting */
798cac028baSdrh   int eOrder = ORDER_ASC;     /* Access order */
799a2e71cedSdrh   int isUpdateTest = 0;       /* Do in-place updates rather than reads */
800a2e71cedSdrh   int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
801a2e71cedSdrh   int noSync = 0;             /* Disable synchronous mode */
802f1518907Sdrh   int doFsync = 0;            /* Update disk files synchronously */
80316d62901Sdrh   int doMultiTrans = 0;       /* Each operation in its own transaction */
80416d62901Sdrh   int noCheckpoint = 0;       /* Omit the checkpoint in WAL mode */
805cac028baSdrh   sqlite3 *db = 0;            /* Database connection */
806cac028baSdrh   sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
807cac028baSdrh   sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
808cac028baSdrh   sqlite3_int64 tmStart;      /* Start time */
809cac028baSdrh   sqlite3_int64 tmElapsed;    /* Elapsed time */
810bace3240Sdrh   int mmapSize = 0;           /* --mmap N argument */
811cf1393dfSdrh   sqlite3_int64 nData = 0;    /* Bytes of data */
812cac028baSdrh   sqlite3_int64 nTotal = 0;   /* Total data read */
813eacb4412Sdrh   unsigned char *pData = 0;   /* Content of the blob */
814cf1393dfSdrh   sqlite3_int64 nAlloc = 0;   /* Space allocated for pData[] */
8152b15f6e1Sdrh   const char *zJMode = 0;     /* Journal mode */
816cac028baSdrh 
817cac028baSdrh 
818cac028baSdrh   assert( strcmp(argv[1],"run")==0 );
819cac028baSdrh   assert( argc>=3 );
820cac028baSdrh   zDb = argv[2];
821cac028baSdrh   eType = pathType(zDb);
822cac028baSdrh   if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
823cac028baSdrh   if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
824cac028baSdrh   for(i=3; i<argc; i++){
825cac028baSdrh     char *z = argv[i];
826cac028baSdrh     if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
827cac028baSdrh     if( z[1]=='-' ) z++;
82816d62901Sdrh     if( strcmp(z, "-asc")==0 ){
82916d62901Sdrh       eOrder = ORDER_ASC;
83016d62901Sdrh       continue;
83116d62901Sdrh     }
83216d62901Sdrh     if( strcmp(z, "-blob-api")==0 ){
83316d62901Sdrh       bBlobApi = 1;
83416d62901Sdrh       continue;
83516d62901Sdrh     }
83616d62901Sdrh     if( strcmp(z, "-cache-size")==0 ){
83716d62901Sdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
83816d62901Sdrh       iCache = integerValue(argv[++i]);
83916d62901Sdrh       continue;
84016d62901Sdrh     }
841cac028baSdrh     if( strcmp(z, "-count")==0 ){
842cac028baSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
843d1b2566bSdrh       nCount = integerValue(argv[++i]);
844cac028baSdrh       if( nCount<1 ) fatalError("the --count must be positive");
845cac028baSdrh       continue;
846cac028baSdrh     }
84716d62901Sdrh     if( strcmp(z, "-desc")==0 ){
84816d62901Sdrh       eOrder = ORDER_DESC;
84916d62901Sdrh       continue;
85016d62901Sdrh     }
85116d62901Sdrh     if( strcmp(z, "-fsync")==0 ){
85216d62901Sdrh       doFsync = 1;
85316d62901Sdrh       continue;
85416d62901Sdrh     }
85516d62901Sdrh     if( strcmp(z, "-integrity-check")==0 ){
85616d62901Sdrh       doIntegrityCk = 1;
85716d62901Sdrh       continue;
85816d62901Sdrh     }
85916d62901Sdrh     if( strcmp(z, "-jmode")==0 ){
86016d62901Sdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
86116d62901Sdrh       zJMode = argv[++i];
86216d62901Sdrh       continue;
86316d62901Sdrh     }
864bace3240Sdrh     if( strcmp(z, "-mmap")==0 ){
865bace3240Sdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
866d1b2566bSdrh       mmapSize = integerValue(argv[++i]);
867bace3240Sdrh       if( nCount<0 ) fatalError("the --mmap must be non-negative");
868bace3240Sdrh       continue;
869bace3240Sdrh     }
870cac028baSdrh     if( strcmp(z, "-max-id")==0 ){
871cac028baSdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
872d1b2566bSdrh       iMax = integerValue(argv[++i]);
873cac028baSdrh       continue;
874cac028baSdrh     }
87516d62901Sdrh     if( strcmp(z, "-multitrans")==0 ){
87616d62901Sdrh       doMultiTrans = 1;
877cac028baSdrh       continue;
878cac028baSdrh     }
87916d62901Sdrh     if( strcmp(z, "-nocheckpoint")==0 ){
88016d62901Sdrh       noCheckpoint = 1;
88161c565f6Sdrh       continue;
88261c565f6Sdrh     }
88316d62901Sdrh     if( strcmp(z, "-nosync")==0 ){
88416d62901Sdrh       noSync = 1;
8852b15f6e1Sdrh       continue;
8862b15f6e1Sdrh     }
887cac028baSdrh     if( strcmp(z, "-random")==0 ){
888cac028baSdrh       eOrder = ORDER_RANDOM;
889cac028baSdrh       continue;
890cac028baSdrh     }
89116d62901Sdrh     if( strcmp(z, "-start")==0 ){
89216d62901Sdrh       if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
89316d62901Sdrh       iKey = integerValue(argv[++i]);
89416d62901Sdrh       if( iKey<1 ) fatalError("the --start must be positive");
895cac028baSdrh       continue;
896cac028baSdrh     }
897befcd8adSdan     if( strcmp(z, "-stats")==0 ){
898befcd8adSdan       bStats = 1;
899befcd8adSdan       continue;
900befcd8adSdan     }
901a2e71cedSdrh     if( strcmp(z, "-update")==0 ){
902a2e71cedSdrh       isUpdateTest = 1;
903a2e71cedSdrh       continue;
904a2e71cedSdrh     }
905cac028baSdrh     fatalError("unknown option: \"%s\"", argv[i]);
906cac028baSdrh   }
907a2e71cedSdrh   if( eType==PATH_DB ){
908a2e71cedSdrh     /* Recover any prior crashes prior to starting the timer */
909a2e71cedSdrh     sqlite3_open(zDb, &db);
910*067b92baSdrh     sqlite3_exec(db, "SELECT rowid FROM sqlite_schema LIMIT 1", 0, 0, 0);
911a2e71cedSdrh     sqlite3_close(db);
9125930f663Sdrh     db = 0;
913a2e71cedSdrh   }
914cac028baSdrh   tmStart = timeOfDay();
915cac028baSdrh   if( eType==PATH_DB ){
91661c565f6Sdrh     char *zSql;
917cac028baSdrh     rc = sqlite3_open(zDb, &db);
918cac028baSdrh     if( rc ){
919cac028baSdrh       fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
920cac028baSdrh     }
921bace3240Sdrh     zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize);
922bace3240Sdrh     sqlite3_exec(db, zSql, 0, 0, 0);
923a2e71cedSdrh     sqlite3_free(zSql);
92461c565f6Sdrh     zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache);
92561c565f6Sdrh     sqlite3_exec(db, zSql, 0, 0, 0);
92661c565f6Sdrh     sqlite3_free(zSql);
927a2e71cedSdrh     if( noSync ){
928a2e71cedSdrh       sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0);
929a2e71cedSdrh     }
93061c565f6Sdrh     pStmt = 0;
93161c565f6Sdrh     sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0);
93261c565f6Sdrh     if( sqlite3_step(pStmt)==SQLITE_ROW ){
93361c565f6Sdrh       iPagesize = sqlite3_column_int(pStmt, 0);
93461c565f6Sdrh     }
93561c565f6Sdrh     sqlite3_finalize(pStmt);
93661c565f6Sdrh     sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0);
93761c565f6Sdrh     if( sqlite3_step(pStmt)==SQLITE_ROW ){
93861c565f6Sdrh       iCache = sqlite3_column_int(pStmt, 0);
93961c565f6Sdrh     }else{
94061c565f6Sdrh       iCache = 0;
94161c565f6Sdrh     }
94261c565f6Sdrh     sqlite3_finalize(pStmt);
94361c565f6Sdrh     pStmt = 0;
9442b15f6e1Sdrh     if( zJMode ){
9452b15f6e1Sdrh       zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);
9462b15f6e1Sdrh       sqlite3_exec(db, zSql, 0, 0, 0);
9472b15f6e1Sdrh       sqlite3_free(zSql);
94816d62901Sdrh       if( noCheckpoint ){
94916d62901Sdrh         sqlite3_exec(db, "PRAGMA wal_autocheckpoint=0", 0, 0, 0);
95016d62901Sdrh       }
9512b15f6e1Sdrh     }
9522b15f6e1Sdrh     sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);
9532b15f6e1Sdrh     if( sqlite3_step(pStmt)==SQLITE_ROW ){
9542b15f6e1Sdrh       zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
9552b15f6e1Sdrh     }else{
9562b15f6e1Sdrh       zJMode = "???";
9572b15f6e1Sdrh     }
9582b15f6e1Sdrh     sqlite3_finalize(pStmt);
9592b15f6e1Sdrh     if( iMax<=0 ){
9602b15f6e1Sdrh       sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);
9612b15f6e1Sdrh       if( sqlite3_step(pStmt)==SQLITE_ROW ){
9622b15f6e1Sdrh         iMax = sqlite3_column_int(pStmt, 0);
9632b15f6e1Sdrh       }
9642b15f6e1Sdrh       sqlite3_finalize(pStmt);
9652b15f6e1Sdrh     }
9662b15f6e1Sdrh     pStmt = 0;
96716d62901Sdrh     if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);
968cac028baSdrh   }
9692b15f6e1Sdrh   if( iMax<=0 ) iMax = 1000;
970cac028baSdrh   for(i=0; i<nCount; i++){
9711346fdadSdrh     if( eType==PATH_DIR || eType==PATH_TREE ){
9721346fdadSdrh       /* CASE 1: Reading or writing blobs out of separate files */
973cac028baSdrh       char *zKey;
9741346fdadSdrh       if( eType==PATH_DIR ){
975cac028baSdrh         zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
9761346fdadSdrh       }else{
9771346fdadSdrh         zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
9781346fdadSdrh                                (iKey/100)%100, iKey%100);
9791346fdadSdrh       }
980cac028baSdrh       nData = 0;
981a2e71cedSdrh       if( isUpdateTest ){
982f1518907Sdrh         updateFile(zKey, &nData, doFsync);
983a2e71cedSdrh       }else{
984cac028baSdrh         pData = readFile(zKey, &nData);
985cac028baSdrh         sqlite3_free(pData);
986a2e71cedSdrh       }
987a2e71cedSdrh       sqlite3_free(zKey);
988cac028baSdrh     }else if( bBlobApi ){
989cac028baSdrh       /* CASE 2: Reading from database using the incremental BLOB I/O API */
990cac028baSdrh       if( pBlob==0 ){
991a2e71cedSdrh         rc = sqlite3_blob_open(db, "main", "kv", "v", iKey,
992a2e71cedSdrh                                isUpdateTest, &pBlob);
993cac028baSdrh         if( rc ){
994cac028baSdrh           fatalError("could not open sqlite3_blob handle: %s",
995cac028baSdrh                      sqlite3_errmsg(db));
996cac028baSdrh         }
997cac028baSdrh       }else{
998cac028baSdrh         rc = sqlite3_blob_reopen(pBlob, iKey);
999cac028baSdrh       }
1000cac028baSdrh       if( rc==SQLITE_OK ){
1001cac028baSdrh         nData = sqlite3_blob_bytes(pBlob);
1002184d902dSdrh         if( nAlloc<nData+1 ){
1003184d902dSdrh           nAlloc = nData+100;
1004cf1393dfSdrh           pData = sqlite3_realloc64(pData, nAlloc);
1005184d902dSdrh         }
1006cac028baSdrh         if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
1007a2e71cedSdrh         if( isUpdateTest ){
1008a2e71cedSdrh           sqlite3_randomness((int)nData, pData);
1009cf1393dfSdrh           rc = sqlite3_blob_write(pBlob, pData, (int)nData, 0);
1010a2e71cedSdrh           if( rc!=SQLITE_OK ){
1011a2e71cedSdrh             fatalError("could not write the blob at %d: %s", iKey,
1012a2e71cedSdrh                       sqlite3_errmsg(db));
1013a2e71cedSdrh           }
1014a2e71cedSdrh         }else{
1015cf1393dfSdrh           rc = sqlite3_blob_read(pBlob, pData, (int)nData, 0);
1016cac028baSdrh           if( rc!=SQLITE_OK ){
1017cac028baSdrh             fatalError("could not read the blob at %d: %s", iKey,
1018cac028baSdrh                       sqlite3_errmsg(db));
1019cac028baSdrh           }
1020cac028baSdrh         }
1021a2e71cedSdrh       }
1022cac028baSdrh     }else{
1023cac028baSdrh       /* CASE 3: Reading from database using SQL */
1024cac028baSdrh       if( pStmt==0 ){
1025a2e71cedSdrh         if( isUpdateTest ){
1026a2e71cedSdrh           sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,
1027a2e71cedSdrh                                   rememberFunc, 0, 0);
1028a2e71cedSdrh 
1029a2e71cedSdrh           rc = sqlite3_prepare_v2(db,
1030a2e71cedSdrh             "UPDATE kv SET v=randomblob(remember(length(v),?2))"
1031a2e71cedSdrh             " WHERE k=?1", -1, &pStmt, 0);
1032a2e71cedSdrh           sqlite3_bind_int64(pStmt, 2, SQLITE_PTR_TO_INT(&nData));
1033a2e71cedSdrh         }else{
1034cac028baSdrh           rc = sqlite3_prepare_v2(db,
1035cac028baSdrh                  "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0);
1036a2e71cedSdrh         }
1037cac028baSdrh         if( rc ){
1038cac028baSdrh           fatalError("cannot prepare query: %s", sqlite3_errmsg(db));
1039cac028baSdrh         }
1040cac028baSdrh       }else{
1041cac028baSdrh         sqlite3_reset(pStmt);
1042cac028baSdrh       }
1043cac028baSdrh       sqlite3_bind_int(pStmt, 1, iKey);
1044a2e71cedSdrh       nData = 0;
1045cac028baSdrh       rc = sqlite3_step(pStmt);
1046cac028baSdrh       if( rc==SQLITE_ROW ){
1047cac028baSdrh         nData = sqlite3_column_bytes(pStmt, 0);
1048cac028baSdrh         pData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
1049cac028baSdrh       }
1050cac028baSdrh     }
1051cac028baSdrh     if( eOrder==ORDER_ASC ){
1052cac028baSdrh       iKey++;
1053cac028baSdrh       if( iKey>iMax ) iKey = 1;
1054cac028baSdrh     }else if( eOrder==ORDER_DESC ){
1055cac028baSdrh       iKey--;
1056cac028baSdrh       if( iKey<=0 ) iKey = iMax;
1057cac028baSdrh     }else{
1058cac028baSdrh       iKey = (randInt()%iMax)+1;
1059cac028baSdrh     }
1060cac028baSdrh     nTotal += nData;
1061cac028baSdrh     if( nData==0 ){ nCount++; nExtra++; }
1062cac028baSdrh   }
1063184d902dSdrh   if( nAlloc ) sqlite3_free(pData);
1064cac028baSdrh   if( pStmt ) sqlite3_finalize(pStmt);
1065cac028baSdrh   if( pBlob ) sqlite3_blob_close(pBlob);
1066befcd8adSdan   if( bStats ){
1067befcd8adSdan     display_stats(db, 0);
1068befcd8adSdan   }
1069a2e71cedSdrh   if( db ){
107016d62901Sdrh     if( !doMultiTrans ) sqlite3_exec(db, "COMMIT", 0, 0, 0);
107116d62901Sdrh     if( !noCheckpoint ){
1072a2e71cedSdrh       sqlite3_close(db);
107316d62901Sdrh       db = 0;
107416d62901Sdrh     }
1075a2e71cedSdrh   }
1076cac028baSdrh   tmElapsed = timeOfDay() - tmStart;
107716d62901Sdrh   if( db && noCheckpoint ){
107816d62901Sdrh     sqlite3_close(db);
107916d62901Sdrh     db = 0;
108016d62901Sdrh   }
1081cac028baSdrh   if( nExtra ){
1082cac028baSdrh     printf("%d cycles due to %d misses\n", nCount, nExtra);
1083cac028baSdrh   }
108461c565f6Sdrh   if( eType==PATH_DB ){
108561c565f6Sdrh     printf("SQLite version: %s\n", sqlite3_libversion());
1086a2e71cedSdrh     if( doIntegrityCk ){
1087a2e71cedSdrh       sqlite3_open(zDb, &db);
1088a2e71cedSdrh       sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);
1089a2e71cedSdrh       while( sqlite3_step(pStmt)==SQLITE_ROW ){
1090a2e71cedSdrh         printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
1091a2e71cedSdrh       }
1092a2e71cedSdrh       sqlite3_finalize(pStmt);
1093a2e71cedSdrh       sqlite3_close(db);
10945930f663Sdrh       db = 0;
1095a2e71cedSdrh     }
109661c565f6Sdrh   }
10976739c692Sdrh   printf("--count %d --max-id %d", nCount-nExtra, iMax);
109861c565f6Sdrh   switch( eOrder ){
109961c565f6Sdrh     case ORDER_RANDOM:  printf(" --random\n");  break;
110061c565f6Sdrh     case ORDER_DESC:    printf(" --desc\n");    break;
110161c565f6Sdrh     default:            printf(" --asc\n");     break;
110261c565f6Sdrh   }
11032b15f6e1Sdrh   if( eType==PATH_DB ){
11042b15f6e1Sdrh     printf("--cache-size %d --jmode %s\n", iCache, zJMode);
11052b15f6e1Sdrh     printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : "");
1106a2e71cedSdrh     if( noSync ) printf("--nosync\n");
11072b15f6e1Sdrh   }
110861c565f6Sdrh   if( iPagesize ) printf("Database page size: %d\n", iPagesize);
1109cac028baSdrh   printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
1110a2e71cedSdrh   if( isUpdateTest ){
1111a2e71cedSdrh     printf("Microseconds per BLOB write: %.3f\n", tmElapsed*1000.0/nCount);
1112a2e71cedSdrh     printf("Content write rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
1113a2e71cedSdrh   }else{
1114cac028baSdrh     printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
1115cac028baSdrh     printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
1116a2e71cedSdrh   }
1117cac028baSdrh   return 0;
1118cac028baSdrh }
1119cac028baSdrh 
1120cac028baSdrh 
main(int argc,char ** argv)1121cac028baSdrh int main(int argc, char **argv){
1122cac028baSdrh   if( argc<3 ) showHelp();
1123cac028baSdrh   if( strcmp(argv[1],"init")==0 ){
1124cac028baSdrh     return initMain(argc, argv);
1125cac028baSdrh   }
1126cac028baSdrh   if( strcmp(argv[1],"export")==0 ){
1127cac028baSdrh     return exportMain(argc, argv);
1128cac028baSdrh   }
1129cac028baSdrh   if( strcmp(argv[1],"run")==0 ){
1130cac028baSdrh     return runMain(argc, argv);
1131cac028baSdrh   }
11329dc7ad1dSdrh   if( strcmp(argv[1],"stat")==0 ){
11339dc7ad1dSdrh     return statMain(argc, argv);
11349dc7ad1dSdrh   }
1135cac028baSdrh   showHelp();
1136cac028baSdrh   return 0;
1137cac028baSdrh }
1138