xref: /sqlite-3.40.0/tool/sqldiff.c (revision a1250000)
1 /*
2 ** 2015-04-06
3 **
4 ** The author disclaims copyright to this source code.  In place of
5 ** a legal notice, here is a blessing:
6 **
7 **    May you do good and not evil.
8 **    May you find forgiveness for yourself and forgive others.
9 **    May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 **
13 ** This is a utility program that computes the differences in content
14 ** between two SQLite databases.
15 **
16 ** To compile, simply link against SQLite.
17 **
18 ** See the showHelp() routine below for a brief description of how to
19 ** run the utility.
20 */
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <stdarg.h>
24 #include <ctype.h>
25 #include <string.h>
26 #include <assert.h>
27 #include "sqlite3.h"
28 
29 /*
30 ** All global variables are gathered into the "g" singleton.
31 */
32 struct GlobalVars {
33   const char *zArgv0;       /* Name of program */
34   int bSchemaOnly;          /* Only show schema differences */
35   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
36   int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
37   unsigned fDebug;          /* Debug flags */
38   int bSchemaCompare;       /* Doing single-table sqlite_schema compare */
39   sqlite3 *db;              /* The database connection */
40 } g;
41 
42 /*
43 ** Allowed values for g.fDebug
44 */
45 #define DEBUG_COLUMN_NAMES  0x000001
46 #define DEBUG_DIFF_SQL      0x000002
47 
48 /*
49 ** Dynamic string object
50 */
51 typedef struct Str Str;
52 struct Str {
53   char *z;        /* Text of the string */
54   int nAlloc;     /* Bytes allocated in z[] */
55   int nUsed;      /* Bytes actually used in z[] */
56 };
57 
58 /*
59 ** Initialize a Str object
60 */
strInit(Str * p)61 static void strInit(Str *p){
62   p->z = 0;
63   p->nAlloc = 0;
64   p->nUsed = 0;
65 }
66 
67 /*
68 ** Print an error resulting from faulting command-line arguments and
69 ** abort the program.
70 */
cmdlineError(const char * zFormat,...)71 static void cmdlineError(const char *zFormat, ...){
72   va_list ap;
73   fprintf(stderr, "%s: ", g.zArgv0);
74   va_start(ap, zFormat);
75   vfprintf(stderr, zFormat, ap);
76   va_end(ap);
77   fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
78   exit(1);
79 }
80 
81 /*
82 ** Print an error message for an error that occurs at runtime, then
83 ** abort the program.
84 */
runtimeError(const char * zFormat,...)85 static void runtimeError(const char *zFormat, ...){
86   va_list ap;
87   fprintf(stderr, "%s: ", g.zArgv0);
88   va_start(ap, zFormat);
89   vfprintf(stderr, zFormat, ap);
90   va_end(ap);
91   fprintf(stderr, "\n");
92   exit(1);
93 }
94 
95 /*
96 ** Free all memory held by a Str object
97 */
strFree(Str * p)98 static void strFree(Str *p){
99   sqlite3_free(p->z);
100   strInit(p);
101 }
102 
103 /*
104 ** Add formatted text to the end of a Str object
105 */
strPrintf(Str * p,const char * zFormat,...)106 static void strPrintf(Str *p, const char *zFormat, ...){
107   int nNew;
108   for(;;){
109     if( p->z ){
110       va_list ap;
111       va_start(ap, zFormat);
112       sqlite3_vsnprintf(p->nAlloc-p->nUsed, p->z+p->nUsed, zFormat, ap);
113       va_end(ap);
114       nNew = (int)strlen(p->z + p->nUsed);
115     }else{
116       nNew = p->nAlloc;
117     }
118     if( p->nUsed+nNew < p->nAlloc-1 ){
119       p->nUsed += nNew;
120       break;
121     }
122     p->nAlloc = p->nAlloc*2 + 1000;
123     p->z = sqlite3_realloc(p->z, p->nAlloc);
124     if( p->z==0 ) runtimeError("out of memory");
125   }
126 }
127 
128 
129 
130 /* Safely quote an SQL identifier.  Use the minimum amount of transformation
131 ** necessary to allow the string to be used with %s.
132 **
133 ** Space to hold the returned string is obtained from sqlite3_malloc().  The
134 ** caller is responsible for ensuring this space is freed when no longer
135 ** needed.
136 */
safeId(const char * zId)137 static char *safeId(const char *zId){
138   int i, x;
139   char c;
140   if( zId[0]==0 ) return sqlite3_mprintf("\"\"");
141   for(i=x=0; (c = zId[i])!=0; i++){
142     if( !isalpha(c) && c!='_' ){
143       if( i>0 && isdigit(c) ){
144         x++;
145       }else{
146         return sqlite3_mprintf("\"%w\"", zId);
147       }
148     }
149   }
150   if( x || !sqlite3_keyword_check(zId,i) ){
151     return sqlite3_mprintf("%s", zId);
152   }
153   return sqlite3_mprintf("\"%w\"", zId);
154 }
155 
156 /*
157 ** Prepare a new SQL statement.  Print an error and abort if anything
158 ** goes wrong.
159 */
db_vprepare(const char * zFormat,va_list ap)160 static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
161   char *zSql;
162   int rc;
163   sqlite3_stmt *pStmt;
164 
165   zSql = sqlite3_vmprintf(zFormat, ap);
166   if( zSql==0 ) runtimeError("out of memory");
167   rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
168   if( rc ){
169     runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
170                  zSql);
171   }
172   sqlite3_free(zSql);
173   return pStmt;
174 }
db_prepare(const char * zFormat,...)175 static sqlite3_stmt *db_prepare(const char *zFormat, ...){
176   va_list ap;
177   sqlite3_stmt *pStmt;
178   va_start(ap, zFormat);
179   pStmt = db_vprepare(zFormat, ap);
180   va_end(ap);
181   return pStmt;
182 }
183 
184 /*
185 ** Free a list of strings
186 */
namelistFree(char ** az)187 static void namelistFree(char **az){
188   if( az ){
189     int i;
190     for(i=0; az[i]; i++) sqlite3_free(az[i]);
191     sqlite3_free(az);
192   }
193 }
194 
195 /*
196 ** Return a list of column names [a] for the table zDb.zTab.  Space to
197 ** hold the list is obtained from sqlite3_malloc() and should released
198 ** using namelistFree() when no longer needed.
199 **
200 ** Primary key columns are listed first, followed by data columns.
201 ** The number of columns in the primary key is returned in *pnPkey.
202 **
203 ** Normally [a], the "primary key" in the previous sentence is the true
204 ** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables
205 ** or the declared PRIMARY KEY for WITHOUT ROWID tables.  However, if
206 ** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is
207 ** used in all cases.  In that case, entries that have NULL values in
208 ** any of their primary key fields will be excluded from the analysis.
209 **
210 ** If the primary key for a table is the rowid but rowid is inaccessible,
211 ** then this routine returns a NULL pointer.
212 **
213 ** [a. If the lone, named table is "sqlite_schema", "rootpage" column is
214 **  omitted and the "type" and "name" columns are made to be the PK.]
215 **
216 ** Examples:
217 **    CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c));
218 **    *pnPKey = 1;
219 **    az = { "rowid", "a", "b", "c", 0 }  // Normal case
220 **    az = { "c", "a", "b", 0 }           // g.bSchemaPK==1
221 **
222 **    CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b));
223 **    *pnPKey = 1;
224 **    az = { "b", "a", "c", 0 }
225 **
226 **    CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z));
227 **    *pnPKey = 1                         // Normal case
228 **    az = { "rowid", "x", "y", "z", 0 }  // Normal case
229 **    *pnPKey = 2                         // g.bSchemaPK==1
230 **    az = { "y", "x", "z", 0 }           // g.bSchemaPK==1
231 **
232 **    CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID;
233 **    *pnPKey = 2
234 **    az = { "y", "z", "x", 0 }
235 **
236 **    CREATE TABLE t5(rowid,_rowid_,oid);
237 **    az = 0     // The rowid is not accessible
238 */
columnNames(const char * zDb,const char * zTab,int * pnPKey,int * pbRowid)239 static char **columnNames(
240   const char *zDb,                /* Database ("main" or "aux") to query */
241   const char *zTab,               /* Name of table to return details of */
242   int *pnPKey,                    /* OUT: Number of PK columns */
243   int *pbRowid                    /* OUT: True if PK is an implicit rowid */
244 ){
245   char **az = 0;           /* List of column names to be returned */
246   int naz = 0;             /* Number of entries in az[] */
247   sqlite3_stmt *pStmt;     /* SQL statement being run */
248   char *zPkIdxName = 0;    /* Name of the PRIMARY KEY index */
249   int truePk = 0;          /* PRAGMA table_info indentifies the PK to use */
250   int nPK = 0;             /* Number of PRIMARY KEY columns */
251   int i, j;                /* Loop counters */
252 
253   if( g.bSchemaPK==0 ){
254     /* Normal case:  Figure out what the true primary key is for the table.
255     **   *  For WITHOUT ROWID tables, the true primary key is the same as
256     **      the schema PRIMARY KEY, which is guaranteed to be present.
257     **   *  For rowid tables with an INTEGER PRIMARY KEY, the true primary
258     **      key is the INTEGER PRIMARY KEY.
259     **   *  For all other rowid tables, the rowid is the true primary key.
260     */
261     pStmt = db_prepare("PRAGMA %s.index_list=%Q", zDb, zTab);
262     while( SQLITE_ROW==sqlite3_step(pStmt) ){
263       if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,3),"pk")==0 ){
264         zPkIdxName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
265         break;
266       }
267     }
268     sqlite3_finalize(pStmt);
269     if( zPkIdxName ){
270       int nKey = 0;
271       int nCol = 0;
272       truePk = 0;
273       pStmt = db_prepare("PRAGMA %s.index_xinfo=%Q", zDb, zPkIdxName);
274       while( SQLITE_ROW==sqlite3_step(pStmt) ){
275         nCol++;
276         if( sqlite3_column_int(pStmt,5) ){ nKey++; continue; }
277         if( sqlite3_column_int(pStmt,1)>=0 ) truePk = 1;
278       }
279       if( nCol==nKey ) truePk = 1;
280       if( truePk ){
281         nPK = nKey;
282       }else{
283         nPK = 1;
284       }
285       sqlite3_finalize(pStmt);
286       sqlite3_free(zPkIdxName);
287     }else{
288       truePk = 1;
289       nPK = 1;
290     }
291     pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab);
292   }else{
293     /* The g.bSchemaPK==1 case:  Use whatever primary key is declared
294     ** in the schema.  The "rowid" will still be used as the primary key
295     ** if the table definition does not contain a PRIMARY KEY.
296     */
297     nPK = 0;
298     pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab);
299     while( SQLITE_ROW==sqlite3_step(pStmt) ){
300       if( sqlite3_column_int(pStmt,5)>0 ) nPK++;
301     }
302     sqlite3_reset(pStmt);
303     if( nPK==0 ) nPK = 1;
304     truePk = 1;
305   }
306   if( g.bSchemaCompare ){
307     assert( sqlite3_stricmp(zTab,"sqlite_schema")==0
308             || sqlite3_stricmp(zTab,"sqlite_master")==0 );
309     /* For sqlite_schema, will use type and name as the PK. */
310     nPK = 2;
311     truePk = 0;
312   }
313   *pnPKey = nPK;
314   naz = nPK;
315   az = sqlite3_malloc( sizeof(char*)*(nPK+1) );
316   if( az==0 ) runtimeError("out of memory");
317   memset(az, 0, sizeof(char*)*(nPK+1));
318   if( g.bSchemaCompare ){
319     az[0] = sqlite3_mprintf("%s", "type");
320     az[1] = sqlite3_mprintf("%s", "name");
321   }
322   while( SQLITE_ROW==sqlite3_step(pStmt) ){
323     char * sid = safeId((char*)sqlite3_column_text(pStmt,1));
324     int iPKey;
325     if( truePk && (iPKey = sqlite3_column_int(pStmt,5))>0 ){
326       az[iPKey-1] = sid;
327     }else{
328       if( !g.bSchemaCompare
329           || !(strcmp(sid,"rootpage")==0
330                ||strcmp(sid,"name")==0
331                ||strcmp(sid,"type")==0)){
332         az = sqlite3_realloc(az, sizeof(char*)*(naz+2) );
333         if( az==0 ) runtimeError("out of memory");
334         az[naz++] = sid;
335       }
336     }
337   }
338   sqlite3_finalize(pStmt);
339   if( az ) az[naz] = 0;
340 
341   /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of
342   ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0).  */
343   if( pbRowid ) *pbRowid = (az[0]==0);
344 
345   /* If this table has an implicit rowid for a PK, figure out how to refer
346   ** to it. There are usually three options - "rowid", "_rowid_" and "oid".
347   ** Any of these will work, unless the table has an explicit column of the
348   ** same name or the sqlite_schema tables are to be compared. In the latter
349   ** case, pretend that the "true" primary key is the name column, which
350   ** avoids extraneous diffs against the schemas due to rowid variance. */
351   if( az[0]==0 ){
352     const char *azRowid[] = { "rowid", "_rowid_", "oid" };
353     for(i=0; i<sizeof(azRowid)/sizeof(azRowid[0]); i++){
354       for(j=1; j<naz; j++){
355         if( sqlite3_stricmp(az[j], azRowid[i])==0 ) break;
356       }
357       if( j>=naz ){
358         az[0] = sqlite3_mprintf("%s", azRowid[i]);
359         break;
360       }
361     }
362     if( az[0]==0 ){
363       for(i=1; i<naz; i++) sqlite3_free(az[i]);
364       sqlite3_free(az);
365       az = 0;
366     }
367   }
368   return az;
369 }
370 
371 /*
372 ** Print the sqlite3_value X as an SQL literal.
373 */
printQuoted(FILE * out,sqlite3_value * X)374 static void printQuoted(FILE *out, sqlite3_value *X){
375   switch( sqlite3_value_type(X) ){
376     case SQLITE_FLOAT: {
377       double r1;
378       char zBuf[50];
379       r1 = sqlite3_value_double(X);
380       sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1);
381       fprintf(out, "%s", zBuf);
382       break;
383     }
384     case SQLITE_INTEGER: {
385       fprintf(out, "%lld", sqlite3_value_int64(X));
386       break;
387     }
388     case SQLITE_BLOB: {
389       const unsigned char *zBlob = sqlite3_value_blob(X);
390       int nBlob = sqlite3_value_bytes(X);
391       if( zBlob ){
392         int i;
393         fprintf(out, "x'");
394         for(i=0; i<nBlob; i++){
395           fprintf(out, "%02x", zBlob[i]);
396         }
397         fprintf(out, "'");
398       }else{
399         /* Could be an OOM, could be a zero-byte blob */
400         fprintf(out, "X''");
401       }
402       break;
403     }
404     case SQLITE_TEXT: {
405       const unsigned char *zArg = sqlite3_value_text(X);
406 
407       if( zArg==0 ){
408         fprintf(out, "NULL");
409       }else{
410         int inctl = 0;
411         int i, j;
412         fprintf(out, "'");
413         for(i=j=0; zArg[i]; i++){
414           char c = zArg[i];
415           int ctl = iscntrl(c);
416           if( ctl>inctl ){
417             inctl = ctl;
418             fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c);
419             j = i+1;
420           }else if( ctl ){
421             fprintf(out, "%02x", c);
422             j = i+1;
423           }else{
424             if( inctl ){
425               inctl = 0;
426               fprintf(out, "'\n||'");
427             }
428             if( c=='\'' ){
429               fprintf(out, "%.*s'", i-j+1, &zArg[j]);
430               j = i+1;
431             }
432           }
433         }
434         fprintf(out, "%s'", &zArg[j]);
435       }
436       break;
437     }
438     case SQLITE_NULL: {
439       fprintf(out, "NULL");
440       break;
441     }
442   }
443 }
444 
445 /*
446 ** Output SQL that will recreate the aux.zTab table.
447 */
dump_table(const char * zTab,FILE * out)448 static void dump_table(const char *zTab, FILE *out){
449   char *zId = safeId(zTab); /* Name of the table */
450   char **az = 0;            /* List of columns */
451   int nPk;                  /* Number of true primary key columns */
452   int nCol;                 /* Number of data columns */
453   int i;                    /* Loop counter */
454   sqlite3_stmt *pStmt;      /* SQL statement */
455   const char *zSep;         /* Separator string */
456   Str ins;                  /* Beginning of the INSERT statement */
457 
458   pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab);
459   if( SQLITE_ROW==sqlite3_step(pStmt) ){
460     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
461   }
462   sqlite3_finalize(pStmt);
463   if( !g.bSchemaOnly ){
464     az = columnNames("aux", zTab, &nPk, 0);
465     strInit(&ins);
466     if( az==0 ){
467       pStmt = db_prepare("SELECT * FROM aux.%s", zId);
468       strPrintf(&ins,"INSERT INTO %s VALUES", zId);
469     }else{
470       Str sql;
471       strInit(&sql);
472       zSep =  "SELECT";
473       for(i=0; az[i]; i++){
474         strPrintf(&sql, "%s %s", zSep, az[i]);
475         zSep = ",";
476       }
477       strPrintf(&sql," FROM aux.%s", zId);
478       zSep = " ORDER BY";
479       for(i=1; i<=nPk; i++){
480         strPrintf(&sql, "%s %d", zSep, i);
481         zSep = ",";
482       }
483       pStmt = db_prepare("%s", sql.z);
484       strFree(&sql);
485       strPrintf(&ins, "INSERT INTO %s", zId);
486       zSep = "(";
487       for(i=0; az[i]; i++){
488         strPrintf(&ins, "%s%s", zSep, az[i]);
489         zSep = ",";
490       }
491       strPrintf(&ins,") VALUES");
492       namelistFree(az);
493     }
494     nCol = sqlite3_column_count(pStmt);
495     while( SQLITE_ROW==sqlite3_step(pStmt) ){
496       fprintf(out, "%s",ins.z);
497       zSep = "(";
498       for(i=0; i<nCol; i++){
499         fprintf(out, "%s",zSep);
500         printQuoted(out, sqlite3_column_value(pStmt,i));
501         zSep = ",";
502       }
503       fprintf(out, ");\n");
504     }
505     sqlite3_finalize(pStmt);
506     strFree(&ins);
507   } /* endif !g.bSchemaOnly */
508   pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema"
509                      " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
510                      zTab);
511   while( SQLITE_ROW==sqlite3_step(pStmt) ){
512     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
513   }
514   sqlite3_finalize(pStmt);
515   sqlite3_free(zId);
516 }
517 
518 
519 /*
520 ** Compute all differences for a single table, except if the
521 ** table name is sqlite_schema, ignore the rootpage column.
522 */
diff_one_table(const char * zTab,FILE * out)523 static void diff_one_table(const char *zTab, FILE *out){
524   char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */
525   char **az = 0;            /* Columns in main */
526   char **az2 = 0;           /* Columns in aux */
527   int nPk;                  /* Primary key columns in main */
528   int nPk2;                 /* Primary key columns in aux */
529   int n = 0;                /* Number of columns in main */
530   int n2;                   /* Number of columns in aux */
531   int nQ;                   /* Number of output columns in the diff query */
532   int i;                    /* Loop counter */
533   const char *zSep;         /* Separator string */
534   Str sql;                  /* Comparison query */
535   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
536   const char *zLead =       /* Becomes line-comment for sqlite_schema */
537     (g.bSchemaCompare)? "-- " : "";
538 
539   strInit(&sql);
540   if( g.fDebug==DEBUG_COLUMN_NAMES ){
541     /* Simply run columnNames() on all tables of the origin
542     ** database and show the results.  This is used for testing
543     ** and debugging of the columnNames() function.
544     */
545     az = columnNames("aux",zTab, &nPk, 0);
546     if( az==0 ){
547       printf("Rowid not accessible for %s\n", zId);
548     }else{
549       printf("%s:", zId);
550       for(i=0; az[i]; i++){
551         printf(" %s", az[i]);
552         if( i+1==nPk ) printf(" *");
553       }
554       printf("\n");
555     }
556     goto end_diff_one_table;
557   }
558 
559   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
560     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
561       /* Table missing from second database. */
562       if( g.bSchemaCompare )
563         fprintf(out, "-- 2nd DB has no %s table\n", zTab);
564       else
565         fprintf(out, "DROP TABLE %s;\n", zId);
566     }
567     goto end_diff_one_table;
568   }
569 
570   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
571     /* Table missing from source */
572     if( g.bSchemaCompare )
573       fprintf(out, "-- 1st DB has no %s table\n", zTab);
574     else
575       dump_table(zTab, out);
576     goto end_diff_one_table;
577   }
578 
579   az = columnNames("main", zTab, &nPk, 0);
580   az2 = columnNames("aux", zTab, &nPk2, 0);
581   if( az && az2 ){
582     for(n=0; az[n] && az2[n]; n++){
583       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
584     }
585   }
586   if( az==0
587    || az2==0
588    || nPk!=nPk2
589    || az[n]
590   ){
591     /* Schema mismatch */
592     fprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId);
593     dump_table(zTab, out);
594     goto end_diff_one_table;
595   }
596 
597   /* Build the comparison query */
598   for(n2=n; az2[n2]; n2++){
599     fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, safeId(az2[n2]));
600   }
601   nQ = nPk2+1+2*(n2-nPk2);
602   if( n2>nPk2 ){
603     zSep = "SELECT ";
604     for(i=0; i<nPk; i++){
605       strPrintf(&sql, "%sB.%s", zSep, az[i]);
606       zSep = ", ";
607     }
608     strPrintf(&sql, ", 1%s -- changed row\n", nPk==n ? "" : ",");
609     while( az[i] ){
610       strPrintf(&sql, "       A.%s IS NOT B.%s, B.%s%s\n",
611                 az[i], az2[i], az2[i], az2[i+1]==0 ? "" : ",");
612       i++;
613     }
614     while( az2[i] ){
615       strPrintf(&sql, "       B.%s IS NOT NULL, B.%s%s\n",
616                 az2[i], az2[i], az2[i+1]==0 ? "" : ",");
617       i++;
618     }
619     strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
620     zSep = " WHERE";
621     for(i=0; i<nPk; i++){
622       strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
623       zSep = " AND";
624     }
625     zSep = "\n   AND (";
626     while( az[i] ){
627       strPrintf(&sql, "%sA.%s IS NOT B.%s%s\n",
628                 zSep, az[i], az2[i], az2[i+1]==0 ? ")" : "");
629       zSep = "        OR ";
630       i++;
631     }
632     while( az2[i] ){
633       strPrintf(&sql, "%sB.%s IS NOT NULL%s\n",
634                 zSep, az2[i], az2[i+1]==0 ? ")" : "");
635       zSep = "        OR ";
636       i++;
637     }
638     strPrintf(&sql, " UNION ALL\n");
639   }
640   zSep = "SELECT ";
641   for(i=0; i<nPk; i++){
642     strPrintf(&sql, "%sA.%s", zSep, az[i]);
643     zSep = ", ";
644   }
645   strPrintf(&sql, ", 2%s -- deleted row\n", nPk==n ? "" : ",");
646   while( az2[i] ){
647     strPrintf(&sql, "       NULL, NULL%s\n", i==n2-1 ? "" : ",");
648     i++;
649   }
650   strPrintf(&sql, "  FROM main.%s A\n", zId);
651   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
652   zSep =          "                   WHERE";
653   for(i=0; i<nPk; i++){
654     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
655     zSep = " AND";
656   }
657   strPrintf(&sql, ")\n");
658   zSep = " UNION ALL\nSELECT ";
659   for(i=0; i<nPk; i++){
660     strPrintf(&sql, "%sB.%s", zSep, az[i]);
661     zSep = ", ";
662   }
663   strPrintf(&sql, ", 3%s -- inserted row\n", nPk==n ? "" : ",");
664   while( az2[i] ){
665     strPrintf(&sql, "       1, B.%s%s\n", az2[i], az2[i+1]==0 ? "" : ",");
666     i++;
667   }
668   strPrintf(&sql, "  FROM aux.%s B\n", zId);
669   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
670   zSep =          "                   WHERE";
671   for(i=0; i<nPk; i++){
672     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
673     zSep = " AND";
674   }
675   strPrintf(&sql, ")\n ORDER BY");
676   zSep = " ";
677   for(i=1; i<=nPk; i++){
678     strPrintf(&sql, "%s%d", zSep, i);
679     zSep = ", ";
680   }
681   strPrintf(&sql, ";\n");
682 
683   if( g.fDebug & DEBUG_DIFF_SQL ){
684     printf("SQL for %s:\n%s\n", zId, sql.z);
685     goto end_diff_one_table;
686   }
687 
688   /* Drop indexes that are missing in the destination */
689   pStmt = db_prepare(
690     "SELECT name FROM main.sqlite_schema"
691     " WHERE type='index' AND tbl_name=%Q"
692     "   AND sql IS NOT NULL"
693     "   AND sql NOT IN (SELECT sql FROM aux.sqlite_schema"
694     "                    WHERE type='index' AND tbl_name=%Q"
695     "                      AND sql IS NOT NULL)",
696     zTab, zTab);
697   while( SQLITE_ROW==sqlite3_step(pStmt) ){
698     char *z = safeId((const char*)sqlite3_column_text(pStmt,0));
699     fprintf(out, "DROP INDEX %s;\n", z);
700     sqlite3_free(z);
701   }
702   sqlite3_finalize(pStmt);
703 
704   /* Run the query and output differences */
705   if( !g.bSchemaOnly ){
706     pStmt = db_prepare("%s", sql.z);
707     while( SQLITE_ROW==sqlite3_step(pStmt) ){
708       int iType = sqlite3_column_int(pStmt, nPk);
709       if( iType==1 || iType==2 ){
710         if( iType==1 ){       /* Change the content of a row */
711           fprintf(out, "%sUPDATE %s", zLead, zId);
712           zSep = " SET";
713           for(i=nPk+1; i<nQ; i+=2){
714             if( sqlite3_column_int(pStmt,i)==0 ) continue;
715             fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
716             zSep = ",";
717             printQuoted(out, sqlite3_column_value(pStmt,i+1));
718           }
719         }else{                /* Delete a row */
720           fprintf(out, "%sDELETE FROM %s", zLead, zId);
721         }
722         zSep = " WHERE";
723         for(i=0; i<nPk; i++){
724           fprintf(out, "%s %s=", zSep, az2[i]);
725           printQuoted(out, sqlite3_column_value(pStmt,i));
726           zSep = " AND";
727         }
728         fprintf(out, ";\n");
729       }else{                  /* Insert a row */
730         fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]);
731         for(i=1; az2[i]; i++) fprintf(out, ",%s", az2[i]);
732         fprintf(out, ") VALUES");
733         zSep = "(";
734         for(i=0; i<nPk2; i++){
735           fprintf(out, "%s", zSep);
736           zSep = ",";
737           printQuoted(out, sqlite3_column_value(pStmt,i));
738         }
739         for(i=nPk2+2; i<nQ; i+=2){
740           fprintf(out, ",");
741           printQuoted(out, sqlite3_column_value(pStmt,i));
742         }
743         fprintf(out, ");\n");
744       }
745     }
746     sqlite3_finalize(pStmt);
747   } /* endif !g.bSchemaOnly */
748 
749   /* Create indexes that are missing in the source */
750   pStmt = db_prepare(
751     "SELECT sql FROM aux.sqlite_schema"
752     " WHERE type='index' AND tbl_name=%Q"
753     "   AND sql IS NOT NULL"
754     "   AND sql NOT IN (SELECT sql FROM main.sqlite_schema"
755     "                    WHERE type='index' AND tbl_name=%Q"
756     "                      AND sql IS NOT NULL)",
757     zTab, zTab);
758   while( SQLITE_ROW==sqlite3_step(pStmt) ){
759     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
760   }
761   sqlite3_finalize(pStmt);
762 
763 end_diff_one_table:
764   strFree(&sql);
765   sqlite3_free(zId);
766   namelistFree(az);
767   namelistFree(az2);
768   return;
769 }
770 
771 /*
772 ** Check that table zTab exists and has the same schema in both the "main"
773 ** and "aux" databases currently opened by the global db handle. If they
774 ** do not, output an error message on stderr and exit(1). Otherwise, if
775 ** the schemas do match, return control to the caller.
776 */
checkSchemasMatch(const char * zTab)777 static void checkSchemasMatch(const char *zTab){
778   sqlite3_stmt *pStmt = db_prepare(
779       "SELECT A.sql=B.sql FROM main.sqlite_schema A, aux.sqlite_schema B"
780       " WHERE A.name=%Q AND B.name=%Q", zTab, zTab
781   );
782   if( SQLITE_ROW==sqlite3_step(pStmt) ){
783     if( sqlite3_column_int(pStmt,0)==0 ){
784       runtimeError("schema changes for table %s", safeId(zTab));
785     }
786   }else{
787     runtimeError("table %s missing from one or both databases", safeId(zTab));
788   }
789   sqlite3_finalize(pStmt);
790 }
791 
792 /**************************************************************************
793 ** The following code is copied from fossil. It is used to generate the
794 ** fossil delta blobs sometimes used in RBU update records.
795 */
796 
797 typedef unsigned short u16;
798 typedef unsigned int u32;
799 typedef unsigned char u8;
800 
801 /*
802 ** The width of a hash window in bytes.  The algorithm only works if this
803 ** is a power of 2.
804 */
805 #define NHASH 16
806 
807 /*
808 ** The current state of the rolling hash.
809 **
810 ** z[] holds the values that have been hashed.  z[] is a circular buffer.
811 ** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of
812 ** the window.
813 **
814 ** Hash.a is the sum of all elements of hash.z[].  Hash.b is a weighted
815 ** sum.  Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1.
816 ** (Each index for z[] should be module NHASH, of course.  The %NHASH operator
817 ** is omitted in the prior expression for brevity.)
818 */
819 typedef struct hash hash;
820 struct hash {
821   u16 a, b;         /* Hash values */
822   u16 i;            /* Start of the hash window */
823   char z[NHASH];    /* The values that have been hashed */
824 };
825 
826 /*
827 ** Initialize the rolling hash using the first NHASH characters of z[]
828 */
hash_init(hash * pHash,const char * z)829 static void hash_init(hash *pHash, const char *z){
830   u16 a, b, i;
831   a = b = 0;
832   for(i=0; i<NHASH; i++){
833     a += z[i];
834     b += (NHASH-i)*z[i];
835     pHash->z[i] = z[i];
836   }
837   pHash->a = a & 0xffff;
838   pHash->b = b & 0xffff;
839   pHash->i = 0;
840 }
841 
842 /*
843 ** Advance the rolling hash by a single character "c"
844 */
hash_next(hash * pHash,int c)845 static void hash_next(hash *pHash, int c){
846   u16 old = pHash->z[pHash->i];
847   pHash->z[pHash->i] = (char)c;
848   pHash->i = (pHash->i+1)&(NHASH-1);
849   pHash->a = pHash->a - old + (char)c;
850   pHash->b = pHash->b - NHASH*old + pHash->a;
851 }
852 
853 /*
854 ** Return a 32-bit hash value
855 */
hash_32bit(hash * pHash)856 static u32 hash_32bit(hash *pHash){
857   return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16);
858 }
859 
860 /*
861 ** Write an base-64 integer into the given buffer.
862 */
putInt(unsigned int v,char ** pz)863 static void putInt(unsigned int v, char **pz){
864   static const char zDigits[] =
865     "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~";
866   /*  123456789 123456789 123456789 123456789 123456789 123456789 123 */
867   int i, j;
868   char zBuf[20];
869   if( v==0 ){
870     *(*pz)++ = '0';
871     return;
872   }
873   for(i=0; v>0; i++, v>>=6){
874     zBuf[i] = zDigits[v&0x3f];
875   }
876   for(j=i-1; j>=0; j--){
877     *(*pz)++ = zBuf[j];
878   }
879 }
880 
881 /*
882 ** Return the number digits in the base-64 representation of a positive integer
883 */
digit_count(int v)884 static int digit_count(int v){
885   unsigned int i, x;
886   for(i=1, x=64; (unsigned int)v>=x; i++, x <<= 6){}
887   return i;
888 }
889 
890 /*
891 ** Compute a 32-bit checksum on the N-byte buffer.  Return the result.
892 */
checksum(const char * zIn,size_t N)893 static unsigned int checksum(const char *zIn, size_t N){
894   const unsigned char *z = (const unsigned char *)zIn;
895   unsigned sum0 = 0;
896   unsigned sum1 = 0;
897   unsigned sum2 = 0;
898   unsigned sum3 = 0;
899   while(N >= 16){
900     sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
901     sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
902     sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
903     sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
904     z += 16;
905     N -= 16;
906   }
907   while(N >= 4){
908     sum0 += z[0];
909     sum1 += z[1];
910     sum2 += z[2];
911     sum3 += z[3];
912     z += 4;
913     N -= 4;
914   }
915   sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
916   switch(N){
917     case 3:   sum3 += (z[2] << 8);
918     case 2:   sum3 += (z[1] << 16);
919     case 1:   sum3 += (z[0] << 24);
920     default:  ;
921   }
922   return sum3;
923 }
924 
925 /*
926 ** Create a new delta.
927 **
928 ** The delta is written into a preallocated buffer, zDelta, which
929 ** should be at least 60 bytes longer than the target file, zOut.
930 ** The delta string will be NUL-terminated, but it might also contain
931 ** embedded NUL characters if either the zSrc or zOut files are
932 ** binary.  This function returns the length of the delta string
933 ** in bytes, excluding the final NUL terminator character.
934 **
935 ** Output Format:
936 **
937 ** The delta begins with a base64 number followed by a newline.  This
938 ** number is the number of bytes in the TARGET file.  Thus, given a
939 ** delta file z, a program can compute the size of the output file
940 ** simply by reading the first line and decoding the base-64 number
941 ** found there.  The delta_output_size() routine does exactly this.
942 **
943 ** After the initial size number, the delta consists of a series of
944 ** literal text segments and commands to copy from the SOURCE file.
945 ** A copy command looks like this:
946 **
947 **     NNN@MMM,
948 **
949 ** where NNN is the number of bytes to be copied and MMM is the offset
950 ** into the source file of the first byte (both base-64).   If NNN is 0
951 ** it means copy the rest of the input file.  Literal text is like this:
952 **
953 **     NNN:TTTTT
954 **
955 ** where NNN is the number of bytes of text (base-64) and TTTTT is the text.
956 **
957 ** The last term is of the form
958 **
959 **     NNN;
960 **
961 ** In this case, NNN is a 32-bit bigendian checksum of the output file
962 ** that can be used to verify that the delta applied correctly.  All
963 ** numbers are in base-64.
964 **
965 ** Pure text files generate a pure text delta.  Binary files generate a
966 ** delta that may contain some binary data.
967 **
968 ** Algorithm:
969 **
970 ** The encoder first builds a hash table to help it find matching
971 ** patterns in the source file.  16-byte chunks of the source file
972 ** sampled at evenly spaced intervals are used to populate the hash
973 ** table.
974 **
975 ** Next we begin scanning the target file using a sliding 16-byte
976 ** window.  The hash of the 16-byte window in the target is used to
977 ** search for a matching section in the source file.  When a match
978 ** is found, a copy command is added to the delta.  An effort is
979 ** made to extend the matching section to regions that come before
980 ** and after the 16-byte hash window.  A copy command is only issued
981 ** if the result would use less space that just quoting the text
982 ** literally. Literal text is added to the delta for sections that
983 ** do not match or which can not be encoded efficiently using copy
984 ** commands.
985 */
rbuDeltaCreate(const char * zSrc,unsigned int lenSrc,const char * zOut,unsigned int lenOut,char * zDelta)986 static int rbuDeltaCreate(
987   const char *zSrc,      /* The source or pattern file */
988   unsigned int lenSrc,   /* Length of the source file */
989   const char *zOut,      /* The target file */
990   unsigned int lenOut,   /* Length of the target file */
991   char *zDelta           /* Write the delta into this buffer */
992 ){
993   unsigned int i, base;
994   char *zOrigDelta = zDelta;
995   hash h;
996   int nHash;                 /* Number of hash table entries */
997   int *landmark;             /* Primary hash table */
998   int *collide;              /* Collision chain */
999   int lastRead = -1;         /* Last byte of zSrc read by a COPY command */
1000 
1001   /* Add the target file size to the beginning of the delta
1002   */
1003   putInt(lenOut, &zDelta);
1004   *(zDelta++) = '\n';
1005 
1006   /* If the source file is very small, it means that we have no
1007   ** chance of ever doing a copy command.  Just output a single
1008   ** literal segment for the entire target and exit.
1009   */
1010   if( lenSrc<=NHASH ){
1011     putInt(lenOut, &zDelta);
1012     *(zDelta++) = ':';
1013     memcpy(zDelta, zOut, lenOut);
1014     zDelta += lenOut;
1015     putInt(checksum(zOut, lenOut), &zDelta);
1016     *(zDelta++) = ';';
1017     return (int)(zDelta - zOrigDelta);
1018   }
1019 
1020   /* Compute the hash table used to locate matching sections in the
1021   ** source file.
1022   */
1023   nHash = lenSrc/NHASH;
1024   collide = sqlite3_malloc( nHash*2*sizeof(int) );
1025   landmark = &collide[nHash];
1026   memset(landmark, -1, nHash*sizeof(int));
1027   memset(collide, -1, nHash*sizeof(int));
1028   for(i=0; i<lenSrc-NHASH; i+=NHASH){
1029     int hv;
1030     hash_init(&h, &zSrc[i]);
1031     hv = hash_32bit(&h) % nHash;
1032     collide[i/NHASH] = landmark[hv];
1033     landmark[hv] = i/NHASH;
1034   }
1035 
1036   /* Begin scanning the target file and generating copy commands and
1037   ** literal sections of the delta.
1038   */
1039   base = 0;    /* We have already generated everything before zOut[base] */
1040   while( base+NHASH<lenOut ){
1041     int iSrc, iBlock;
1042     int bestCnt, bestOfst=0, bestLitsz=0;
1043     hash_init(&h, &zOut[base]);
1044     i = 0;     /* Trying to match a landmark against zOut[base+i] */
1045     bestCnt = 0;
1046     while( 1 ){
1047       int hv;
1048       int limit = 250;
1049 
1050       hv = hash_32bit(&h) % nHash;
1051       iBlock = landmark[hv];
1052       while( iBlock>=0 && (limit--)>0 ){
1053         /*
1054         ** The hash window has identified a potential match against
1055         ** landmark block iBlock.  But we need to investigate further.
1056         **
1057         ** Look for a region in zOut that matches zSrc. Anchor the search
1058         ** at zSrc[iSrc] and zOut[base+i].  Do not include anything prior to
1059         ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen].
1060         **
1061         ** Set cnt equal to the length of the match and set ofst so that
1062         ** zSrc[ofst] is the first element of the match.  litsz is the number
1063         ** of characters between zOut[base] and the beginning of the match.
1064         ** sz will be the overhead (in bytes) needed to encode the copy
1065         ** command.  Only generate copy command if the overhead of the
1066         ** copy command is less than the amount of literal text to be copied.
1067         */
1068         int cnt, ofst, litsz;
1069         int j, k, x, y;
1070         int sz;
1071 
1072         /* Beginning at iSrc, match forwards as far as we can.  j counts
1073         ** the number of characters that match */
1074         iSrc = iBlock*NHASH;
1075         for(
1076           j=0, x=iSrc, y=base+i;
1077           (unsigned int)x<lenSrc && (unsigned int)y<lenOut;
1078           j++, x++, y++
1079         ){
1080           if( zSrc[x]!=zOut[y] ) break;
1081         }
1082         j--;
1083 
1084         /* Beginning at iSrc-1, match backwards as far as we can.  k counts
1085         ** the number of characters that match */
1086         for(k=1; k<iSrc && (unsigned int)k<=i; k++){
1087           if( zSrc[iSrc-k]!=zOut[base+i-k] ) break;
1088         }
1089         k--;
1090 
1091         /* Compute the offset and size of the matching region */
1092         ofst = iSrc-k;
1093         cnt = j+k+1;
1094         litsz = i-k;  /* Number of bytes of literal text before the copy */
1095         /* sz will hold the number of bytes needed to encode the "insert"
1096         ** command and the copy command, not counting the "insert" text */
1097         sz = digit_count(i-k)+digit_count(cnt)+digit_count(ofst)+3;
1098         if( cnt>=sz && cnt>bestCnt ){
1099           /* Remember this match only if it is the best so far and it
1100           ** does not increase the file size */
1101           bestCnt = cnt;
1102           bestOfst = iSrc-k;
1103           bestLitsz = litsz;
1104         }
1105 
1106         /* Check the next matching block */
1107         iBlock = collide[iBlock];
1108       }
1109 
1110       /* We have a copy command that does not cause the delta to be larger
1111       ** than a literal insert.  So add the copy command to the delta.
1112       */
1113       if( bestCnt>0 ){
1114         if( bestLitsz>0 ){
1115           /* Add an insert command before the copy */
1116           putInt(bestLitsz,&zDelta);
1117           *(zDelta++) = ':';
1118           memcpy(zDelta, &zOut[base], bestLitsz);
1119           zDelta += bestLitsz;
1120           base += bestLitsz;
1121         }
1122         base += bestCnt;
1123         putInt(bestCnt, &zDelta);
1124         *(zDelta++) = '@';
1125         putInt(bestOfst, &zDelta);
1126         *(zDelta++) = ',';
1127         if( bestOfst + bestCnt -1 > lastRead ){
1128           lastRead = bestOfst + bestCnt - 1;
1129         }
1130         bestCnt = 0;
1131         break;
1132       }
1133 
1134       /* If we reach this point, it means no match is found so far */
1135       if( base+i+NHASH>=lenOut ){
1136         /* We have reached the end of the file and have not found any
1137         ** matches.  Do an "insert" for everything that does not match */
1138         putInt(lenOut-base, &zDelta);
1139         *(zDelta++) = ':';
1140         memcpy(zDelta, &zOut[base], lenOut-base);
1141         zDelta += lenOut-base;
1142         base = lenOut;
1143         break;
1144       }
1145 
1146       /* Advance the hash by one character.  Keep looking for a match */
1147       hash_next(&h, zOut[base+i+NHASH]);
1148       i++;
1149     }
1150   }
1151   /* Output a final "insert" record to get all the text at the end of
1152   ** the file that does not match anything in the source file.
1153   */
1154   if( base<lenOut ){
1155     putInt(lenOut-base, &zDelta);
1156     *(zDelta++) = ':';
1157     memcpy(zDelta, &zOut[base], lenOut-base);
1158     zDelta += lenOut-base;
1159   }
1160   /* Output the final checksum record. */
1161   putInt(checksum(zOut, lenOut), &zDelta);
1162   *(zDelta++) = ';';
1163   sqlite3_free(collide);
1164   return (int)(zDelta - zOrigDelta);
1165 }
1166 
1167 /*
1168 ** End of code copied from fossil.
1169 **************************************************************************/
1170 
strPrintfArray(Str * pStr,const char * zSep,const char * zFmt,char ** az,int n)1171 static void strPrintfArray(
1172   Str *pStr,                      /* String object to append to */
1173   const char *zSep,               /* Separator string */
1174   const char *zFmt,               /* Format for each entry */
1175   char **az, int n                /* Array of strings & its size (or -1) */
1176 ){
1177   int i;
1178   for(i=0; az[i] && (i<n || n<0); i++){
1179     if( i!=0 ) strPrintf(pStr, "%s", zSep);
1180     strPrintf(pStr, zFmt, az[i], az[i], az[i]);
1181   }
1182 }
1183 
getRbudiffQuery(const char * zTab,char ** azCol,int nPK,int bOtaRowid,Str * pSql)1184 static void getRbudiffQuery(
1185   const char *zTab,
1186   char **azCol,
1187   int nPK,
1188   int bOtaRowid,
1189   Str *pSql
1190 ){
1191   int i;
1192 
1193   /* First the newly inserted rows: **/
1194   strPrintf(pSql, "SELECT ");
1195   strPrintfArray(pSql, ", ", "%s", azCol, -1);
1196   strPrintf(pSql, ", 0, ");       /* Set ota_control to 0 for an insert */
1197   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
1198   strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
1199   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
1200   strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
1201   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1202   strPrintf(pSql, "\n) AND ");
1203   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
1204 
1205   /* Deleted rows: */
1206   strPrintf(pSql, "\nUNION ALL\nSELECT ");
1207   strPrintfArray(pSql, ", ", "%s", azCol, nPK);
1208   if( azCol[nPK] ){
1209     strPrintf(pSql, ", ");
1210     strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
1211   }
1212   strPrintf(pSql, ", 1, ");       /* Set ota_control to 1 for a delete */
1213   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
1214   strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
1215   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
1216   strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
1217   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1218   strPrintf(pSql, "\n) AND ");
1219   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
1220 
1221   /* Updated rows. If all table columns are part of the primary key, there
1222   ** can be no updates. In this case this part of the compound SELECT can
1223   ** be omitted altogether. */
1224   if( azCol[nPK] ){
1225     strPrintf(pSql, "\nUNION ALL\nSELECT ");
1226     strPrintfArray(pSql, ", ", "n.%s", azCol, nPK);
1227     strPrintf(pSql, ",\n");
1228     strPrintfArray(pSql, " ,\n",
1229         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1
1230     );
1231 
1232     if( bOtaRowid==0 ){
1233       strPrintf(pSql, ", '");
1234       strPrintfArray(pSql, "", ".", azCol, nPK);
1235       strPrintf(pSql, "' ||\n");
1236     }else{
1237       strPrintf(pSql, ",\n");
1238     }
1239     strPrintfArray(pSql, " ||\n",
1240         "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
1241     );
1242     strPrintf(pSql, "\nAS ota_control, ");
1243     strPrintfArray(pSql, ", ", "NULL", azCol, nPK);
1244     strPrintf(pSql, ",\n");
1245     strPrintfArray(pSql, " ,\n",
1246         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol[nPK], -1
1247     );
1248 
1249     strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
1250     strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1251     strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
1252   }
1253 
1254   /* Now add an ORDER BY clause to sort everything by PK. */
1255   strPrintf(pSql, "\nORDER BY ");
1256   for(i=1; i<=nPK; i++) strPrintf(pSql, "%s%d", ((i>1)?", ":""), i);
1257 }
1258 
rbudiff_one_table(const char * zTab,FILE * out)1259 static void rbudiff_one_table(const char *zTab, FILE *out){
1260   int bOtaRowid;                  /* True to use an ota_rowid column */
1261   int nPK;                        /* Number of primary key columns in table */
1262   char **azCol;                   /* NULL terminated array of col names */
1263   int i;
1264   int nCol;
1265   Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
1266   Str sql = {0, 0, 0};            /* Query to find differences */
1267   Str insert = {0, 0, 0};         /* First part of output INSERT statement */
1268   sqlite3_stmt *pStmt = 0;
1269   int nRow = 0;                   /* Total rows in data_xxx table */
1270 
1271   /* --rbu mode must use real primary keys. */
1272   g.bSchemaPK = 1;
1273 
1274   /* Check that the schemas of the two tables match. Exit early otherwise. */
1275   checkSchemasMatch(zTab);
1276 
1277   /* Grab the column names and PK details for the table(s). If no usable PK
1278   ** columns are found, bail out early.  */
1279   azCol = columnNames("main", zTab, &nPK, &bOtaRowid);
1280   if( azCol==0 ){
1281     runtimeError("table %s has no usable PK columns", zTab);
1282   }
1283   for(nCol=0; azCol[nCol]; nCol++);
1284 
1285   /* Build and output the CREATE TABLE statement for the data_xxx table */
1286   strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
1287   if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
1288   strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
1289   strPrintf(&ct, ", rbu_control);");
1290 
1291   /* Get the SQL for the query to retrieve data from the two databases */
1292   getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);
1293 
1294   /* Build the first part of the INSERT statement output for each row
1295   ** in the data_xxx table. */
1296   strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
1297   if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
1298   strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
1299   strPrintf(&insert, ", rbu_control) VALUES(");
1300 
1301   pStmt = db_prepare("%s", sql.z);
1302 
1303   while( sqlite3_step(pStmt)==SQLITE_ROW ){
1304 
1305     /* If this is the first row output, print out the CREATE TABLE
1306     ** statement first. And then set ct.z to NULL so that it is not
1307     ** printed again.  */
1308     if( ct.z ){
1309       fprintf(out, "%s\n", ct.z);
1310       strFree(&ct);
1311     }
1312 
1313     /* Output the first part of the INSERT statement */
1314     fprintf(out, "%s", insert.z);
1315     nRow++;
1316 
1317     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
1318       for(i=0; i<=nCol; i++){
1319         if( i>0 ) fprintf(out, ", ");
1320         printQuoted(out, sqlite3_column_value(pStmt, i));
1321       }
1322     }else{
1323       char *zOtaControl;
1324       int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
1325 
1326       zOtaControl = (char*)sqlite3_malloc(nOtaControl+1);
1327       memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
1328 
1329       for(i=0; i<nCol; i++){
1330         int bDone = 0;
1331         if( i>=nPK
1332             && sqlite3_column_type(pStmt, i)==SQLITE_BLOB
1333             && sqlite3_column_type(pStmt, nCol+1+i)==SQLITE_BLOB
1334         ){
1335           const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i);
1336           int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i);
1337           const char *aFinal = sqlite3_column_blob(pStmt, i);
1338           int nFinal = sqlite3_column_bytes(pStmt, i);
1339           char *aDelta;
1340           int nDelta;
1341 
1342           aDelta = sqlite3_malloc(nFinal + 60);
1343           nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta);
1344           if( nDelta<nFinal ){
1345             int j;
1346             fprintf(out, "x'");
1347             for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]);
1348             fprintf(out, "'");
1349             zOtaControl[i-bOtaRowid] = 'f';
1350             bDone = 1;
1351           }
1352           sqlite3_free(aDelta);
1353         }
1354 
1355         if( bDone==0 ){
1356           printQuoted(out, sqlite3_column_value(pStmt, i));
1357         }
1358         fprintf(out, ", ");
1359       }
1360       fprintf(out, "'%s'", zOtaControl);
1361       sqlite3_free(zOtaControl);
1362     }
1363 
1364     /* And the closing bracket of the insert statement */
1365     fprintf(out, ");\n");
1366   }
1367 
1368   sqlite3_finalize(pStmt);
1369   if( nRow>0 ){
1370     Str cnt = {0, 0, 0};
1371     strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
1372     fprintf(out, "%s\n", cnt.z);
1373     strFree(&cnt);
1374   }
1375 
1376   strFree(&ct);
1377   strFree(&sql);
1378   strFree(&insert);
1379 }
1380 
1381 /*
1382 ** Display a summary of differences between two versions of the same
1383 ** table table.
1384 **
1385 **   *  Number of rows changed
1386 **   *  Number of rows added
1387 **   *  Number of rows deleted
1388 **   *  Number of identical rows
1389 */
summarize_one_table(const char * zTab,FILE * out)1390 static void summarize_one_table(const char *zTab, FILE *out){
1391   char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */
1392   char **az = 0;            /* Columns in main */
1393   char **az2 = 0;           /* Columns in aux */
1394   int nPk;                  /* Primary key columns in main */
1395   int nPk2;                 /* Primary key columns in aux */
1396   int n = 0;                /* Number of columns in main */
1397   int n2;                   /* Number of columns in aux */
1398   int i;                    /* Loop counter */
1399   const char *zSep;         /* Separator string */
1400   Str sql;                  /* Comparison query */
1401   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
1402   sqlite3_int64 nUpdate;    /* Number of updated rows */
1403   sqlite3_int64 nUnchanged; /* Number of unmodified rows */
1404   sqlite3_int64 nDelete;    /* Number of deleted rows */
1405   sqlite3_int64 nInsert;    /* Number of inserted rows */
1406 
1407   strInit(&sql);
1408   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
1409     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
1410       /* Table missing from second database. */
1411       fprintf(out, "%s: missing from second database\n", zTab);
1412     }
1413     goto end_summarize_one_table;
1414   }
1415 
1416   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
1417     /* Table missing from source */
1418     fprintf(out, "%s: missing from first database\n", zTab);
1419     goto end_summarize_one_table;
1420   }
1421 
1422   az = columnNames("main", zTab, &nPk, 0);
1423   az2 = columnNames("aux", zTab, &nPk2, 0);
1424   if( az && az2 ){
1425     for(n=0; az[n]; n++){
1426       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
1427     }
1428   }
1429   if( az==0
1430    || az2==0
1431    || nPk!=nPk2
1432    || az[n]
1433   ){
1434     /* Schema mismatch */
1435     fprintf(out, "%s: incompatible schema\n", zTab);
1436     goto end_summarize_one_table;
1437   }
1438 
1439   /* Build the comparison query */
1440   for(n2=n; az[n2]; n2++){}
1441   strPrintf(&sql, "SELECT 1, count(*)");
1442   if( n2==nPk2 ){
1443     strPrintf(&sql, ", 0\n");
1444   }else{
1445     zSep = ", sum(";
1446     for(i=nPk; az[i]; i++){
1447       strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]);
1448       zSep = " OR ";
1449     }
1450     strPrintf(&sql, ")\n");
1451   }
1452   strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
1453   zSep = " WHERE";
1454   for(i=0; i<nPk; i++){
1455     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1456     zSep = " AND";
1457   }
1458   strPrintf(&sql, " UNION ALL\n");
1459   strPrintf(&sql, "SELECT 2, count(*), 0\n");
1460   strPrintf(&sql, "  FROM main.%s A\n", zId);
1461   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId);
1462   zSep = "WHERE";
1463   for(i=0; i<nPk; i++){
1464     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1465     zSep = " AND";
1466   }
1467   strPrintf(&sql, ")\n");
1468   strPrintf(&sql, " UNION ALL\n");
1469   strPrintf(&sql, "SELECT 3, count(*), 0\n");
1470   strPrintf(&sql, "  FROM aux.%s B\n", zId);
1471   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId);
1472   zSep = "WHERE";
1473   for(i=0; i<nPk; i++){
1474     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1475     zSep = " AND";
1476   }
1477   strPrintf(&sql, ")\n ORDER BY 1;\n");
1478 
1479   if( (g.fDebug & DEBUG_DIFF_SQL)!=0 ){
1480     printf("SQL for %s:\n%s\n", zId, sql.z);
1481     goto end_summarize_one_table;
1482   }
1483 
1484   /* Run the query and output difference summary */
1485   pStmt = db_prepare("%s", sql.z);
1486   nUpdate = 0;
1487   nInsert = 0;
1488   nDelete = 0;
1489   nUnchanged = 0;
1490   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1491     switch( sqlite3_column_int(pStmt,0) ){
1492       case 1:
1493         nUpdate = sqlite3_column_int64(pStmt,2);
1494         nUnchanged = sqlite3_column_int64(pStmt,1) - nUpdate;
1495         break;
1496       case 2:
1497         nDelete = sqlite3_column_int64(pStmt,1);
1498         break;
1499       case 3:
1500         nInsert = sqlite3_column_int64(pStmt,1);
1501         break;
1502     }
1503   }
1504   sqlite3_finalize(pStmt);
1505   fprintf(out, "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
1506           zTab, nUpdate, nInsert, nDelete, nUnchanged);
1507 
1508 end_summarize_one_table:
1509   strFree(&sql);
1510   sqlite3_free(zId);
1511   namelistFree(az);
1512   namelistFree(az2);
1513   return;
1514 }
1515 
1516 /*
1517 ** Write a 64-bit signed integer as a varint onto out
1518 */
putsVarint(FILE * out,sqlite3_uint64 v)1519 static void putsVarint(FILE *out, sqlite3_uint64 v){
1520   int i, n;
1521   unsigned char p[12];
1522   if( v & (((sqlite3_uint64)0xff000000)<<32) ){
1523     p[8] = (unsigned char)v;
1524     v >>= 8;
1525     for(i=7; i>=0; i--){
1526       p[i] = (unsigned char)((v & 0x7f) | 0x80);
1527       v >>= 7;
1528     }
1529     fwrite(p, 8, 1, out);
1530   }else{
1531     n = 9;
1532     do{
1533       p[n--] = (unsigned char)((v & 0x7f) | 0x80);
1534       v >>= 7;
1535     }while( v!=0 );
1536     p[9] &= 0x7f;
1537     fwrite(p+n+1, 9-n, 1, out);
1538   }
1539 }
1540 
1541 /*
1542 ** Write an SQLite value onto out.
1543 */
putValue(FILE * out,sqlite3_stmt * pStmt,int k)1544 static void putValue(FILE *out, sqlite3_stmt *pStmt, int k){
1545   int iDType = sqlite3_column_type(pStmt, k);
1546   sqlite3_int64 iX;
1547   double rX;
1548   sqlite3_uint64 uX;
1549   int j;
1550 
1551   putc(iDType, out);
1552   switch( iDType ){
1553     case SQLITE_INTEGER:
1554       iX = sqlite3_column_int64(pStmt, k);
1555       memcpy(&uX, &iX, 8);
1556       for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out);
1557       break;
1558     case SQLITE_FLOAT:
1559       rX = sqlite3_column_double(pStmt, k);
1560       memcpy(&uX, &rX, 8);
1561       for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out);
1562       break;
1563     case SQLITE_TEXT:
1564       iX = sqlite3_column_bytes(pStmt, k);
1565       putsVarint(out, (sqlite3_uint64)iX);
1566       fwrite(sqlite3_column_text(pStmt, k),1,(size_t)iX,out);
1567       break;
1568     case SQLITE_BLOB:
1569       iX = sqlite3_column_bytes(pStmt, k);
1570       putsVarint(out, (sqlite3_uint64)iX);
1571       fwrite(sqlite3_column_blob(pStmt, k),1,(size_t)iX,out);
1572       break;
1573     case SQLITE_NULL:
1574       break;
1575   }
1576 }
1577 
1578 /*
1579 ** Generate a CHANGESET for all differences from main.zTab to aux.zTab.
1580 */
changeset_one_table(const char * zTab,FILE * out)1581 static void changeset_one_table(const char *zTab, FILE *out){
1582   sqlite3_stmt *pStmt;          /* SQL statment */
1583   char *zId = safeId(zTab);     /* Escaped name of the table */
1584   char **azCol = 0;             /* List of escaped column names */
1585   int nCol = 0;                 /* Number of columns */
1586   int *aiFlg = 0;               /* 0 if column is not part of PK */
1587   int *aiPk = 0;                /* Column numbers for each PK column */
1588   int nPk = 0;                  /* Number of PRIMARY KEY columns */
1589   Str sql;                      /* SQL for the diff query */
1590   int i, k;                     /* Loop counters */
1591   const char *zSep;             /* List separator */
1592 
1593   /* Check that the schemas of the two tables match. Exit early otherwise. */
1594   checkSchemasMatch(zTab);
1595   strInit(&sql);
1596 
1597   pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab);
1598   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1599     nCol++;
1600     azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol);
1601     if( azCol==0 ) runtimeError("out of memory");
1602     aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol);
1603     if( aiFlg==0 ) runtimeError("out of memory");
1604     azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1));
1605     aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5);
1606     if( i>0 ){
1607       if( i>nPk ){
1608         nPk = i;
1609         aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk);
1610         if( aiPk==0 ) runtimeError("out of memory");
1611       }
1612       aiPk[i-1] = nCol-1;
1613     }
1614   }
1615   sqlite3_finalize(pStmt);
1616   if( nPk==0 ) goto end_changeset_one_table;
1617   if( nCol>nPk ){
1618     strPrintf(&sql, "SELECT %d", SQLITE_UPDATE);
1619     for(i=0; i<nCol; i++){
1620       if( aiFlg[i] ){
1621         strPrintf(&sql, ",\n       A.%s", azCol[i]);
1622       }else{
1623         strPrintf(&sql, ",\n       A.%s IS NOT B.%s, A.%s, B.%s",
1624                   azCol[i], azCol[i], azCol[i], azCol[i]);
1625       }
1626     }
1627     strPrintf(&sql,"\n  FROM main.%s A, aux.%s B\n", zId, zId);
1628     zSep = " WHERE";
1629     for(i=0; i<nPk; i++){
1630       strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1631       zSep = " AND";
1632     }
1633     zSep = "\n   AND (";
1634     for(i=0; i<nCol; i++){
1635       if( aiFlg[i] ) continue;
1636       strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, azCol[i], azCol[i]);
1637       zSep = " OR\n        ";
1638     }
1639     strPrintf(&sql,")\n UNION ALL\n");
1640   }
1641   strPrintf(&sql, "SELECT %d", SQLITE_DELETE);
1642   for(i=0; i<nCol; i++){
1643     if( aiFlg[i] ){
1644       strPrintf(&sql, ",\n       A.%s", azCol[i]);
1645     }else{
1646       strPrintf(&sql, ",\n       1, A.%s, NULL", azCol[i]);
1647     }
1648   }
1649   strPrintf(&sql, "\n  FROM main.%s A\n", zId);
1650   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
1651   zSep =          "                   WHERE";
1652   for(i=0; i<nPk; i++){
1653     strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1654     zSep = " AND";
1655   }
1656   strPrintf(&sql, ")\n UNION ALL\n");
1657   strPrintf(&sql, "SELECT %d", SQLITE_INSERT);
1658   for(i=0; i<nCol; i++){
1659     if( aiFlg[i] ){
1660       strPrintf(&sql, ",\n       B.%s", azCol[i]);
1661     }else{
1662       strPrintf(&sql, ",\n       1, NULL, B.%s", azCol[i]);
1663     }
1664   }
1665   strPrintf(&sql, "\n  FROM aux.%s B\n", zId);
1666   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
1667   zSep =          "                   WHERE";
1668   for(i=0; i<nPk; i++){
1669     strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1670     zSep = " AND";
1671   }
1672   strPrintf(&sql, ")\n");
1673   strPrintf(&sql, " ORDER BY");
1674   zSep = " ";
1675   for(i=0; i<nPk; i++){
1676     strPrintf(&sql, "%s %d", zSep, aiPk[i]+2);
1677     zSep = ",";
1678   }
1679   strPrintf(&sql, ";\n");
1680 
1681   if( g.fDebug & DEBUG_DIFF_SQL ){
1682     printf("SQL for %s:\n%s\n", zId, sql.z);
1683     goto end_changeset_one_table;
1684   }
1685 
1686   putc('T', out);
1687   putsVarint(out, (sqlite3_uint64)nCol);
1688   for(i=0; i<nCol; i++) putc(aiFlg[i], out);
1689   fwrite(zTab, 1, strlen(zTab), out);
1690   putc(0, out);
1691 
1692   pStmt = db_prepare("%s", sql.z);
1693   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1694     int iType = sqlite3_column_int(pStmt,0);
1695     putc(iType, out);
1696     putc(0, out);
1697     switch( sqlite3_column_int(pStmt,0) ){
1698       case SQLITE_UPDATE: {
1699         for(k=1, i=0; i<nCol; i++){
1700           if( aiFlg[i] ){
1701             putValue(out, pStmt, k);
1702             k++;
1703           }else if( sqlite3_column_int(pStmt,k) ){
1704             putValue(out, pStmt, k+1);
1705             k += 3;
1706           }else{
1707             putc(0, out);
1708             k += 3;
1709           }
1710         }
1711         for(k=1, i=0; i<nCol; i++){
1712           if( aiFlg[i] ){
1713             putc(0, out);
1714             k++;
1715           }else if( sqlite3_column_int(pStmt,k) ){
1716             putValue(out, pStmt, k+2);
1717             k += 3;
1718           }else{
1719             putc(0, out);
1720             k += 3;
1721           }
1722         }
1723         break;
1724       }
1725       case SQLITE_INSERT: {
1726         for(k=1, i=0; i<nCol; i++){
1727           if( aiFlg[i] ){
1728             putValue(out, pStmt, k);
1729             k++;
1730           }else{
1731             putValue(out, pStmt, k+2);
1732             k += 3;
1733           }
1734         }
1735         break;
1736       }
1737       case SQLITE_DELETE: {
1738         for(k=1, i=0; i<nCol; i++){
1739           if( aiFlg[i] ){
1740             putValue(out, pStmt, k);
1741             k++;
1742           }else{
1743             putValue(out, pStmt, k+1);
1744             k += 3;
1745           }
1746         }
1747         break;
1748       }
1749     }
1750   }
1751   sqlite3_finalize(pStmt);
1752 
1753 end_changeset_one_table:
1754   while( nCol>0 ) sqlite3_free(azCol[--nCol]);
1755   sqlite3_free(azCol);
1756   sqlite3_free(aiPk);
1757   sqlite3_free(zId);
1758   sqlite3_free(aiFlg);
1759   strFree(&sql);
1760 }
1761 
1762 /*
1763 ** Return true if the ascii character passed as the only argument is a
1764 ** whitespace character. Otherwise return false.
1765 */
is_whitespace(char x)1766 static int is_whitespace(char x){
1767   return (x==' ' || x=='\t' || x=='\n' || x=='\r');
1768 }
1769 
1770 /*
1771 ** Extract the next SQL keyword or quoted string from buffer zIn and copy it
1772 ** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
1773 ** Return a pointer to the character within zIn immediately following
1774 ** the token or quoted string just extracted.
1775 */
gobble_token(const char * zIn,char * zBuf,int nBuf)1776 static const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
1777   const char *p = zIn;
1778   char *pOut = zBuf;
1779   char *pEnd = &pOut[nBuf-1];
1780   char q = 0;                     /* quote character, if any */
1781 
1782   if( p==0 ) return 0;
1783   while( is_whitespace(*p) ) p++;
1784   switch( *p ){
1785     case '"': q = '"'; break;
1786     case '\'': q = '\''; break;
1787     case '`': q = '`'; break;
1788     case '[': q = ']'; break;
1789   }
1790 
1791   if( q ){
1792     p++;
1793     while( *p && pOut<pEnd ){
1794       if( *p==q ){
1795         p++;
1796         if( *p!=q ) break;
1797       }
1798       if( pOut<pEnd ) *pOut++ = *p;
1799       p++;
1800     }
1801   }else{
1802     while( *p && !is_whitespace(*p) && *p!='(' ){
1803       if( pOut<pEnd ) *pOut++ = *p;
1804       p++;
1805     }
1806   }
1807 
1808   *pOut = '\0';
1809   return p;
1810 }
1811 
1812 /*
1813 ** This function is the implementation of SQL scalar function "module_name":
1814 **
1815 **   module_name(SQL)
1816 **
1817 ** The only argument should be an SQL statement of the type that may appear
1818 ** in the sqlite_schema table. If the statement is a "CREATE VIRTUAL TABLE"
1819 ** statement, then the value returned is the name of the module that it
1820 ** uses. Otherwise, if the statement is not a CVT, NULL is returned.
1821 */
module_name_func(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)1822 static void module_name_func(
1823   sqlite3_context *pCtx,
1824   int nVal, sqlite3_value **apVal
1825 ){
1826   const char *zSql;
1827   char zToken[32];
1828 
1829   assert( nVal==1 );
1830   zSql = (const char*)sqlite3_value_text(apVal[0]);
1831 
1832   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1833   if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
1834   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1835   if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
1836   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1837   if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
1838   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1839   if( zSql==0 ) return;
1840   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1841   if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
1842   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1843 
1844   sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
1845 }
1846 
1847 /*
1848 ** Return the text of an SQL statement that itself returns the list of
1849 ** tables to process within the database.
1850 */
all_tables_sql()1851 const char *all_tables_sql(){
1852   if( g.bHandleVtab ){
1853     int rc;
1854 
1855     rc = sqlite3_exec(g.db,
1856         "CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);"
1857         "INSERT INTO temp.tblmap VALUES"
1858         "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
1859 
1860         "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
1861         "('fts4', '_docsize'), ('fts4', '_stat'),"
1862 
1863         "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
1864         "('fts5', '_docsize'), ('fts5', '_config'),"
1865 
1866         "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
1867         , 0, 0, 0
1868     );
1869     assert( rc==SQLITE_OK );
1870 
1871     rc = sqlite3_create_function(
1872         g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
1873     );
1874     assert( rc==SQLITE_OK );
1875 
1876     return
1877       "SELECT name FROM main.sqlite_schema\n"
1878       " WHERE type='table' AND (\n"
1879       "    module_name(sql) IS NULL OR \n"
1880       "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1881       " ) AND name NOT IN (\n"
1882       "  SELECT a.name || b.postfix \n"
1883         "FROM main.sqlite_schema AS a, temp.tblmap AS b \n"
1884         "WHERE module_name(a.sql) = b.module\n"
1885       " )\n"
1886       "UNION \n"
1887       "SELECT name FROM aux.sqlite_schema\n"
1888       " WHERE type='table' AND (\n"
1889       "    module_name(sql) IS NULL OR \n"
1890       "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1891       " ) AND name NOT IN (\n"
1892       "  SELECT a.name || b.postfix \n"
1893         "FROM aux.sqlite_schema AS a, temp.tblmap AS b \n"
1894         "WHERE module_name(a.sql) = b.module\n"
1895       " )\n"
1896       " ORDER BY name";
1897   }else{
1898     return
1899       "SELECT name FROM main.sqlite_schema\n"
1900       " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1901       " UNION\n"
1902       "SELECT name FROM aux.sqlite_schema\n"
1903       " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1904       " ORDER BY name";
1905   }
1906 }
1907 
1908 /*
1909 ** Print sketchy documentation for this utility program
1910 */
showHelp(void)1911 static void showHelp(void){
1912   printf("Usage: %s [options] DB1 DB2\n", g.zArgv0);
1913   printf(
1914 "Output SQL text that would transform DB1 into DB2.\n"
1915 "Options:\n"
1916 "  --changeset FILE      Write a CHANGESET into FILE\n"
1917 "  -L|--lib LIBRARY      Load an SQLite extension library\n"
1918 "  --primarykey          Use schema-defined PRIMARY KEYs\n"
1919 "  --rbu                 Output SQL to create/populate RBU table(s)\n"
1920 "  --schema              Show only differences in the schema\n"
1921 "  --summary             Show only a summary of the differences\n"
1922 "  --table TAB           Show only differences in table TAB\n"
1923 "  --transaction         Show SQL output inside a transaction\n"
1924 "  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
1925 "See https://sqlite.org/sqldiff.html for detailed explanation.\n"
1926   );
1927 }
1928 
main(int argc,char ** argv)1929 int main(int argc, char **argv){
1930   const char *zDb1 = 0;
1931   const char *zDb2 = 0;
1932   int i;
1933   int rc;
1934   char *zErrMsg = 0;
1935   char *zSql;
1936   sqlite3_stmt *pStmt;
1937   char *zTab = 0;
1938   FILE *out = stdout;
1939   void (*xDiff)(const char*,FILE*) = diff_one_table;
1940 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1941   int nExt = 0;
1942   char **azExt = 0;
1943 #endif
1944   int useTransaction = 0;
1945   int neverUseTransaction = 0;
1946 
1947   g.zArgv0 = argv[0];
1948   sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
1949   for(i=1; i<argc; i++){
1950     const char *z = argv[i];
1951     if( z[0]=='-' ){
1952       z++;
1953       if( z[0]=='-' ) z++;
1954       if( strcmp(z,"changeset")==0 ){
1955         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1956         out = fopen(argv[++i], "wb");
1957         if( out==0 ) cmdlineError("cannot open: %s", argv[i]);
1958         xDiff = changeset_one_table;
1959         neverUseTransaction = 1;
1960       }else
1961       if( strcmp(z,"debug")==0 ){
1962         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1963         g.fDebug = strtol(argv[++i], 0, 0);
1964       }else
1965       if( strcmp(z,"help")==0 ){
1966         showHelp();
1967         return 0;
1968       }else
1969 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1970       if( strcmp(z,"lib")==0 || strcmp(z,"L")==0 ){
1971         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1972         azExt = realloc(azExt, sizeof(azExt[0])*(nExt+1));
1973         if( azExt==0 ) cmdlineError("out of memory");
1974         azExt[nExt++] = argv[++i];
1975       }else
1976 #endif
1977       if( strcmp(z,"primarykey")==0 ){
1978         g.bSchemaPK = 1;
1979       }else
1980       if( strcmp(z,"rbu")==0 ){
1981         xDiff = rbudiff_one_table;
1982       }else
1983       if( strcmp(z,"schema")==0 ){
1984         g.bSchemaOnly = 1;
1985       }else
1986       if( strcmp(z,"summary")==0 ){
1987         xDiff = summarize_one_table;
1988       }else
1989       if( strcmp(z,"table")==0 ){
1990         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1991         zTab = argv[++i];
1992         g.bSchemaCompare =
1993           sqlite3_stricmp(zTab, "sqlite_schema")==0
1994           || sqlite3_stricmp(zTab, "sqlite_master")==0;
1995       }else
1996       if( strcmp(z,"transaction")==0 ){
1997         useTransaction = 1;
1998       }else
1999       if( strcmp(z,"vtab")==0 ){
2000         g.bHandleVtab = 1;
2001       }else
2002       {
2003         cmdlineError("unknown option: %s", argv[i]);
2004       }
2005     }else if( zDb1==0 ){
2006       zDb1 = argv[i];
2007     }else if( zDb2==0 ){
2008       zDb2 = argv[i];
2009     }else{
2010       cmdlineError("unknown argument: %s", argv[i]);
2011     }
2012   }
2013   if( zDb2==0 ){
2014     cmdlineError("two database arguments required");
2015   }
2016   if( g.bSchemaOnly && g.bSchemaCompare ){
2017     cmdlineError("The --schema option is useless with --table %s .", zTab);
2018   }
2019   rc = sqlite3_open(zDb1, &g.db);
2020   if( rc ){
2021     cmdlineError("cannot open database file \"%s\"", zDb1);
2022   }
2023   rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_schema", 0, 0, &zErrMsg);
2024   if( rc || zErrMsg ){
2025     cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1);
2026   }
2027 #ifndef SQLITE_OMIT_LOAD_EXTENSION
2028   sqlite3_enable_load_extension(g.db, 1);
2029   for(i=0; i<nExt; i++){
2030     rc = sqlite3_load_extension(g.db, azExt[i], 0, &zErrMsg);
2031     if( rc || zErrMsg ){
2032       cmdlineError("error loading %s: %s", azExt[i], zErrMsg);
2033     }
2034   }
2035   free(azExt);
2036 #endif
2037   zSql = sqlite3_mprintf("ATTACH %Q as aux;", zDb2);
2038   rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg);
2039   sqlite3_free(zSql);
2040   zSql = 0;
2041   if( rc || zErrMsg ){
2042     cmdlineError("cannot attach database \"%s\"", zDb2);
2043   }
2044   rc = sqlite3_exec(g.db, "SELECT * FROM aux.sqlite_schema", 0, 0, &zErrMsg);
2045   if( rc || zErrMsg ){
2046     cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2);
2047   }
2048 
2049   if( neverUseTransaction ) useTransaction = 0;
2050   if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n");
2051   if( xDiff==rbudiff_one_table ){
2052     fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
2053            "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
2054            "WITHOUT ROWID;\n"
2055     );
2056   }
2057   if( zTab ){
2058     xDiff(zTab, out);
2059   }else{
2060     /* Handle tables one by one */
2061     pStmt = db_prepare("%s", all_tables_sql() );
2062     while( SQLITE_ROW==sqlite3_step(pStmt) ){
2063       xDiff((const char*)sqlite3_column_text(pStmt,0), out);
2064     }
2065     sqlite3_finalize(pStmt);
2066   }
2067   if( useTransaction ) printf("COMMIT;\n");
2068 
2069   /* TBD: Handle trigger differences */
2070   /* TBD: Handle view differences */
2071   sqlite3_close(g.db);
2072   return 0;
2073 }
2074