4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
13 ** This is a utility program that computes the differences in content
14 ** between two SQLite databases.
16 ** To compile, simply link against SQLite.
18 ** See the showHelp() routine below for a brief description of how to
30 ** All global variables are gathered into the "g" singleton.
33 const char *zArgv0
; /* Name of program */
34 int bSchemaOnly
; /* Only show schema differences */
35 int bSchemaPK
; /* Use the schema-defined PK, not the true PK */
36 int bHandleVtab
; /* Handle fts3, fts4, fts5 and rtree vtabs */
37 unsigned fDebug
; /* Debug flags */
38 int bSchemaCompare
; /* Doing single-table sqlite_schema compare */
39 sqlite3
*db
; /* The database connection */
43 ** Allowed values for g.fDebug
45 #define DEBUG_COLUMN_NAMES 0x000001
46 #define DEBUG_DIFF_SQL 0x000002
49 ** Dynamic string object
51 typedef struct Str Str
;
53 char *z
; /* Text of the string */
54 int nAlloc
; /* Bytes allocated in z[] */
55 int nUsed
; /* Bytes actually used in z[] */
59 ** Initialize a Str object
61 static void strInit(Str
*p
){
68 ** Print an error resulting from faulting command-line arguments and
71 static void cmdlineError(const char *zFormat
, ...){
73 fprintf(stderr
, "%s: ", g
.zArgv0
);
74 va_start(ap
, zFormat
);
75 vfprintf(stderr
, zFormat
, ap
);
77 fprintf(stderr
, "\n\"%s --help\" for more help\n", g
.zArgv0
);
82 ** Print an error message for an error that occurs at runtime, then
85 static void runtimeError(const char *zFormat
, ...){
87 fprintf(stderr
, "%s: ", g
.zArgv0
);
88 va_start(ap
, zFormat
);
89 vfprintf(stderr
, zFormat
, ap
);
91 fprintf(stderr
, "\n");
96 ** Free all memory held by a Str object
98 static void strFree(Str
*p
){
104 ** Add formatted text to the end of a Str object
106 static void strPrintf(Str
*p
, const char *zFormat
, ...){
111 va_start(ap
, zFormat
);
112 sqlite3_vsnprintf(p
->nAlloc
-p
->nUsed
, p
->z
+p
->nUsed
, zFormat
, ap
);
114 nNew
= (int)strlen(p
->z
+ p
->nUsed
);
118 if( p
->nUsed
+nNew
< p
->nAlloc
-1 ){
122 p
->nAlloc
= p
->nAlloc
*2 + 1000;
123 p
->z
= sqlite3_realloc(p
->z
, p
->nAlloc
);
124 if( p
->z
==0 ) runtimeError("out of memory");
130 /* Safely quote an SQL identifier. Use the minimum amount of transformation
131 ** necessary to allow the string to be used with %s.
133 ** Space to hold the returned string is obtained from sqlite3_malloc(). The
134 ** caller is responsible for ensuring this space is freed when no longer
137 static char *safeId(const char *zId
){
140 if( zId
[0]==0 ) return sqlite3_mprintf("\"\"");
141 for(i
=x
=0; (c
= zId
[i
])!=0; i
++){
142 if( !isalpha(c
) && c
!='_' ){
143 if( i
>0 && isdigit(c
) ){
146 return sqlite3_mprintf("\"%w\"", zId
);
150 if( x
|| !sqlite3_keyword_check(zId
,i
) ){
151 return sqlite3_mprintf("%s", zId
);
153 return sqlite3_mprintf("\"%w\"", zId
);
157 ** Prepare a new SQL statement. Print an error and abort if anything
160 static sqlite3_stmt
*db_vprepare(const char *zFormat
, va_list ap
){
165 zSql
= sqlite3_vmprintf(zFormat
, ap
);
166 if( zSql
==0 ) runtimeError("out of memory");
167 rc
= sqlite3_prepare_v2(g
.db
, zSql
, -1, &pStmt
, 0);
169 runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g
.db
),
175 static sqlite3_stmt
*db_prepare(const char *zFormat
, ...){
178 va_start(ap
, zFormat
);
179 pStmt
= db_vprepare(zFormat
, ap
);
185 ** Free a list of strings
187 static void namelistFree(char **az
){
190 for(i
=0; az
[i
]; i
++) sqlite3_free(az
[i
]);
196 ** Return a list of column names [a] for the table zDb.zTab. Space to
197 ** hold the list is obtained from sqlite3_malloc() and should released
198 ** using namelistFree() when no longer needed.
200 ** Primary key columns are listed first, followed by data columns.
201 ** The number of columns in the primary key is returned in *pnPkey.
203 ** Normally [a], the "primary key" in the previous sentence is the true
204 ** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables
205 ** or the declared PRIMARY KEY for WITHOUT ROWID tables. However, if
206 ** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is
207 ** used in all cases. In that case, entries that have NULL values in
208 ** any of their primary key fields will be excluded from the analysis.
210 ** If the primary key for a table is the rowid but rowid is inaccessible,
211 ** then this routine returns a NULL pointer.
213 ** [a. If the lone, named table is "sqlite_schema", "rootpage" column is
214 ** omitted and the "type" and "name" columns are made to be the PK.]
217 ** CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c));
219 ** az = { "rowid", "a", "b", "c", 0 } // Normal case
220 ** az = { "c", "a", "b", 0 } // g.bSchemaPK==1
222 ** CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b));
224 ** az = { "b", "a", "c", 0 }
226 ** CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z));
227 ** *pnPKey = 1 // Normal case
228 ** az = { "rowid", "x", "y", "z", 0 } // Normal case
229 ** *pnPKey = 2 // g.bSchemaPK==1
230 ** az = { "y", "x", "z", 0 } // g.bSchemaPK==1
232 ** CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID;
234 ** az = { "y", "z", "x", 0 }
236 ** CREATE TABLE t5(rowid,_rowid_,oid);
237 ** az = 0 // The rowid is not accessible
239 static char **columnNames(
240 const char *zDb
, /* Database ("main" or "aux") to query */
241 const char *zTab
, /* Name of table to return details of */
242 int *pnPKey
, /* OUT: Number of PK columns */
243 int *pbRowid
/* OUT: True if PK is an implicit rowid */
245 char **az
= 0; /* List of column names to be returned */
246 int naz
= 0; /* Number of entries in az[] */
247 sqlite3_stmt
*pStmt
; /* SQL statement being run */
248 char *zPkIdxName
= 0; /* Name of the PRIMARY KEY index */
249 int truePk
= 0; /* PRAGMA table_info indentifies the PK to use */
250 int nPK
= 0; /* Number of PRIMARY KEY columns */
251 int i
, j
; /* Loop counters */
253 if( g
.bSchemaPK
==0 ){
254 /* Normal case: Figure out what the true primary key is for the table.
255 ** * For WITHOUT ROWID tables, the true primary key is the same as
256 ** the schema PRIMARY KEY, which is guaranteed to be present.
257 ** * For rowid tables with an INTEGER PRIMARY KEY, the true primary
258 ** key is the INTEGER PRIMARY KEY.
259 ** * For all other rowid tables, the rowid is the true primary key.
261 pStmt
= db_prepare("PRAGMA %s.index_list=%Q", zDb
, zTab
);
262 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
263 if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt
,3),"pk")==0 ){
264 zPkIdxName
= sqlite3_mprintf("%s", sqlite3_column_text(pStmt
, 1));
268 sqlite3_finalize(pStmt
);
273 pStmt
= db_prepare("PRAGMA %s.index_xinfo=%Q", zDb
, zPkIdxName
);
274 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
276 if( sqlite3_column_int(pStmt
,5) ){ nKey
++; continue; }
277 if( sqlite3_column_int(pStmt
,1)>=0 ) truePk
= 1;
279 if( nCol
==nKey
) truePk
= 1;
285 sqlite3_finalize(pStmt
);
286 sqlite3_free(zPkIdxName
);
291 pStmt
= db_prepare("PRAGMA %s.table_info=%Q", zDb
, zTab
);
293 /* The g.bSchemaPK==1 case: Use whatever primary key is declared
294 ** in the schema. The "rowid" will still be used as the primary key
295 ** if the table definition does not contain a PRIMARY KEY.
298 pStmt
= db_prepare("PRAGMA %s.table_info=%Q", zDb
, zTab
);
299 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
300 if( sqlite3_column_int(pStmt
,5)>0 ) nPK
++;
302 sqlite3_reset(pStmt
);
303 if( nPK
==0 ) nPK
= 1;
306 if( g
.bSchemaCompare
){
307 assert( sqlite3_stricmp(zTab
,"sqlite_schema")==0
308 || sqlite3_stricmp(zTab
,"sqlite_master")==0 );
309 /* For sqlite_schema, will use type and name as the PK. */
315 az
= sqlite3_malloc( sizeof(char*)*(nPK
+1) );
316 if( az
==0 ) runtimeError("out of memory");
317 memset(az
, 0, sizeof(char*)*(nPK
+1));
318 if( g
.bSchemaCompare
){
319 az
[0] = sqlite3_mprintf("%s", "type");
320 az
[1] = sqlite3_mprintf("%s", "name");
322 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
323 char * sid
= safeId((char*)sqlite3_column_text(pStmt
,1));
325 if( truePk
&& (iPKey
= sqlite3_column_int(pStmt
,5))>0 ){
328 if( !g
.bSchemaCompare
329 || !(strcmp(sid
,"rootpage")==0
330 ||strcmp(sid
,"name")==0
331 ||strcmp(sid
,"type")==0)){
332 az
= sqlite3_realloc(az
, sizeof(char*)*(naz
+2) );
333 if( az
==0 ) runtimeError("out of memory");
338 sqlite3_finalize(pStmt
);
339 if( az
) az
[naz
] = 0;
341 /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of
342 ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0). */
343 if( pbRowid
) *pbRowid
= (az
[0]==0);
345 /* If this table has an implicit rowid for a PK, figure out how to refer
346 ** to it. There are usually three options - "rowid", "_rowid_" and "oid".
347 ** Any of these will work, unless the table has an explicit column of the
348 ** same name or the sqlite_schema tables are to be compared. In the latter
349 ** case, pretend that the "true" primary key is the name column, which
350 ** avoids extraneous diffs against the schemas due to rowid variance. */
352 const char *azRowid
[] = { "rowid", "_rowid_", "oid" };
353 for(i
=0; i
<sizeof(azRowid
)/sizeof(azRowid
[0]); i
++){
354 for(j
=1; j
<naz
; j
++){
355 if( sqlite3_stricmp(az
[j
], azRowid
[i
])==0 ) break;
358 az
[0] = sqlite3_mprintf("%s", azRowid
[i
]);
363 for(i
=1; i
<naz
; i
++) sqlite3_free(az
[i
]);
372 ** Print the sqlite3_value X as an SQL literal.
374 static void printQuoted(FILE *out
, sqlite3_value
*X
){
375 switch( sqlite3_value_type(X
) ){
379 r1
= sqlite3_value_double(X
);
380 sqlite3_snprintf(sizeof(zBuf
), zBuf
, "%!.15g", r1
);
381 fprintf(out
, "%s", zBuf
);
384 case SQLITE_INTEGER
: {
385 fprintf(out
, "%lld", sqlite3_value_int64(X
));
389 const unsigned char *zBlob
= sqlite3_value_blob(X
);
390 int nBlob
= sqlite3_value_bytes(X
);
394 for(i
=0; i
<nBlob
; i
++){
395 fprintf(out
, "%02x", zBlob
[i
]);
399 /* Could be an OOM, could be a zero-byte blob */
405 const unsigned char *zArg
= sqlite3_value_text(X
);
408 fprintf(out
, "NULL");
413 for(i
=j
=0; zArg
[i
]; i
++){
415 int ctl
= iscntrl(c
);
418 fprintf(out
, "%.*s'||X'%02x", i
-j
, &zArg
[j
], c
);
421 fprintf(out
, "%02x", c
);
426 fprintf(out
, "'\n||'");
429 fprintf(out
, "%.*s'", i
-j
+1, &zArg
[j
]);
434 fprintf(out
, "%s'", &zArg
[j
]);
439 fprintf(out
, "NULL");
446 ** Output SQL that will recreate the aux.zTab table.
448 static void dump_table(const char *zTab
, FILE *out
){
449 char *zId
= safeId(zTab
); /* Name of the table */
450 char **az
= 0; /* List of columns */
451 int nPk
; /* Number of true primary key columns */
452 int nCol
; /* Number of data columns */
453 int i
; /* Loop counter */
454 sqlite3_stmt
*pStmt
; /* SQL statement */
455 const char *zSep
; /* Separator string */
456 Str ins
; /* Beginning of the INSERT statement */
458 pStmt
= db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab
);
459 if( SQLITE_ROW
==sqlite3_step(pStmt
) ){
460 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
462 sqlite3_finalize(pStmt
);
463 if( !g
.bSchemaOnly
){
464 az
= columnNames("aux", zTab
, &nPk
, 0);
467 pStmt
= db_prepare("SELECT * FROM aux.%s", zId
);
468 strPrintf(&ins
,"INSERT INTO %s VALUES", zId
);
473 for(i
=0; az
[i
]; i
++){
474 strPrintf(&sql
, "%s %s", zSep
, az
[i
]);
477 strPrintf(&sql
," FROM aux.%s", zId
);
479 for(i
=1; i
<=nPk
; i
++){
480 strPrintf(&sql
, "%s %d", zSep
, i
);
483 pStmt
= db_prepare("%s", sql
.z
);
485 strPrintf(&ins
, "INSERT INTO %s", zId
);
487 for(i
=0; az
[i
]; i
++){
488 strPrintf(&ins
, "%s%s", zSep
, az
[i
]);
491 strPrintf(&ins
,") VALUES");
494 nCol
= sqlite3_column_count(pStmt
);
495 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
496 fprintf(out
, "%s",ins
.z
);
498 for(i
=0; i
<nCol
; i
++){
499 fprintf(out
, "%s",zSep
);
500 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
503 fprintf(out
, ");\n");
505 sqlite3_finalize(pStmt
);
507 } /* endif !g.bSchemaOnly */
508 pStmt
= db_prepare("SELECT sql FROM aux.sqlite_schema"
509 " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
511 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
512 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
514 sqlite3_finalize(pStmt
);
520 ** Compute all differences for a single table, except if the
521 ** table name is sqlite_schema, ignore the rootpage column.
523 static void diff_one_table(const char *zTab
, FILE *out
){
524 char *zId
= safeId(zTab
); /* Name of table (translated for us in SQL) */
525 char **az
= 0; /* Columns in main */
526 char **az2
= 0; /* Columns in aux */
527 int nPk
; /* Primary key columns in main */
528 int nPk2
; /* Primary key columns in aux */
529 int n
= 0; /* Number of columns in main */
530 int n2
; /* Number of columns in aux */
531 int nQ
; /* Number of output columns in the diff query */
532 int i
; /* Loop counter */
533 const char *zSep
; /* Separator string */
534 Str sql
; /* Comparison query */
535 sqlite3_stmt
*pStmt
; /* Query statement to do the diff */
536 const char *zLead
= /* Becomes line-comment for sqlite_schema */
537 (g
.bSchemaCompare
)? "-- " : "";
540 if( g
.fDebug
==DEBUG_COLUMN_NAMES
){
541 /* Simply run columnNames() on all tables of the origin
542 ** database and show the results. This is used for testing
543 ** and debugging of the columnNames() function.
545 az
= columnNames("aux",zTab
, &nPk
, 0);
547 printf("Rowid not accessible for %s\n", zId
);
550 for(i
=0; az
[i
]; i
++){
551 printf(" %s", az
[i
]);
552 if( i
+1==nPk
) printf(" *");
556 goto end_diff_one_table
;
559 if( sqlite3_table_column_metadata(g
.db
,"aux",zTab
,0,0,0,0,0,0) ){
560 if( !sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
561 /* Table missing from second database. */
562 if( g
.bSchemaCompare
)
563 fprintf(out
, "-- 2nd DB has no %s table\n", zTab
);
565 fprintf(out
, "DROP TABLE %s;\n", zId
);
567 goto end_diff_one_table
;
570 if( sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
571 /* Table missing from source */
572 if( g
.bSchemaCompare
)
573 fprintf(out
, "-- 1st DB has no %s table\n", zTab
);
575 dump_table(zTab
, out
);
576 goto end_diff_one_table
;
579 az
= columnNames("main", zTab
, &nPk
, 0);
580 az2
= columnNames("aux", zTab
, &nPk2
, 0);
582 for(n
=0; az
[n
] && az2
[n
]; n
++){
583 if( sqlite3_stricmp(az
[n
],az2
[n
])!=0 ) break;
591 /* Schema mismatch */
592 fprintf(out
, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead
, zId
);
593 dump_table(zTab
, out
);
594 goto end_diff_one_table
;
597 /* Build the comparison query */
598 for(n2
=n
; az2
[n2
]; n2
++){
599 fprintf(out
, "ALTER TABLE %s ADD COLUMN %s;\n", zId
, safeId(az2
[n2
]));
601 nQ
= nPk2
+1+2*(n2
-nPk2
);
604 for(i
=0; i
<nPk
; i
++){
605 strPrintf(&sql
, "%sB.%s", zSep
, az
[i
]);
608 strPrintf(&sql
, ", 1%s -- changed row\n", nPk
==n
? "" : ",");
610 strPrintf(&sql
, " A.%s IS NOT B.%s, B.%s%s\n",
611 az
[i
], az2
[i
], az2
[i
], az2
[i
+1]==0 ? "" : ",");
615 strPrintf(&sql
, " B.%s IS NOT NULL, B.%s%s\n",
616 az2
[i
], az2
[i
], az2
[i
+1]==0 ? "" : ",");
619 strPrintf(&sql
, " FROM main.%s A, aux.%s B\n", zId
, zId
);
621 for(i
=0; i
<nPk
; i
++){
622 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
627 strPrintf(&sql
, "%sA.%s IS NOT B.%s%s\n",
628 zSep
, az
[i
], az2
[i
], az2
[i
+1]==0 ? ")" : "");
633 strPrintf(&sql
, "%sB.%s IS NOT NULL%s\n",
634 zSep
, az2
[i
], az2
[i
+1]==0 ? ")" : "");
638 strPrintf(&sql
, " UNION ALL\n");
641 for(i
=0; i
<nPk
; i
++){
642 strPrintf(&sql
, "%sA.%s", zSep
, az
[i
]);
645 strPrintf(&sql
, ", 2%s -- deleted row\n", nPk
==n
? "" : ",");
647 strPrintf(&sql
, " NULL, NULL%s\n", i
==n2
-1 ? "" : ",");
650 strPrintf(&sql
, " FROM main.%s A\n", zId
);
651 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId
);
653 for(i
=0; i
<nPk
; i
++){
654 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
657 strPrintf(&sql
, ")\n");
658 zSep
= " UNION ALL\nSELECT ";
659 for(i
=0; i
<nPk
; i
++){
660 strPrintf(&sql
, "%sB.%s", zSep
, az
[i
]);
663 strPrintf(&sql
, ", 3%s -- inserted row\n", nPk
==n
? "" : ",");
665 strPrintf(&sql
, " 1, B.%s%s\n", az2
[i
], az2
[i
+1]==0 ? "" : ",");
668 strPrintf(&sql
, " FROM aux.%s B\n", zId
);
669 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId
);
671 for(i
=0; i
<nPk
; i
++){
672 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
675 strPrintf(&sql
, ")\n ORDER BY");
677 for(i
=1; i
<=nPk
; i
++){
678 strPrintf(&sql
, "%s%d", zSep
, i
);
681 strPrintf(&sql
, ";\n");
683 if( g
.fDebug
& DEBUG_DIFF_SQL
){
684 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
685 goto end_diff_one_table
;
688 /* Drop indexes that are missing in the destination */
690 "SELECT name FROM main.sqlite_schema"
691 " WHERE type='index' AND tbl_name=%Q"
692 " AND sql IS NOT NULL"
693 " AND sql NOT IN (SELECT sql FROM aux.sqlite_schema"
694 " WHERE type='index' AND tbl_name=%Q"
695 " AND sql IS NOT NULL)",
697 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
698 char *z
= safeId((const char*)sqlite3_column_text(pStmt
,0));
699 fprintf(out
, "DROP INDEX %s;\n", z
);
702 sqlite3_finalize(pStmt
);
704 /* Run the query and output differences */
705 if( !g
.bSchemaOnly
){
706 pStmt
= db_prepare("%s", sql
.z
);
707 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
708 int iType
= sqlite3_column_int(pStmt
, nPk
);
709 if( iType
==1 || iType
==2 ){
710 if( iType
==1 ){ /* Change the content of a row */
711 fprintf(out
, "%sUPDATE %s", zLead
, zId
);
713 for(i
=nPk
+1; i
<nQ
; i
+=2){
714 if( sqlite3_column_int(pStmt
,i
)==0 ) continue;
715 fprintf(out
, "%s %s=", zSep
, az2
[(i
+nPk
-1)/2]);
717 printQuoted(out
, sqlite3_column_value(pStmt
,i
+1));
719 }else{ /* Delete a row */
720 fprintf(out
, "%sDELETE FROM %s", zLead
, zId
);
723 for(i
=0; i
<nPk
; i
++){
724 fprintf(out
, "%s %s=", zSep
, az2
[i
]);
725 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
729 }else{ /* Insert a row */
730 fprintf(out
, "%sINSERT INTO %s(%s", zLead
, zId
, az2
[0]);
731 for(i
=1; az2
[i
]; i
++) fprintf(out
, ",%s", az2
[i
]);
732 fprintf(out
, ") VALUES");
734 for(i
=0; i
<nPk2
; i
++){
735 fprintf(out
, "%s", zSep
);
737 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
739 for(i
=nPk2
+2; i
<nQ
; i
+=2){
741 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
743 fprintf(out
, ");\n");
746 sqlite3_finalize(pStmt
);
747 } /* endif !g.bSchemaOnly */
749 /* Create indexes that are missing in the source */
751 "SELECT sql FROM aux.sqlite_schema"
752 " WHERE type='index' AND tbl_name=%Q"
753 " AND sql IS NOT NULL"
754 " AND sql NOT IN (SELECT sql FROM main.sqlite_schema"
755 " WHERE type='index' AND tbl_name=%Q"
756 " AND sql IS NOT NULL)",
758 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
759 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
761 sqlite3_finalize(pStmt
);
772 ** Check that table zTab exists and has the same schema in both the "main"
773 ** and "aux" databases currently opened by the global db handle. If they
774 ** do not, output an error message on stderr and exit(1). Otherwise, if
775 ** the schemas do match, return control to the caller.
777 static void checkSchemasMatch(const char *zTab
){
778 sqlite3_stmt
*pStmt
= db_prepare(
779 "SELECT A.sql=B.sql FROM main.sqlite_schema A, aux.sqlite_schema B"
780 " WHERE A.name=%Q AND B.name=%Q", zTab
, zTab
782 if( SQLITE_ROW
==sqlite3_step(pStmt
) ){
783 if( sqlite3_column_int(pStmt
,0)==0 ){
784 runtimeError("schema changes for table %s", safeId(zTab
));
787 runtimeError("table %s missing from one or both databases", safeId(zTab
));
789 sqlite3_finalize(pStmt
);
792 /**************************************************************************
793 ** The following code is copied from fossil. It is used to generate the
794 ** fossil delta blobs sometimes used in RBU update records.
797 typedef unsigned short u16
;
798 typedef unsigned int u32
;
799 typedef unsigned char u8
;
802 ** The width of a hash window in bytes. The algorithm only works if this
808 ** The current state of the rolling hash.
810 ** z[] holds the values that have been hashed. z[] is a circular buffer.
811 ** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of
814 ** Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted
815 ** sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1.
816 ** (Each index for z[] should be module NHASH, of course. The %NHASH operator
817 ** is omitted in the prior expression for brevity.)
819 typedef struct hash hash
;
821 u16 a
, b
; /* Hash values */
822 u16 i
; /* Start of the hash window */
823 char z
[NHASH
]; /* The values that have been hashed */
827 ** Initialize the rolling hash using the first NHASH characters of z[]
829 static void hash_init(hash
*pHash
, const char *z
){
832 for(i
=0; i
<NHASH
; i
++){
837 pHash
->a
= a
& 0xffff;
838 pHash
->b
= b
& 0xffff;
843 ** Advance the rolling hash by a single character "c"
845 static void hash_next(hash
*pHash
, int c
){
846 u16 old
= pHash
->z
[pHash
->i
];
847 pHash
->z
[pHash
->i
] = (char)c
;
848 pHash
->i
= (pHash
->i
+1)&(NHASH
-1);
849 pHash
->a
= pHash
->a
- old
+ (char)c
;
850 pHash
->b
= pHash
->b
- NHASH
*old
+ pHash
->a
;
854 ** Return a 32-bit hash value
856 static u32
hash_32bit(hash
*pHash
){
857 return (pHash
->a
& 0xffff) | (((u32
)(pHash
->b
& 0xffff))<<16);
861 ** Write an base-64 integer into the given buffer.
863 static void putInt(unsigned int v
, char **pz
){
864 static const char zDigits
[] =
865 "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~";
866 /* 123456789 123456789 123456789 123456789 123456789 123456789 123 */
873 for(i
=0; v
>0; i
++, v
>>=6){
874 zBuf
[i
] = zDigits
[v
&0x3f];
876 for(j
=i
-1; j
>=0; j
--){
882 ** Return the number digits in the base-64 representation of a positive integer
884 static int digit_count(int v
){
886 for(i
=1, x
=64; (unsigned int)v
>=x
; i
++, x
<<= 6){}
891 ** Compute a 32-bit checksum on the N-byte buffer. Return the result.
893 static unsigned int checksum(const char *zIn
, size_t N
){
894 const unsigned char *z
= (const unsigned char *)zIn
;
900 sum0
+= ((unsigned)z
[0] + z
[4] + z
[8] + z
[12]);
901 sum1
+= ((unsigned)z
[1] + z
[5] + z
[9] + z
[13]);
902 sum2
+= ((unsigned)z
[2] + z
[6] + z
[10]+ z
[14]);
903 sum3
+= ((unsigned)z
[3] + z
[7] + z
[11]+ z
[15]);
915 sum3
+= (sum2
<< 8) + (sum1
<< 16) + (sum0
<< 24);
917 case 3: sum3
+= (z
[2] << 8);
918 case 2: sum3
+= (z
[1] << 16);
919 case 1: sum3
+= (z
[0] << 24);
926 ** Create a new delta.
928 ** The delta is written into a preallocated buffer, zDelta, which
929 ** should be at least 60 bytes longer than the target file, zOut.
930 ** The delta string will be NUL-terminated, but it might also contain
931 ** embedded NUL characters if either the zSrc or zOut files are
932 ** binary. This function returns the length of the delta string
933 ** in bytes, excluding the final NUL terminator character.
937 ** The delta begins with a base64 number followed by a newline. This
938 ** number is the number of bytes in the TARGET file. Thus, given a
939 ** delta file z, a program can compute the size of the output file
940 ** simply by reading the first line and decoding the base-64 number
941 ** found there. The delta_output_size() routine does exactly this.
943 ** After the initial size number, the delta consists of a series of
944 ** literal text segments and commands to copy from the SOURCE file.
945 ** A copy command looks like this:
949 ** where NNN is the number of bytes to be copied and MMM is the offset
950 ** into the source file of the first byte (both base-64). If NNN is 0
951 ** it means copy the rest of the input file. Literal text is like this:
955 ** where NNN is the number of bytes of text (base-64) and TTTTT is the text.
957 ** The last term is of the form
961 ** In this case, NNN is a 32-bit bigendian checksum of the output file
962 ** that can be used to verify that the delta applied correctly. All
963 ** numbers are in base-64.
965 ** Pure text files generate a pure text delta. Binary files generate a
966 ** delta that may contain some binary data.
970 ** The encoder first builds a hash table to help it find matching
971 ** patterns in the source file. 16-byte chunks of the source file
972 ** sampled at evenly spaced intervals are used to populate the hash
975 ** Next we begin scanning the target file using a sliding 16-byte
976 ** window. The hash of the 16-byte window in the target is used to
977 ** search for a matching section in the source file. When a match
978 ** is found, a copy command is added to the delta. An effort is
979 ** made to extend the matching section to regions that come before
980 ** and after the 16-byte hash window. A copy command is only issued
981 ** if the result would use less space that just quoting the text
982 ** literally. Literal text is added to the delta for sections that
983 ** do not match or which can not be encoded efficiently using copy
986 static int rbuDeltaCreate(
987 const char *zSrc
, /* The source or pattern file */
988 unsigned int lenSrc
, /* Length of the source file */
989 const char *zOut
, /* The target file */
990 unsigned int lenOut
, /* Length of the target file */
991 char *zDelta
/* Write the delta into this buffer */
993 unsigned int i
, base
;
994 char *zOrigDelta
= zDelta
;
996 int nHash
; /* Number of hash table entries */
997 int *landmark
; /* Primary hash table */
998 int *collide
; /* Collision chain */
999 int lastRead
= -1; /* Last byte of zSrc read by a COPY command */
1001 /* Add the target file size to the beginning of the delta
1003 putInt(lenOut
, &zDelta
);
1006 /* If the source file is very small, it means that we have no
1007 ** chance of ever doing a copy command. Just output a single
1008 ** literal segment for the entire target and exit.
1010 if( lenSrc
<=NHASH
){
1011 putInt(lenOut
, &zDelta
);
1013 memcpy(zDelta
, zOut
, lenOut
);
1015 putInt(checksum(zOut
, lenOut
), &zDelta
);
1017 return (int)(zDelta
- zOrigDelta
);
1020 /* Compute the hash table used to locate matching sections in the
1023 nHash
= lenSrc
/NHASH
;
1024 collide
= sqlite3_malloc( nHash
*2*sizeof(int) );
1025 landmark
= &collide
[nHash
];
1026 memset(landmark
, -1, nHash
*sizeof(int));
1027 memset(collide
, -1, nHash
*sizeof(int));
1028 for(i
=0; i
<lenSrc
-NHASH
; i
+=NHASH
){
1030 hash_init(&h
, &zSrc
[i
]);
1031 hv
= hash_32bit(&h
) % nHash
;
1032 collide
[i
/NHASH
] = landmark
[hv
];
1033 landmark
[hv
] = i
/NHASH
;
1036 /* Begin scanning the target file and generating copy commands and
1037 ** literal sections of the delta.
1039 base
= 0; /* We have already generated everything before zOut[base] */
1040 while( base
+NHASH
<lenOut
){
1042 int bestCnt
, bestOfst
=0, bestLitsz
=0;
1043 hash_init(&h
, &zOut
[base
]);
1044 i
= 0; /* Trying to match a landmark against zOut[base+i] */
1050 hv
= hash_32bit(&h
) % nHash
;
1051 iBlock
= landmark
[hv
];
1052 while( iBlock
>=0 && (limit
--)>0 ){
1054 ** The hash window has identified a potential match against
1055 ** landmark block iBlock. But we need to investigate further.
1057 ** Look for a region in zOut that matches zSrc. Anchor the search
1058 ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to
1059 ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen].
1061 ** Set cnt equal to the length of the match and set ofst so that
1062 ** zSrc[ofst] is the first element of the match. litsz is the number
1063 ** of characters between zOut[base] and the beginning of the match.
1064 ** sz will be the overhead (in bytes) needed to encode the copy
1065 ** command. Only generate copy command if the overhead of the
1066 ** copy command is less than the amount of literal text to be copied.
1068 int cnt
, ofst
, litsz
;
1072 /* Beginning at iSrc, match forwards as far as we can. j counts
1073 ** the number of characters that match */
1074 iSrc
= iBlock
*NHASH
;
1076 j
=0, x
=iSrc
, y
=base
+i
;
1077 (unsigned int)x
<lenSrc
&& (unsigned int)y
<lenOut
;
1080 if( zSrc
[x
]!=zOut
[y
] ) break;
1084 /* Beginning at iSrc-1, match backwards as far as we can. k counts
1085 ** the number of characters that match */
1086 for(k
=1; k
<iSrc
&& (unsigned int)k
<=i
; k
++){
1087 if( zSrc
[iSrc
-k
]!=zOut
[base
+i
-k
] ) break;
1091 /* Compute the offset and size of the matching region */
1094 litsz
= i
-k
; /* Number of bytes of literal text before the copy */
1095 /* sz will hold the number of bytes needed to encode the "insert"
1096 ** command and the copy command, not counting the "insert" text */
1097 sz
= digit_count(i
-k
)+digit_count(cnt
)+digit_count(ofst
)+3;
1098 if( cnt
>=sz
&& cnt
>bestCnt
){
1099 /* Remember this match only if it is the best so far and it
1100 ** does not increase the file size */
1106 /* Check the next matching block */
1107 iBlock
= collide
[iBlock
];
1110 /* We have a copy command that does not cause the delta to be larger
1111 ** than a literal insert. So add the copy command to the delta.
1115 /* Add an insert command before the copy */
1116 putInt(bestLitsz
,&zDelta
);
1118 memcpy(zDelta
, &zOut
[base
], bestLitsz
);
1119 zDelta
+= bestLitsz
;
1123 putInt(bestCnt
, &zDelta
);
1125 putInt(bestOfst
, &zDelta
);
1127 if( bestOfst
+ bestCnt
-1 > lastRead
){
1128 lastRead
= bestOfst
+ bestCnt
- 1;
1134 /* If we reach this point, it means no match is found so far */
1135 if( base
+i
+NHASH
>=lenOut
){
1136 /* We have reached the end of the file and have not found any
1137 ** matches. Do an "insert" for everything that does not match */
1138 putInt(lenOut
-base
, &zDelta
);
1140 memcpy(zDelta
, &zOut
[base
], lenOut
-base
);
1141 zDelta
+= lenOut
-base
;
1146 /* Advance the hash by one character. Keep looking for a match */
1147 hash_next(&h
, zOut
[base
+i
+NHASH
]);
1151 /* Output a final "insert" record to get all the text at the end of
1152 ** the file that does not match anything in the source file.
1155 putInt(lenOut
-base
, &zDelta
);
1157 memcpy(zDelta
, &zOut
[base
], lenOut
-base
);
1158 zDelta
+= lenOut
-base
;
1160 /* Output the final checksum record. */
1161 putInt(checksum(zOut
, lenOut
), &zDelta
);
1163 sqlite3_free(collide
);
1164 return (int)(zDelta
- zOrigDelta
);
1168 ** End of code copied from fossil.
1169 **************************************************************************/
1171 static void strPrintfArray(
1172 Str
*pStr
, /* String object to append to */
1173 const char *zSep
, /* Separator string */
1174 const char *zFmt
, /* Format for each entry */
1175 char **az
, int n
/* Array of strings & its size (or -1) */
1178 for(i
=0; az
[i
] && (i
<n
|| n
<0); i
++){
1179 if( i
!=0 ) strPrintf(pStr
, "%s", zSep
);
1180 strPrintf(pStr
, zFmt
, az
[i
], az
[i
], az
[i
]);
1184 static void getRbudiffQuery(
1193 /* First the newly inserted rows: **/
1194 strPrintf(pSql
, "SELECT ");
1195 strPrintfArray(pSql
, ", ", "%s", azCol
, -1);
1196 strPrintf(pSql
, ", 0, "); /* Set ota_control to 0 for an insert */
1197 strPrintfArray(pSql
, ", ", "NULL", azCol
, -1);
1198 strPrintf(pSql
, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab
);
1199 strPrintf(pSql
, " SELECT 1 FROM ", zTab
);
1200 strPrintf(pSql
, " main.%Q AS o WHERE ", zTab
);
1201 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1202 strPrintf(pSql
, "\n) AND ");
1203 strPrintfArray(pSql
, " AND ", "(n.%Q IS NOT NULL)", azCol
, nPK
);
1206 strPrintf(pSql
, "\nUNION ALL\nSELECT ");
1207 strPrintfArray(pSql
, ", ", "%s", azCol
, nPK
);
1209 strPrintf(pSql
, ", ");
1210 strPrintfArray(pSql
, ", ", "NULL", &azCol
[nPK
], -1);
1212 strPrintf(pSql
, ", 1, "); /* Set ota_control to 1 for a delete */
1213 strPrintfArray(pSql
, ", ", "NULL", azCol
, -1);
1214 strPrintf(pSql
, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab
);
1215 strPrintf(pSql
, " SELECT 1 FROM ", zTab
);
1216 strPrintf(pSql
, " aux.%Q AS o WHERE ", zTab
);
1217 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1218 strPrintf(pSql
, "\n) AND ");
1219 strPrintfArray(pSql
, " AND ", "(n.%Q IS NOT NULL)", azCol
, nPK
);
1221 /* Updated rows. If all table columns are part of the primary key, there
1222 ** can be no updates. In this case this part of the compound SELECT can
1223 ** be omitted altogether. */
1225 strPrintf(pSql
, "\nUNION ALL\nSELECT ");
1226 strPrintfArray(pSql
, ", ", "n.%s", azCol
, nPK
);
1227 strPrintf(pSql
, ",\n");
1228 strPrintfArray(pSql
, " ,\n",
1229 " CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol
[nPK
], -1
1233 strPrintf(pSql
, ", '");
1234 strPrintfArray(pSql
, "", ".", azCol
, nPK
);
1235 strPrintf(pSql
, "' ||\n");
1237 strPrintf(pSql
, ",\n");
1239 strPrintfArray(pSql
, " ||\n",
1240 " CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol
[nPK
], -1
1242 strPrintf(pSql
, "\nAS ota_control, ");
1243 strPrintfArray(pSql
, ", ", "NULL", azCol
, nPK
);
1244 strPrintf(pSql
, ",\n");
1245 strPrintfArray(pSql
, " ,\n",
1246 " CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol
[nPK
], -1
1249 strPrintf(pSql
, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab
, zTab
);
1250 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1251 strPrintf(pSql
, " AND ota_control LIKE '%%x%%'");
1254 /* Now add an ORDER BY clause to sort everything by PK. */
1255 strPrintf(pSql
, "\nORDER BY ");
1256 for(i
=1; i
<=nPK
; i
++) strPrintf(pSql
, "%s%d", ((i
>1)?", ":""), i
);
1259 static void rbudiff_one_table(const char *zTab
, FILE *out
){
1260 int bOtaRowid
; /* True to use an ota_rowid column */
1261 int nPK
; /* Number of primary key columns in table */
1262 char **azCol
; /* NULL terminated array of col names */
1265 Str ct
= {0, 0, 0}; /* The "CREATE TABLE data_xxx" statement */
1266 Str sql
= {0, 0, 0}; /* Query to find differences */
1267 Str insert
= {0, 0, 0}; /* First part of output INSERT statement */
1268 sqlite3_stmt
*pStmt
= 0;
1269 int nRow
= 0; /* Total rows in data_xxx table */
1271 /* --rbu mode must use real primary keys. */
1274 /* Check that the schemas of the two tables match. Exit early otherwise. */
1275 checkSchemasMatch(zTab
);
1277 /* Grab the column names and PK details for the table(s). If no usable PK
1278 ** columns are found, bail out early. */
1279 azCol
= columnNames("main", zTab
, &nPK
, &bOtaRowid
);
1281 runtimeError("table %s has no usable PK columns", zTab
);
1283 for(nCol
=0; azCol
[nCol
]; nCol
++);
1285 /* Build and output the CREATE TABLE statement for the data_xxx table */
1286 strPrintf(&ct
, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab
);
1287 if( bOtaRowid
) strPrintf(&ct
, "rbu_rowid, ");
1288 strPrintfArray(&ct
, ", ", "%s", &azCol
[bOtaRowid
], -1);
1289 strPrintf(&ct
, ", rbu_control);");
1291 /* Get the SQL for the query to retrieve data from the two databases */
1292 getRbudiffQuery(zTab
, azCol
, nPK
, bOtaRowid
, &sql
);
1294 /* Build the first part of the INSERT statement output for each row
1295 ** in the data_xxx table. */
1296 strPrintf(&insert
, "INSERT INTO 'data_%q' (", zTab
);
1297 if( bOtaRowid
) strPrintf(&insert
, "rbu_rowid, ");
1298 strPrintfArray(&insert
, ", ", "%s", &azCol
[bOtaRowid
], -1);
1299 strPrintf(&insert
, ", rbu_control) VALUES(");
1301 pStmt
= db_prepare("%s", sql
.z
);
1303 while( sqlite3_step(pStmt
)==SQLITE_ROW
){
1305 /* If this is the first row output, print out the CREATE TABLE
1306 ** statement first. And then set ct.z to NULL so that it is not
1307 ** printed again. */
1309 fprintf(out
, "%s\n", ct
.z
);
1313 /* Output the first part of the INSERT statement */
1314 fprintf(out
, "%s", insert
.z
);
1317 if( sqlite3_column_type(pStmt
, nCol
)==SQLITE_INTEGER
){
1318 for(i
=0; i
<=nCol
; i
++){
1319 if( i
>0 ) fprintf(out
, ", ");
1320 printQuoted(out
, sqlite3_column_value(pStmt
, i
));
1324 int nOtaControl
= sqlite3_column_bytes(pStmt
, nCol
);
1326 zOtaControl
= (char*)sqlite3_malloc(nOtaControl
+1);
1327 memcpy(zOtaControl
, sqlite3_column_text(pStmt
, nCol
), nOtaControl
+1);
1329 for(i
=0; i
<nCol
; i
++){
1332 && sqlite3_column_type(pStmt
, i
)==SQLITE_BLOB
1333 && sqlite3_column_type(pStmt
, nCol
+1+i
)==SQLITE_BLOB
1335 const char *aSrc
= sqlite3_column_blob(pStmt
, nCol
+1+i
);
1336 int nSrc
= sqlite3_column_bytes(pStmt
, nCol
+1+i
);
1337 const char *aFinal
= sqlite3_column_blob(pStmt
, i
);
1338 int nFinal
= sqlite3_column_bytes(pStmt
, i
);
1342 aDelta
= sqlite3_malloc(nFinal
+ 60);
1343 nDelta
= rbuDeltaCreate(aSrc
, nSrc
, aFinal
, nFinal
, aDelta
);
1344 if( nDelta
<nFinal
){
1347 for(j
=0; j
<nDelta
; j
++) fprintf(out
, "%02x", (u8
)aDelta
[j
]);
1349 zOtaControl
[i
-bOtaRowid
] = 'f';
1352 sqlite3_free(aDelta
);
1356 printQuoted(out
, sqlite3_column_value(pStmt
, i
));
1360 fprintf(out
, "'%s'", zOtaControl
);
1361 sqlite3_free(zOtaControl
);
1364 /* And the closing bracket of the insert statement */
1365 fprintf(out
, ");\n");
1368 sqlite3_finalize(pStmt
);
1370 Str cnt
= {0, 0, 0};
1371 strPrintf(&cnt
, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab
, nRow
);
1372 fprintf(out
, "%s\n", cnt
.z
);
1382 ** Display a summary of differences between two versions of the same
1385 ** * Number of rows changed
1386 ** * Number of rows added
1387 ** * Number of rows deleted
1388 ** * Number of identical rows
1390 static void summarize_one_table(const char *zTab
, FILE *out
){
1391 char *zId
= safeId(zTab
); /* Name of table (translated for us in SQL) */
1392 char **az
= 0; /* Columns in main */
1393 char **az2
= 0; /* Columns in aux */
1394 int nPk
; /* Primary key columns in main */
1395 int nPk2
; /* Primary key columns in aux */
1396 int n
= 0; /* Number of columns in main */
1397 int n2
; /* Number of columns in aux */
1398 int i
; /* Loop counter */
1399 const char *zSep
; /* Separator string */
1400 Str sql
; /* Comparison query */
1401 sqlite3_stmt
*pStmt
; /* Query statement to do the diff */
1402 sqlite3_int64 nUpdate
; /* Number of updated rows */
1403 sqlite3_int64 nUnchanged
; /* Number of unmodified rows */
1404 sqlite3_int64 nDelete
; /* Number of deleted rows */
1405 sqlite3_int64 nInsert
; /* Number of inserted rows */
1408 if( sqlite3_table_column_metadata(g
.db
,"aux",zTab
,0,0,0,0,0,0) ){
1409 if( !sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
1410 /* Table missing from second database. */
1411 fprintf(out
, "%s: missing from second database\n", zTab
);
1413 goto end_summarize_one_table
;
1416 if( sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
1417 /* Table missing from source */
1418 fprintf(out
, "%s: missing from first database\n", zTab
);
1419 goto end_summarize_one_table
;
1422 az
= columnNames("main", zTab
, &nPk
, 0);
1423 az2
= columnNames("aux", zTab
, &nPk2
, 0);
1425 for(n
=0; az
[n
]; n
++){
1426 if( sqlite3_stricmp(az
[n
],az2
[n
])!=0 ) break;
1434 /* Schema mismatch */
1435 fprintf(out
, "%s: incompatible schema\n", zTab
);
1436 goto end_summarize_one_table
;
1439 /* Build the comparison query */
1440 for(n2
=n
; az
[n2
]; n2
++){}
1441 strPrintf(&sql
, "SELECT 1, count(*)");
1443 strPrintf(&sql
, ", 0\n");
1446 for(i
=nPk
; az
[i
]; i
++){
1447 strPrintf(&sql
, "%sA.%s IS NOT B.%s", zSep
, az
[i
], az
[i
]);
1450 strPrintf(&sql
, ")\n");
1452 strPrintf(&sql
, " FROM main.%s A, aux.%s B\n", zId
, zId
);
1454 for(i
=0; i
<nPk
; i
++){
1455 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1458 strPrintf(&sql
, " UNION ALL\n");
1459 strPrintf(&sql
, "SELECT 2, count(*), 0\n");
1460 strPrintf(&sql
, " FROM main.%s A\n", zId
);
1461 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId
);
1463 for(i
=0; i
<nPk
; i
++){
1464 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1467 strPrintf(&sql
, ")\n");
1468 strPrintf(&sql
, " UNION ALL\n");
1469 strPrintf(&sql
, "SELECT 3, count(*), 0\n");
1470 strPrintf(&sql
, " FROM aux.%s B\n", zId
);
1471 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId
);
1473 for(i
=0; i
<nPk
; i
++){
1474 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1477 strPrintf(&sql
, ")\n ORDER BY 1;\n");
1479 if( (g
.fDebug
& DEBUG_DIFF_SQL
)!=0 ){
1480 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
1481 goto end_summarize_one_table
;
1484 /* Run the query and output difference summary */
1485 pStmt
= db_prepare("%s", sql
.z
);
1490 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1491 switch( sqlite3_column_int(pStmt
,0) ){
1493 nUpdate
= sqlite3_column_int64(pStmt
,2);
1494 nUnchanged
= sqlite3_column_int64(pStmt
,1) - nUpdate
;
1497 nDelete
= sqlite3_column_int64(pStmt
,1);
1500 nInsert
= sqlite3_column_int64(pStmt
,1);
1504 sqlite3_finalize(pStmt
);
1505 fprintf(out
, "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
1506 zTab
, nUpdate
, nInsert
, nDelete
, nUnchanged
);
1508 end_summarize_one_table
:
1517 ** Write a 64-bit signed integer as a varint onto out
1519 static void putsVarint(FILE *out
, sqlite3_uint64 v
){
1521 unsigned char p
[12];
1522 if( v
& (((sqlite3_uint64
)0xff000000)<<32) ){
1523 p
[8] = (unsigned char)v
;
1525 for(i
=7; i
>=0; i
--){
1526 p
[i
] = (unsigned char)((v
& 0x7f) | 0x80);
1529 fwrite(p
, 8, 1, out
);
1533 p
[n
--] = (unsigned char)((v
& 0x7f) | 0x80);
1537 fwrite(p
+n
+1, 9-n
, 1, out
);
1542 ** Write an SQLite value onto out.
1544 static void putValue(FILE *out
, sqlite3_stmt
*pStmt
, int k
){
1545 int iDType
= sqlite3_column_type(pStmt
, k
);
1553 case SQLITE_INTEGER
:
1554 iX
= sqlite3_column_int64(pStmt
, k
);
1555 memcpy(&uX
, &iX
, 8);
1556 for(j
=56; j
>=0; j
-=8) putc((uX
>>j
)&0xff, out
);
1559 rX
= sqlite3_column_double(pStmt
, k
);
1560 memcpy(&uX
, &rX
, 8);
1561 for(j
=56; j
>=0; j
-=8) putc((uX
>>j
)&0xff, out
);
1564 iX
= sqlite3_column_bytes(pStmt
, k
);
1565 putsVarint(out
, (sqlite3_uint64
)iX
);
1566 fwrite(sqlite3_column_text(pStmt
, k
),1,(size_t)iX
,out
);
1569 iX
= sqlite3_column_bytes(pStmt
, k
);
1570 putsVarint(out
, (sqlite3_uint64
)iX
);
1571 fwrite(sqlite3_column_blob(pStmt
, k
),1,(size_t)iX
,out
);
1579 ** Generate a CHANGESET for all differences from main.zTab to aux.zTab.
1581 static void changeset_one_table(const char *zTab
, FILE *out
){
1582 sqlite3_stmt
*pStmt
; /* SQL statment */
1583 char *zId
= safeId(zTab
); /* Escaped name of the table */
1584 char **azCol
= 0; /* List of escaped column names */
1585 int nCol
= 0; /* Number of columns */
1586 int *aiFlg
= 0; /* 0 if column is not part of PK */
1587 int *aiPk
= 0; /* Column numbers for each PK column */
1588 int nPk
= 0; /* Number of PRIMARY KEY columns */
1589 Str sql
; /* SQL for the diff query */
1590 int i
, k
; /* Loop counters */
1591 const char *zSep
; /* List separator */
1593 /* Check that the schemas of the two tables match. Exit early otherwise. */
1594 checkSchemasMatch(zTab
);
1597 pStmt
= db_prepare("PRAGMA main.table_info=%Q", zTab
);
1598 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1600 azCol
= sqlite3_realloc(azCol
, sizeof(char*)*nCol
);
1601 if( azCol
==0 ) runtimeError("out of memory");
1602 aiFlg
= sqlite3_realloc(aiFlg
, sizeof(int)*nCol
);
1603 if( aiFlg
==0 ) runtimeError("out of memory");
1604 azCol
[nCol
-1] = safeId((const char*)sqlite3_column_text(pStmt
,1));
1605 aiFlg
[nCol
-1] = i
= sqlite3_column_int(pStmt
,5);
1609 aiPk
= sqlite3_realloc(aiPk
, sizeof(int)*nPk
);
1610 if( aiPk
==0 ) runtimeError("out of memory");
1615 sqlite3_finalize(pStmt
);
1616 if( nPk
==0 ) goto end_changeset_one_table
;
1618 strPrintf(&sql
, "SELECT %d", SQLITE_UPDATE
);
1619 for(i
=0; i
<nCol
; i
++){
1621 strPrintf(&sql
, ",\n A.%s", azCol
[i
]);
1623 strPrintf(&sql
, ",\n A.%s IS NOT B.%s, A.%s, B.%s",
1624 azCol
[i
], azCol
[i
], azCol
[i
], azCol
[i
]);
1627 strPrintf(&sql
,"\n FROM main.%s A, aux.%s B\n", zId
, zId
);
1629 for(i
=0; i
<nPk
; i
++){
1630 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1634 for(i
=0; i
<nCol
; i
++){
1635 if( aiFlg
[i
] ) continue;
1636 strPrintf(&sql
, "%sA.%s IS NOT B.%s", zSep
, azCol
[i
], azCol
[i
]);
1639 strPrintf(&sql
,")\n UNION ALL\n");
1641 strPrintf(&sql
, "SELECT %d", SQLITE_DELETE
);
1642 for(i
=0; i
<nCol
; i
++){
1644 strPrintf(&sql
, ",\n A.%s", azCol
[i
]);
1646 strPrintf(&sql
, ",\n 1, A.%s, NULL", azCol
[i
]);
1649 strPrintf(&sql
, "\n FROM main.%s A\n", zId
);
1650 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId
);
1652 for(i
=0; i
<nPk
; i
++){
1653 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1656 strPrintf(&sql
, ")\n UNION ALL\n");
1657 strPrintf(&sql
, "SELECT %d", SQLITE_INSERT
);
1658 for(i
=0; i
<nCol
; i
++){
1660 strPrintf(&sql
, ",\n B.%s", azCol
[i
]);
1662 strPrintf(&sql
, ",\n 1, NULL, B.%s", azCol
[i
]);
1665 strPrintf(&sql
, "\n FROM aux.%s B\n", zId
);
1666 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId
);
1668 for(i
=0; i
<nPk
; i
++){
1669 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1672 strPrintf(&sql
, ")\n");
1673 strPrintf(&sql
, " ORDER BY");
1675 for(i
=0; i
<nPk
; i
++){
1676 strPrintf(&sql
, "%s %d", zSep
, aiPk
[i
]+2);
1679 strPrintf(&sql
, ";\n");
1681 if( g
.fDebug
& DEBUG_DIFF_SQL
){
1682 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
1683 goto end_changeset_one_table
;
1687 putsVarint(out
, (sqlite3_uint64
)nCol
);
1688 for(i
=0; i
<nCol
; i
++) putc(aiFlg
[i
], out
);
1689 fwrite(zTab
, 1, strlen(zTab
), out
);
1692 pStmt
= db_prepare("%s", sql
.z
);
1693 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1694 int iType
= sqlite3_column_int(pStmt
,0);
1697 switch( sqlite3_column_int(pStmt
,0) ){
1698 case SQLITE_UPDATE
: {
1699 for(k
=1, i
=0; i
<nCol
; i
++){
1701 putValue(out
, pStmt
, k
);
1703 }else if( sqlite3_column_int(pStmt
,k
) ){
1704 putValue(out
, pStmt
, k
+1);
1711 for(k
=1, i
=0; i
<nCol
; i
++){
1715 }else if( sqlite3_column_int(pStmt
,k
) ){
1716 putValue(out
, pStmt
, k
+2);
1725 case SQLITE_INSERT
: {
1726 for(k
=1, i
=0; i
<nCol
; i
++){
1728 putValue(out
, pStmt
, k
);
1731 putValue(out
, pStmt
, k
+2);
1737 case SQLITE_DELETE
: {
1738 for(k
=1, i
=0; i
<nCol
; i
++){
1740 putValue(out
, pStmt
, k
);
1743 putValue(out
, pStmt
, k
+1);
1751 sqlite3_finalize(pStmt
);
1753 end_changeset_one_table
:
1754 while( nCol
>0 ) sqlite3_free(azCol
[--nCol
]);
1755 sqlite3_free(azCol
);
1758 sqlite3_free(aiFlg
);
1763 ** Return true if the ascii character passed as the only argument is a
1764 ** whitespace character. Otherwise return false.
1766 static int is_whitespace(char x
){
1767 return (x
==' ' || x
=='\t' || x
=='\n' || x
=='\r');
1771 ** Extract the next SQL keyword or quoted string from buffer zIn and copy it
1772 ** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
1773 ** Return a pointer to the character within zIn immediately following
1774 ** the token or quoted string just extracted.
1776 static const char *gobble_token(const char *zIn
, char *zBuf
, int nBuf
){
1777 const char *p
= zIn
;
1779 char *pEnd
= &pOut
[nBuf
-1];
1780 char q
= 0; /* quote character, if any */
1782 if( p
==0 ) return 0;
1783 while( is_whitespace(*p
) ) p
++;
1785 case '"': q
= '"'; break;
1786 case '\'': q
= '\''; break;
1787 case '`': q
= '`'; break;
1788 case '[': q
= ']'; break;
1793 while( *p
&& pOut
<pEnd
){
1798 if( pOut
<pEnd
) *pOut
++ = *p
;
1802 while( *p
&& !is_whitespace(*p
) && *p
!='(' ){
1803 if( pOut
<pEnd
) *pOut
++ = *p
;
1813 ** This function is the implementation of SQL scalar function "module_name":
1817 ** The only argument should be an SQL statement of the type that may appear
1818 ** in the sqlite_schema table. If the statement is a "CREATE VIRTUAL TABLE"
1819 ** statement, then the value returned is the name of the module that it
1820 ** uses. Otherwise, if the statement is not a CVT, NULL is returned.
1822 static void module_name_func(
1823 sqlite3_context
*pCtx
,
1824 int nVal
, sqlite3_value
**apVal
1830 zSql
= (const char*)sqlite3_value_text(apVal
[0]);
1832 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1833 if( zSql
==0 || sqlite3_stricmp(zToken
, "create") ) return;
1834 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1835 if( zSql
==0 || sqlite3_stricmp(zToken
, "virtual") ) return;
1836 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1837 if( zSql
==0 || sqlite3_stricmp(zToken
, "table") ) return;
1838 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1839 if( zSql
==0 ) return;
1840 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1841 if( zSql
==0 || sqlite3_stricmp(zToken
, "using") ) return;
1842 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1844 sqlite3_result_text(pCtx
, zToken
, -1, SQLITE_TRANSIENT
);
1848 ** Return the text of an SQL statement that itself returns the list of
1849 ** tables to process within the database.
1851 const char *all_tables_sql(){
1852 if( g
.bHandleVtab
){
1855 rc
= sqlite3_exec(g
.db
,
1856 "CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);"
1857 "INSERT INTO temp.tblmap VALUES"
1858 "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
1860 "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
1861 "('fts4', '_docsize'), ('fts4', '_stat'),"
1863 "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
1864 "('fts5', '_docsize'), ('fts5', '_config'),"
1866 "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
1869 assert( rc
==SQLITE_OK
);
1871 rc
= sqlite3_create_function(
1872 g
.db
, "module_name", 1, SQLITE_UTF8
, 0, module_name_func
, 0, 0
1874 assert( rc
==SQLITE_OK
);
1877 "SELECT name FROM main.sqlite_schema\n"
1878 " WHERE type='table' AND (\n"
1879 " module_name(sql) IS NULL OR \n"
1880 " module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1881 " ) AND name NOT IN (\n"
1882 " SELECT a.name || b.postfix \n"
1883 "FROM main.sqlite_schema AS a, temp.tblmap AS b \n"
1884 "WHERE module_name(a.sql) = b.module\n"
1887 "SELECT name FROM aux.sqlite_schema\n"
1888 " WHERE type='table' AND (\n"
1889 " module_name(sql) IS NULL OR \n"
1890 " module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1891 " ) AND name NOT IN (\n"
1892 " SELECT a.name || b.postfix \n"
1893 "FROM aux.sqlite_schema AS a, temp.tblmap AS b \n"
1894 "WHERE module_name(a.sql) = b.module\n"
1899 "SELECT name FROM main.sqlite_schema\n"
1900 " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1902 "SELECT name FROM aux.sqlite_schema\n"
1903 " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1909 ** Print sketchy documentation for this utility program
1911 static void showHelp(void){
1912 printf("Usage: %s [options] DB1 DB2\n", g
.zArgv0
);
1914 "Output SQL text that would transform DB1 into DB2.\n"
1916 " --changeset FILE Write a CHANGESET into FILE\n"
1917 " -L|--lib LIBRARY Load an SQLite extension library\n"
1918 " --primarykey Use schema-defined PRIMARY KEYs\n"
1919 " --rbu Output SQL to create/populate RBU table(s)\n"
1920 " --schema Show only differences in the schema\n"
1921 " --summary Show only a summary of the differences\n"
1922 " --table TAB Show only differences in table TAB\n"
1923 " --transaction Show SQL output inside a transaction\n"
1924 " --vtab Handle fts3, fts4, fts5 and rtree tables\n"
1925 "See https://sqlite.org/sqldiff.html for detailed explanation.\n"
1929 int main(int argc
, char **argv
){
1930 const char *zDb1
= 0;
1931 const char *zDb2
= 0;
1936 sqlite3_stmt
*pStmt
;
1939 void (*xDiff
)(const char*,FILE*) = diff_one_table
;
1940 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1944 int useTransaction
= 0;
1945 int neverUseTransaction
= 0;
1948 sqlite3_config(SQLITE_CONFIG_SINGLETHREAD
);
1949 for(i
=1; i
<argc
; i
++){
1950 const char *z
= argv
[i
];
1953 if( z
[0]=='-' ) z
++;
1954 if( strcmp(z
,"changeset")==0 ){
1955 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1956 out
= fopen(argv
[++i
], "wb");
1957 if( out
==0 ) cmdlineError("cannot open: %s", argv
[i
]);
1958 xDiff
= changeset_one_table
;
1959 neverUseTransaction
= 1;
1961 if( strcmp(z
,"debug")==0 ){
1962 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1963 g
.fDebug
= strtol(argv
[++i
], 0, 0);
1965 if( strcmp(z
,"help")==0 ){
1969 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1970 if( strcmp(z
,"lib")==0 || strcmp(z
,"L")==0 ){
1971 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1972 azExt
= realloc(azExt
, sizeof(azExt
[0])*(nExt
+1));
1973 if( azExt
==0 ) cmdlineError("out of memory");
1974 azExt
[nExt
++] = argv
[++i
];
1977 if( strcmp(z
,"primarykey")==0 ){
1980 if( strcmp(z
,"rbu")==0 ){
1981 xDiff
= rbudiff_one_table
;
1983 if( strcmp(z
,"schema")==0 ){
1986 if( strcmp(z
,"summary")==0 ){
1987 xDiff
= summarize_one_table
;
1989 if( strcmp(z
,"table")==0 ){
1990 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1993 sqlite3_stricmp(zTab
, "sqlite_schema")==0
1994 || sqlite3_stricmp(zTab
, "sqlite_master")==0;
1996 if( strcmp(z
,"transaction")==0 ){
1999 if( strcmp(z
,"vtab")==0 ){
2003 cmdlineError("unknown option: %s", argv
[i
]);
2005 }else if( zDb1
==0 ){
2007 }else if( zDb2
==0 ){
2010 cmdlineError("unknown argument: %s", argv
[i
]);
2014 cmdlineError("two database arguments required");
2016 if( g
.bSchemaOnly
&& g
.bSchemaCompare
){
2017 cmdlineError("The --schema option is useless with --table %s .", zTab
);
2019 rc
= sqlite3_open(zDb1
, &g
.db
);
2021 cmdlineError("cannot open database file \"%s\"", zDb1
);
2023 rc
= sqlite3_exec(g
.db
, "SELECT * FROM sqlite_schema", 0, 0, &zErrMsg
);
2024 if( rc
|| zErrMsg
){
2025 cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1
);
2027 #ifndef SQLITE_OMIT_LOAD_EXTENSION
2028 sqlite3_enable_load_extension(g
.db
, 1);
2029 for(i
=0; i
<nExt
; i
++){
2030 rc
= sqlite3_load_extension(g
.db
, azExt
[i
], 0, &zErrMsg
);
2031 if( rc
|| zErrMsg
){
2032 cmdlineError("error loading %s: %s", azExt
[i
], zErrMsg
);
2037 zSql
= sqlite3_mprintf("ATTACH %Q as aux;", zDb2
);
2038 rc
= sqlite3_exec(g
.db
, zSql
, 0, 0, &zErrMsg
);
2041 if( rc
|| zErrMsg
){
2042 cmdlineError("cannot attach database \"%s\"", zDb2
);
2044 rc
= sqlite3_exec(g
.db
, "SELECT * FROM aux.sqlite_schema", 0, 0, &zErrMsg
);
2045 if( rc
|| zErrMsg
){
2046 cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2
);
2049 if( neverUseTransaction
) useTransaction
= 0;
2050 if( useTransaction
) fprintf(out
, "BEGIN TRANSACTION;\n");
2051 if( xDiff
==rbudiff_one_table
){
2052 fprintf(out
, "CREATE TABLE IF NOT EXISTS rbu_count"
2053 "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
2060 /* Handle tables one by one */
2061 pStmt
= db_prepare("%s", all_tables_sql() );
2062 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
2063 xDiff((const char*)sqlite3_column_text(pStmt
,0), out
);
2065 sqlite3_finalize(pStmt
);
2067 if( useTransaction
) printf("COMMIT;\n");
2069 /* TBD: Handle trigger differences */
2070 /* TBD: Handle view differences */
2071 sqlite3_close(g
.db
);