4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
13 ** This is a utility program that computes the differences in content
14 ** between two SQLite databases.
16 ** To compile, simply link against SQLite.
18 ** See the showHelp() routine below for a brief description of how to
30 ** All global variables are gathered into the "g" singleton.
33 const char *zArgv0
; /* Name of program */
34 int bSchemaOnly
; /* Only show schema differences */
35 int bSchemaPK
; /* Use the schema-defined PK, not the true PK */
36 int bHandleVtab
; /* Handle fts3, fts4, fts5 and rtree vtabs */
37 unsigned fDebug
; /* Debug flags */
38 sqlite3
*db
; /* The database connection */
42 ** Allowed values for g.fDebug
44 #define DEBUG_COLUMN_NAMES 0x000001
45 #define DEBUG_DIFF_SQL 0x000002
48 ** Dynamic string object
50 typedef struct Str Str
;
52 char *z
; /* Text of the string */
53 int nAlloc
; /* Bytes allocated in z[] */
54 int nUsed
; /* Bytes actually used in z[] */
58 ** Initialize a Str object
60 static void strInit(Str
*p
){
67 ** Print an error resulting from faulting command-line arguments and
70 static void cmdlineError(const char *zFormat
, ...){
72 fprintf(stderr
, "%s: ", g
.zArgv0
);
73 va_start(ap
, zFormat
);
74 vfprintf(stderr
, zFormat
, ap
);
76 fprintf(stderr
, "\n\"%s --help\" for more help\n", g
.zArgv0
);
81 ** Print an error message for an error that occurs at runtime, then
84 static void runtimeError(const char *zFormat
, ...){
86 fprintf(stderr
, "%s: ", g
.zArgv0
);
87 va_start(ap
, zFormat
);
88 vfprintf(stderr
, zFormat
, ap
);
90 fprintf(stderr
, "\n");
95 ** Free all memory held by a Str object
97 static void strFree(Str
*p
){
103 ** Add formatted text to the end of a Str object
105 static void strPrintf(Str
*p
, const char *zFormat
, ...){
110 va_start(ap
, zFormat
);
111 sqlite3_vsnprintf(p
->nAlloc
-p
->nUsed
, p
->z
+p
->nUsed
, zFormat
, ap
);
113 nNew
= (int)strlen(p
->z
+ p
->nUsed
);
117 if( p
->nUsed
+nNew
< p
->nAlloc
-1 ){
121 p
->nAlloc
= p
->nAlloc
*2 + 1000;
122 p
->z
= sqlite3_realloc(p
->z
, p
->nAlloc
);
123 if( p
->z
==0 ) runtimeError("out of memory");
129 /* Safely quote an SQL identifier. Use the minimum amount of transformation
130 ** necessary to allow the string to be used with %s.
132 ** Space to hold the returned string is obtained from sqlite3_malloc(). The
133 ** caller is responsible for ensuring this space is freed when no longer
136 static char *safeId(const char *zId
){
139 if( zId
[0]==0 ) return sqlite3_mprintf("\"\"");
140 for(i
=x
=0; (c
= zId
[i
])!=0; i
++){
141 if( !isalpha(c
) && c
!='_' ){
142 if( i
>0 && isdigit(c
) ){
145 return sqlite3_mprintf("\"%w\"", zId
);
149 if( x
|| !sqlite3_keyword_check(zId
,i
) ){
150 return sqlite3_mprintf("%s", zId
);
152 return sqlite3_mprintf("\"%w\"", zId
);
156 ** Prepare a new SQL statement. Print an error and abort if anything
159 static sqlite3_stmt
*db_vprepare(const char *zFormat
, va_list ap
){
164 zSql
= sqlite3_vmprintf(zFormat
, ap
);
165 if( zSql
==0 ) runtimeError("out of memory");
166 rc
= sqlite3_prepare_v2(g
.db
, zSql
, -1, &pStmt
, 0);
168 runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g
.db
),
174 static sqlite3_stmt
*db_prepare(const char *zFormat
, ...){
177 va_start(ap
, zFormat
);
178 pStmt
= db_vprepare(zFormat
, ap
);
184 ** Free a list of strings
186 static void namelistFree(char **az
){
189 for(i
=0; az
[i
]; i
++) sqlite3_free(az
[i
]);
195 ** Return a list of column names for the table zDb.zTab. Space to
196 ** hold the list is obtained from sqlite3_malloc() and should released
197 ** using namelistFree() when no longer needed.
199 ** Primary key columns are listed first, followed by data columns.
200 ** The number of columns in the primary key is returned in *pnPkey.
202 ** Normally, the "primary key" in the previous sentence is the true
203 ** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables
204 ** or the declared PRIMARY KEY for WITHOUT ROWID tables. However, if
205 ** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is
206 ** used in all cases. In that case, entries that have NULL values in
207 ** any of their primary key fields will be excluded from the analysis.
209 ** If the primary key for a table is the rowid but rowid is inaccessible,
210 ** then this routine returns a NULL pointer.
213 ** CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c));
215 ** az = { "rowid", "a", "b", "c", 0 } // Normal case
216 ** az = { "c", "a", "b", 0 } // g.bSchemaPK==1
218 ** CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b));
220 ** az = { "b", "a", "c", 0 }
222 ** CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z));
223 ** *pnPKey = 1 // Normal case
224 ** az = { "rowid", "x", "y", "z", 0 } // Normal case
225 ** *pnPKey = 2 // g.bSchemaPK==1
226 ** az = { "y", "x", "z", 0 } // g.bSchemaPK==1
228 ** CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID;
230 ** az = { "y", "z", "x", 0 }
232 ** CREATE TABLE t5(rowid,_rowid_,oid);
233 ** az = 0 // The rowid is not accessible
235 static char **columnNames(
236 const char *zDb
, /* Database ("main" or "aux") to query */
237 const char *zTab
, /* Name of table to return details of */
238 int *pnPKey
, /* OUT: Number of PK columns */
239 int *pbRowid
/* OUT: True if PK is an implicit rowid */
241 char **az
= 0; /* List of column names to be returned */
242 int naz
= 0; /* Number of entries in az[] */
243 sqlite3_stmt
*pStmt
; /* SQL statement being run */
244 char *zPkIdxName
= 0; /* Name of the PRIMARY KEY index */
245 int truePk
= 0; /* PRAGMA table_info indentifies the PK to use */
246 int nPK
= 0; /* Number of PRIMARY KEY columns */
247 int i
, j
; /* Loop counters */
249 if( g
.bSchemaPK
==0 ){
250 /* Normal case: Figure out what the true primary key is for the table.
251 ** * For WITHOUT ROWID tables, the true primary key is the same as
252 ** the schema PRIMARY KEY, which is guaranteed to be present.
253 ** * For rowid tables with an INTEGER PRIMARY KEY, the true primary
254 ** key is the INTEGER PRIMARY KEY.
255 ** * For all other rowid tables, the rowid is the true primary key.
257 pStmt
= db_prepare("PRAGMA %s.index_list=%Q", zDb
, zTab
);
258 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
259 if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt
,3),"pk")==0 ){
260 zPkIdxName
= sqlite3_mprintf("%s", sqlite3_column_text(pStmt
, 1));
264 sqlite3_finalize(pStmt
);
269 pStmt
= db_prepare("PRAGMA %s.index_xinfo=%Q", zDb
, zPkIdxName
);
270 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
272 if( sqlite3_column_int(pStmt
,5) ){ nKey
++; continue; }
273 if( sqlite3_column_int(pStmt
,1)>=0 ) truePk
= 1;
275 if( nCol
==nKey
) truePk
= 1;
281 sqlite3_finalize(pStmt
);
282 sqlite3_free(zPkIdxName
);
287 pStmt
= db_prepare("PRAGMA %s.table_info=%Q", zDb
, zTab
);
289 /* The g.bSchemaPK==1 case: Use whatever primary key is declared
290 ** in the schema. The "rowid" will still be used as the primary key
291 ** if the table definition does not contain a PRIMARY KEY.
294 pStmt
= db_prepare("PRAGMA %s.table_info=%Q", zDb
, zTab
);
295 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
296 if( sqlite3_column_int(pStmt
,5)>0 ) nPK
++;
298 sqlite3_reset(pStmt
);
299 if( nPK
==0 ) nPK
= 1;
304 az
= sqlite3_malloc( sizeof(char*)*(nPK
+1) );
305 if( az
==0 ) runtimeError("out of memory");
306 memset(az
, 0, sizeof(char*)*(nPK
+1));
307 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
309 if( truePk
&& (iPKey
= sqlite3_column_int(pStmt
,5))>0 ){
310 az
[iPKey
-1] = safeId((char*)sqlite3_column_text(pStmt
,1));
312 az
= sqlite3_realloc(az
, sizeof(char*)*(naz
+2) );
313 if( az
==0 ) runtimeError("out of memory");
314 az
[naz
++] = safeId((char*)sqlite3_column_text(pStmt
,1));
317 sqlite3_finalize(pStmt
);
318 if( az
) az
[naz
] = 0;
320 /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of
321 ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0). */
322 if( pbRowid
) *pbRowid
= (az
[0]==0);
324 /* If this table has an implicit rowid for a PK, figure out how to refer
325 ** to it. There are three options - "rowid", "_rowid_" and "oid". Any
326 ** of these will work, unless the table has an explicit column of the
329 const char *azRowid
[] = { "rowid", "_rowid_", "oid" };
330 for(i
=0; i
<sizeof(azRowid
)/sizeof(azRowid
[0]); i
++){
331 for(j
=1; j
<naz
; j
++){
332 if( sqlite3_stricmp(az
[j
], azRowid
[i
])==0 ) break;
335 az
[0] = sqlite3_mprintf("%s", azRowid
[i
]);
340 for(i
=1; i
<naz
; i
++) sqlite3_free(az
[i
]);
349 ** Print the sqlite3_value X as an SQL literal.
351 static void printQuoted(FILE *out
, sqlite3_value
*X
){
352 switch( sqlite3_value_type(X
) ){
356 r1
= sqlite3_value_double(X
);
357 sqlite3_snprintf(sizeof(zBuf
), zBuf
, "%!.15g", r1
);
358 fprintf(out
, "%s", zBuf
);
361 case SQLITE_INTEGER
: {
362 fprintf(out
, "%lld", sqlite3_value_int64(X
));
366 const unsigned char *zBlob
= sqlite3_value_blob(X
);
367 int nBlob
= sqlite3_value_bytes(X
);
371 for(i
=0; i
<nBlob
; i
++){
372 fprintf(out
, "%02x", zBlob
[i
]);
376 /* Could be an OOM, could be a zero-byte blob */
382 const unsigned char *zArg
= sqlite3_value_text(X
);
386 fprintf(out
, "NULL");
389 for(i
=j
=0; zArg
[i
]; i
++){
391 fprintf(out
, "%.*s'", i
-j
+1, &zArg
[j
]);
395 fprintf(out
, "%s'", &zArg
[j
]);
400 fprintf(out
, "NULL");
407 ** Output SQL that will recreate the aux.zTab table.
409 static void dump_table(const char *zTab
, FILE *out
){
410 char *zId
= safeId(zTab
); /* Name of the table */
411 char **az
= 0; /* List of columns */
412 int nPk
; /* Number of true primary key columns */
413 int nCol
; /* Number of data columns */
414 int i
; /* Loop counter */
415 sqlite3_stmt
*pStmt
; /* SQL statement */
416 const char *zSep
; /* Separator string */
417 Str ins
; /* Beginning of the INSERT statement */
419 pStmt
= db_prepare("SELECT sql FROM aux.sqlite_master WHERE name=%Q", zTab
);
420 if( SQLITE_ROW
==sqlite3_step(pStmt
) ){
421 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
423 sqlite3_finalize(pStmt
);
424 if( !g
.bSchemaOnly
){
425 az
= columnNames("aux", zTab
, &nPk
, 0);
428 pStmt
= db_prepare("SELECT * FROM aux.%s", zId
);
429 strPrintf(&ins
,"INSERT INTO %s VALUES", zId
);
434 for(i
=0; az
[i
]; i
++){
435 strPrintf(&sql
, "%s %s", zSep
, az
[i
]);
438 strPrintf(&sql
," FROM aux.%s", zId
);
440 for(i
=1; i
<=nPk
; i
++){
441 strPrintf(&sql
, "%s %d", zSep
, i
);
444 pStmt
= db_prepare("%s", sql
.z
);
446 strPrintf(&ins
, "INSERT INTO %s", zId
);
448 for(i
=0; az
[i
]; i
++){
449 strPrintf(&ins
, "%s%s", zSep
, az
[i
]);
452 strPrintf(&ins
,") VALUES");
455 nCol
= sqlite3_column_count(pStmt
);
456 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
457 fprintf(out
, "%s",ins
.z
);
459 for(i
=0; i
<nCol
; i
++){
460 fprintf(out
, "%s",zSep
);
461 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
464 fprintf(out
, ");\n");
466 sqlite3_finalize(pStmt
);
468 } /* endif !g.bSchemaOnly */
469 pStmt
= db_prepare("SELECT sql FROM aux.sqlite_master"
470 " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
472 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
473 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
475 sqlite3_finalize(pStmt
);
480 ** Compute all differences for a single table.
482 static void diff_one_table(const char *zTab
, FILE *out
){
483 char *zId
= safeId(zTab
); /* Name of table (translated for us in SQL) */
484 char **az
= 0; /* Columns in main */
485 char **az2
= 0; /* Columns in aux */
486 int nPk
; /* Primary key columns in main */
487 int nPk2
; /* Primary key columns in aux */
488 int n
= 0; /* Number of columns in main */
489 int n2
; /* Number of columns in aux */
490 int nQ
; /* Number of output columns in the diff query */
491 int i
; /* Loop counter */
492 const char *zSep
; /* Separator string */
493 Str sql
; /* Comparison query */
494 sqlite3_stmt
*pStmt
; /* Query statement to do the diff */
497 if( g
.fDebug
==DEBUG_COLUMN_NAMES
){
498 /* Simply run columnNames() on all tables of the origin
499 ** database and show the results. This is used for testing
500 ** and debugging of the columnNames() function.
502 az
= columnNames("aux",zTab
, &nPk
, 0);
504 printf("Rowid not accessible for %s\n", zId
);
507 for(i
=0; az
[i
]; i
++){
508 printf(" %s", az
[i
]);
509 if( i
+1==nPk
) printf(" *");
513 goto end_diff_one_table
;
517 if( sqlite3_table_column_metadata(g
.db
,"aux",zTab
,0,0,0,0,0,0) ){
518 if( !sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
519 /* Table missing from second database. */
520 fprintf(out
, "DROP TABLE %s;\n", zId
);
522 goto end_diff_one_table
;
525 if( sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
526 /* Table missing from source */
527 dump_table(zTab
, out
);
528 goto end_diff_one_table
;
531 az
= columnNames("main", zTab
, &nPk
, 0);
532 az2
= columnNames("aux", zTab
, &nPk2
, 0);
534 for(n
=0; az
[n
] && az2
[n
]; n
++){
535 if( sqlite3_stricmp(az
[n
],az2
[n
])!=0 ) break;
543 /* Schema mismatch */
544 fprintf(out
, "DROP TABLE %s; -- due to schema mismatch\n", zId
);
545 dump_table(zTab
, out
);
546 goto end_diff_one_table
;
549 /* Build the comparison query */
550 for(n2
=n
; az2
[n2
]; n2
++){
551 fprintf(out
, "ALTER TABLE %s ADD COLUMN %s;\n", zId
, safeId(az2
[n2
]));
553 nQ
= nPk2
+1+2*(n2
-nPk2
);
556 for(i
=0; i
<nPk
; i
++){
557 strPrintf(&sql
, "%sB.%s", zSep
, az
[i
]);
560 strPrintf(&sql
, ", 1%s -- changed row\n", nPk
==n
? "" : ",");
562 strPrintf(&sql
, " A.%s IS NOT B.%s, B.%s%s\n",
563 az
[i
], az2
[i
], az2
[i
], az2
[i
+1]==0 ? "" : ",");
567 strPrintf(&sql
, " B.%s IS NOT NULL, B.%s%s\n",
568 az2
[i
], az2
[i
], az2
[i
+1]==0 ? "" : ",");
571 strPrintf(&sql
, " FROM main.%s A, aux.%s B\n", zId
, zId
);
573 for(i
=0; i
<nPk
; i
++){
574 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
579 strPrintf(&sql
, "%sA.%s IS NOT B.%s%s\n",
580 zSep
, az
[i
], az2
[i
], az2
[i
+1]==0 ? ")" : "");
585 strPrintf(&sql
, "%sB.%s IS NOT NULL%s\n",
586 zSep
, az2
[i
], az2
[i
+1]==0 ? ")" : "");
590 strPrintf(&sql
, " UNION ALL\n");
593 for(i
=0; i
<nPk
; i
++){
594 strPrintf(&sql
, "%sA.%s", zSep
, az
[i
]);
597 strPrintf(&sql
, ", 2%s -- deleted row\n", nPk
==n
? "" : ",");
599 strPrintf(&sql
, " NULL, NULL%s\n", i
==n2
-1 ? "" : ",");
602 strPrintf(&sql
, " FROM main.%s A\n", zId
);
603 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId
);
605 for(i
=0; i
<nPk
; i
++){
606 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
609 strPrintf(&sql
, ")\n");
610 zSep
= " UNION ALL\nSELECT ";
611 for(i
=0; i
<nPk
; i
++){
612 strPrintf(&sql
, "%sB.%s", zSep
, az
[i
]);
615 strPrintf(&sql
, ", 3%s -- inserted row\n", nPk
==n
? "" : ",");
617 strPrintf(&sql
, " 1, B.%s%s\n", az2
[i
], az2
[i
+1]==0 ? "" : ",");
620 strPrintf(&sql
, " FROM aux.%s B\n", zId
);
621 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId
);
623 for(i
=0; i
<nPk
; i
++){
624 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
627 strPrintf(&sql
, ")\n ORDER BY");
629 for(i
=1; i
<=nPk
; i
++){
630 strPrintf(&sql
, "%s%d", zSep
, i
);
633 strPrintf(&sql
, ";\n");
635 if( g
.fDebug
& DEBUG_DIFF_SQL
){
636 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
637 goto end_diff_one_table
;
640 /* Drop indexes that are missing in the destination */
642 "SELECT name FROM main.sqlite_master"
643 " WHERE type='index' AND tbl_name=%Q"
644 " AND sql IS NOT NULL"
645 " AND sql NOT IN (SELECT sql FROM aux.sqlite_master"
646 " WHERE type='index' AND tbl_name=%Q"
647 " AND sql IS NOT NULL)",
649 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
650 char *z
= safeId((const char*)sqlite3_column_text(pStmt
,0));
651 fprintf(out
, "DROP INDEX %s;\n", z
);
654 sqlite3_finalize(pStmt
);
656 /* Run the query and output differences */
657 if( !g
.bSchemaOnly
){
658 pStmt
= db_prepare("%s", sql
.z
);
659 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
660 int iType
= sqlite3_column_int(pStmt
, nPk
);
661 if( iType
==1 || iType
==2 ){
662 if( iType
==1 ){ /* Change the content of a row */
663 fprintf(out
, "UPDATE %s", zId
);
665 for(i
=nPk
+1; i
<nQ
; i
+=2){
666 if( sqlite3_column_int(pStmt
,i
)==0 ) continue;
667 fprintf(out
, "%s %s=", zSep
, az2
[(i
+nPk
-1)/2]);
669 printQuoted(out
, sqlite3_column_value(pStmt
,i
+1));
671 }else{ /* Delete a row */
672 fprintf(out
, "DELETE FROM %s", zId
);
675 for(i
=0; i
<nPk
; i
++){
676 fprintf(out
, "%s %s=", zSep
, az2
[i
]);
677 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
681 }else{ /* Insert a row */
682 fprintf(out
, "INSERT INTO %s(%s", zId
, az2
[0]);
683 for(i
=1; az2
[i
]; i
++) fprintf(out
, ",%s", az2
[i
]);
684 fprintf(out
, ") VALUES");
686 for(i
=0; i
<nPk2
; i
++){
687 fprintf(out
, "%s", zSep
);
689 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
691 for(i
=nPk2
+2; i
<nQ
; i
+=2){
693 printQuoted(out
, sqlite3_column_value(pStmt
,i
));
695 fprintf(out
, ");\n");
698 sqlite3_finalize(pStmt
);
699 } /* endif !g.bSchemaOnly */
701 /* Create indexes that are missing in the source */
703 "SELECT sql FROM aux.sqlite_master"
704 " WHERE type='index' AND tbl_name=%Q"
705 " AND sql IS NOT NULL"
706 " AND sql NOT IN (SELECT sql FROM main.sqlite_master"
707 " WHERE type='index' AND tbl_name=%Q"
708 " AND sql IS NOT NULL)",
710 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
711 fprintf(out
, "%s;\n", sqlite3_column_text(pStmt
,0));
713 sqlite3_finalize(pStmt
);
724 ** Check that table zTab exists and has the same schema in both the "main"
725 ** and "aux" databases currently opened by the global db handle. If they
726 ** do not, output an error message on stderr and exit(1). Otherwise, if
727 ** the schemas do match, return control to the caller.
729 static void checkSchemasMatch(const char *zTab
){
730 sqlite3_stmt
*pStmt
= db_prepare(
731 "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B"
732 " WHERE A.name=%Q AND B.name=%Q", zTab
, zTab
734 if( SQLITE_ROW
==sqlite3_step(pStmt
) ){
735 if( sqlite3_column_int(pStmt
,0)==0 ){
736 runtimeError("schema changes for table %s", safeId(zTab
));
739 runtimeError("table %s missing from one or both databases", safeId(zTab
));
741 sqlite3_finalize(pStmt
);
744 /**************************************************************************
745 ** The following code is copied from fossil. It is used to generate the
746 ** fossil delta blobs sometimes used in RBU update records.
749 typedef unsigned short u16
;
750 typedef unsigned int u32
;
751 typedef unsigned char u8
;
754 ** The width of a hash window in bytes. The algorithm only works if this
760 ** The current state of the rolling hash.
762 ** z[] holds the values that have been hashed. z[] is a circular buffer.
763 ** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of
766 ** Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted
767 ** sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1.
768 ** (Each index for z[] should be module NHASH, of course. The %NHASH operator
769 ** is omitted in the prior expression for brevity.)
771 typedef struct hash hash
;
773 u16 a
, b
; /* Hash values */
774 u16 i
; /* Start of the hash window */
775 char z
[NHASH
]; /* The values that have been hashed */
779 ** Initialize the rolling hash using the first NHASH characters of z[]
781 static void hash_init(hash
*pHash
, const char *z
){
784 for(i
=0; i
<NHASH
; i
++){
789 pHash
->a
= a
& 0xffff;
790 pHash
->b
= b
& 0xffff;
795 ** Advance the rolling hash by a single character "c"
797 static void hash_next(hash
*pHash
, int c
){
798 u16 old
= pHash
->z
[pHash
->i
];
799 pHash
->z
[pHash
->i
] = (char)c
;
800 pHash
->i
= (pHash
->i
+1)&(NHASH
-1);
801 pHash
->a
= pHash
->a
- old
+ (char)c
;
802 pHash
->b
= pHash
->b
- NHASH
*old
+ pHash
->a
;
806 ** Return a 32-bit hash value
808 static u32
hash_32bit(hash
*pHash
){
809 return (pHash
->a
& 0xffff) | (((u32
)(pHash
->b
& 0xffff))<<16);
813 ** Write an base-64 integer into the given buffer.
815 static void putInt(unsigned int v
, char **pz
){
816 static const char zDigits
[] =
817 "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~";
818 /* 123456789 123456789 123456789 123456789 123456789 123456789 123 */
825 for(i
=0; v
>0; i
++, v
>>=6){
826 zBuf
[i
] = zDigits
[v
&0x3f];
828 for(j
=i
-1; j
>=0; j
--){
834 ** Return the number digits in the base-64 representation of a positive integer
836 static int digit_count(int v
){
838 for(i
=1, x
=64; (unsigned int)v
>=x
; i
++, x
<<= 6){}
843 ** Compute a 32-bit checksum on the N-byte buffer. Return the result.
845 static unsigned int checksum(const char *zIn
, size_t N
){
846 const unsigned char *z
= (const unsigned char *)zIn
;
852 sum0
+= ((unsigned)z
[0] + z
[4] + z
[8] + z
[12]);
853 sum1
+= ((unsigned)z
[1] + z
[5] + z
[9] + z
[13]);
854 sum2
+= ((unsigned)z
[2] + z
[6] + z
[10]+ z
[14]);
855 sum3
+= ((unsigned)z
[3] + z
[7] + z
[11]+ z
[15]);
867 sum3
+= (sum2
<< 8) + (sum1
<< 16) + (sum0
<< 24);
869 case 3: sum3
+= (z
[2] << 8);
870 case 2: sum3
+= (z
[1] << 16);
871 case 1: sum3
+= (z
[0] << 24);
878 ** Create a new delta.
880 ** The delta is written into a preallocated buffer, zDelta, which
881 ** should be at least 60 bytes longer than the target file, zOut.
882 ** The delta string will be NUL-terminated, but it might also contain
883 ** embedded NUL characters if either the zSrc or zOut files are
884 ** binary. This function returns the length of the delta string
885 ** in bytes, excluding the final NUL terminator character.
889 ** The delta begins with a base64 number followed by a newline. This
890 ** number is the number of bytes in the TARGET file. Thus, given a
891 ** delta file z, a program can compute the size of the output file
892 ** simply by reading the first line and decoding the base-64 number
893 ** found there. The delta_output_size() routine does exactly this.
895 ** After the initial size number, the delta consists of a series of
896 ** literal text segments and commands to copy from the SOURCE file.
897 ** A copy command looks like this:
901 ** where NNN is the number of bytes to be copied and MMM is the offset
902 ** into the source file of the first byte (both base-64). If NNN is 0
903 ** it means copy the rest of the input file. Literal text is like this:
907 ** where NNN is the number of bytes of text (base-64) and TTTTT is the text.
909 ** The last term is of the form
913 ** In this case, NNN is a 32-bit bigendian checksum of the output file
914 ** that can be used to verify that the delta applied correctly. All
915 ** numbers are in base-64.
917 ** Pure text files generate a pure text delta. Binary files generate a
918 ** delta that may contain some binary data.
922 ** The encoder first builds a hash table to help it find matching
923 ** patterns in the source file. 16-byte chunks of the source file
924 ** sampled at evenly spaced intervals are used to populate the hash
927 ** Next we begin scanning the target file using a sliding 16-byte
928 ** window. The hash of the 16-byte window in the target is used to
929 ** search for a matching section in the source file. When a match
930 ** is found, a copy command is added to the delta. An effort is
931 ** made to extend the matching section to regions that come before
932 ** and after the 16-byte hash window. A copy command is only issued
933 ** if the result would use less space that just quoting the text
934 ** literally. Literal text is added to the delta for sections that
935 ** do not match or which can not be encoded efficiently using copy
938 static int rbuDeltaCreate(
939 const char *zSrc
, /* The source or pattern file */
940 unsigned int lenSrc
, /* Length of the source file */
941 const char *zOut
, /* The target file */
942 unsigned int lenOut
, /* Length of the target file */
943 char *zDelta
/* Write the delta into this buffer */
945 unsigned int i
, base
;
946 char *zOrigDelta
= zDelta
;
948 int nHash
; /* Number of hash table entries */
949 int *landmark
; /* Primary hash table */
950 int *collide
; /* Collision chain */
951 int lastRead
= -1; /* Last byte of zSrc read by a COPY command */
953 /* Add the target file size to the beginning of the delta
955 putInt(lenOut
, &zDelta
);
958 /* If the source file is very small, it means that we have no
959 ** chance of ever doing a copy command. Just output a single
960 ** literal segment for the entire target and exit.
963 putInt(lenOut
, &zDelta
);
965 memcpy(zDelta
, zOut
, lenOut
);
967 putInt(checksum(zOut
, lenOut
), &zDelta
);
969 return (int)(zDelta
- zOrigDelta
);
972 /* Compute the hash table used to locate matching sections in the
975 nHash
= lenSrc
/NHASH
;
976 collide
= sqlite3_malloc( nHash
*2*sizeof(int) );
977 landmark
= &collide
[nHash
];
978 memset(landmark
, -1, nHash
*sizeof(int));
979 memset(collide
, -1, nHash
*sizeof(int));
980 for(i
=0; i
<lenSrc
-NHASH
; i
+=NHASH
){
982 hash_init(&h
, &zSrc
[i
]);
983 hv
= hash_32bit(&h
) % nHash
;
984 collide
[i
/NHASH
] = landmark
[hv
];
985 landmark
[hv
] = i
/NHASH
;
988 /* Begin scanning the target file and generating copy commands and
989 ** literal sections of the delta.
991 base
= 0; /* We have already generated everything before zOut[base] */
992 while( base
+NHASH
<lenOut
){
994 int bestCnt
, bestOfst
=0, bestLitsz
=0;
995 hash_init(&h
, &zOut
[base
]);
996 i
= 0; /* Trying to match a landmark against zOut[base+i] */
1002 hv
= hash_32bit(&h
) % nHash
;
1003 iBlock
= landmark
[hv
];
1004 while( iBlock
>=0 && (limit
--)>0 ){
1006 ** The hash window has identified a potential match against
1007 ** landmark block iBlock. But we need to investigate further.
1009 ** Look for a region in zOut that matches zSrc. Anchor the search
1010 ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to
1011 ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen].
1013 ** Set cnt equal to the length of the match and set ofst so that
1014 ** zSrc[ofst] is the first element of the match. litsz is the number
1015 ** of characters between zOut[base] and the beginning of the match.
1016 ** sz will be the overhead (in bytes) needed to encode the copy
1017 ** command. Only generate copy command if the overhead of the
1018 ** copy command is less than the amount of literal text to be copied.
1020 int cnt
, ofst
, litsz
;
1024 /* Beginning at iSrc, match forwards as far as we can. j counts
1025 ** the number of characters that match */
1026 iSrc
= iBlock
*NHASH
;
1028 j
=0, x
=iSrc
, y
=base
+i
;
1029 (unsigned int)x
<lenSrc
&& (unsigned int)y
<lenOut
;
1032 if( zSrc
[x
]!=zOut
[y
] ) break;
1036 /* Beginning at iSrc-1, match backwards as far as we can. k counts
1037 ** the number of characters that match */
1038 for(k
=1; k
<iSrc
&& (unsigned int)k
<=i
; k
++){
1039 if( zSrc
[iSrc
-k
]!=zOut
[base
+i
-k
] ) break;
1043 /* Compute the offset and size of the matching region */
1046 litsz
= i
-k
; /* Number of bytes of literal text before the copy */
1047 /* sz will hold the number of bytes needed to encode the "insert"
1048 ** command and the copy command, not counting the "insert" text */
1049 sz
= digit_count(i
-k
)+digit_count(cnt
)+digit_count(ofst
)+3;
1050 if( cnt
>=sz
&& cnt
>bestCnt
){
1051 /* Remember this match only if it is the best so far and it
1052 ** does not increase the file size */
1058 /* Check the next matching block */
1059 iBlock
= collide
[iBlock
];
1062 /* We have a copy command that does not cause the delta to be larger
1063 ** than a literal insert. So add the copy command to the delta.
1067 /* Add an insert command before the copy */
1068 putInt(bestLitsz
,&zDelta
);
1070 memcpy(zDelta
, &zOut
[base
], bestLitsz
);
1071 zDelta
+= bestLitsz
;
1075 putInt(bestCnt
, &zDelta
);
1077 putInt(bestOfst
, &zDelta
);
1079 if( bestOfst
+ bestCnt
-1 > lastRead
){
1080 lastRead
= bestOfst
+ bestCnt
- 1;
1086 /* If we reach this point, it means no match is found so far */
1087 if( base
+i
+NHASH
>=lenOut
){
1088 /* We have reached the end of the file and have not found any
1089 ** matches. Do an "insert" for everything that does not match */
1090 putInt(lenOut
-base
, &zDelta
);
1092 memcpy(zDelta
, &zOut
[base
], lenOut
-base
);
1093 zDelta
+= lenOut
-base
;
1098 /* Advance the hash by one character. Keep looking for a match */
1099 hash_next(&h
, zOut
[base
+i
+NHASH
]);
1103 /* Output a final "insert" record to get all the text at the end of
1104 ** the file that does not match anything in the source file.
1107 putInt(lenOut
-base
, &zDelta
);
1109 memcpy(zDelta
, &zOut
[base
], lenOut
-base
);
1110 zDelta
+= lenOut
-base
;
1112 /* Output the final checksum record. */
1113 putInt(checksum(zOut
, lenOut
), &zDelta
);
1115 sqlite3_free(collide
);
1116 return (int)(zDelta
- zOrigDelta
);
1120 ** End of code copied from fossil.
1121 **************************************************************************/
1123 static void strPrintfArray(
1124 Str
*pStr
, /* String object to append to */
1125 const char *zSep
, /* Separator string */
1126 const char *zFmt
, /* Format for each entry */
1127 char **az
, int n
/* Array of strings & its size (or -1) */
1130 for(i
=0; az
[i
] && (i
<n
|| n
<0); i
++){
1131 if( i
!=0 ) strPrintf(pStr
, "%s", zSep
);
1132 strPrintf(pStr
, zFmt
, az
[i
], az
[i
], az
[i
]);
1136 static void getRbudiffQuery(
1145 /* First the newly inserted rows: **/
1146 strPrintf(pSql
, "SELECT ");
1147 strPrintfArray(pSql
, ", ", "%s", azCol
, -1);
1148 strPrintf(pSql
, ", 0, "); /* Set ota_control to 0 for an insert */
1149 strPrintfArray(pSql
, ", ", "NULL", azCol
, -1);
1150 strPrintf(pSql
, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab
);
1151 strPrintf(pSql
, " SELECT 1 FROM ", zTab
);
1152 strPrintf(pSql
, " main.%Q AS o WHERE ", zTab
);
1153 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1154 strPrintf(pSql
, "\n) AND ");
1155 strPrintfArray(pSql
, " AND ", "(n.%Q IS NOT NULL)", azCol
, nPK
);
1158 strPrintf(pSql
, "\nUNION ALL\nSELECT ");
1159 strPrintfArray(pSql
, ", ", "%s", azCol
, nPK
);
1161 strPrintf(pSql
, ", ");
1162 strPrintfArray(pSql
, ", ", "NULL", &azCol
[nPK
], -1);
1164 strPrintf(pSql
, ", 1, "); /* Set ota_control to 1 for a delete */
1165 strPrintfArray(pSql
, ", ", "NULL", azCol
, -1);
1166 strPrintf(pSql
, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab
);
1167 strPrintf(pSql
, " SELECT 1 FROM ", zTab
);
1168 strPrintf(pSql
, " aux.%Q AS o WHERE ", zTab
);
1169 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1170 strPrintf(pSql
, "\n) AND ");
1171 strPrintfArray(pSql
, " AND ", "(n.%Q IS NOT NULL)", azCol
, nPK
);
1173 /* Updated rows. If all table columns are part of the primary key, there
1174 ** can be no updates. In this case this part of the compound SELECT can
1175 ** be omitted altogether. */
1177 strPrintf(pSql
, "\nUNION ALL\nSELECT ");
1178 strPrintfArray(pSql
, ", ", "n.%s", azCol
, nPK
);
1179 strPrintf(pSql
, ",\n");
1180 strPrintfArray(pSql
, " ,\n",
1181 " CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol
[nPK
], -1
1185 strPrintf(pSql
, ", '");
1186 strPrintfArray(pSql
, "", ".", azCol
, nPK
);
1187 strPrintf(pSql
, "' ||\n");
1189 strPrintf(pSql
, ",\n");
1191 strPrintfArray(pSql
, " ||\n",
1192 " CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol
[nPK
], -1
1194 strPrintf(pSql
, "\nAS ota_control, ");
1195 strPrintfArray(pSql
, ", ", "NULL", azCol
, nPK
);
1196 strPrintf(pSql
, ",\n");
1197 strPrintfArray(pSql
, " ,\n",
1198 " CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol
[nPK
], -1
1201 strPrintf(pSql
, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab
, zTab
);
1202 strPrintfArray(pSql
, " AND ", "(n.%Q = o.%Q)", azCol
, nPK
);
1203 strPrintf(pSql
, " AND ota_control LIKE '%%x%%'");
1206 /* Now add an ORDER BY clause to sort everything by PK. */
1207 strPrintf(pSql
, "\nORDER BY ");
1208 for(i
=1; i
<=nPK
; i
++) strPrintf(pSql
, "%s%d", ((i
>1)?", ":""), i
);
1211 static void rbudiff_one_table(const char *zTab
, FILE *out
){
1212 int bOtaRowid
; /* True to use an ota_rowid column */
1213 int nPK
; /* Number of primary key columns in table */
1214 char **azCol
; /* NULL terminated array of col names */
1217 Str ct
= {0, 0, 0}; /* The "CREATE TABLE data_xxx" statement */
1218 Str sql
= {0, 0, 0}; /* Query to find differences */
1219 Str insert
= {0, 0, 0}; /* First part of output INSERT statement */
1220 sqlite3_stmt
*pStmt
= 0;
1221 int nRow
= 0; /* Total rows in data_xxx table */
1223 /* --rbu mode must use real primary keys. */
1226 /* Check that the schemas of the two tables match. Exit early otherwise. */
1227 checkSchemasMatch(zTab
);
1229 /* Grab the column names and PK details for the table(s). If no usable PK
1230 ** columns are found, bail out early. */
1231 azCol
= columnNames("main", zTab
, &nPK
, &bOtaRowid
);
1233 runtimeError("table %s has no usable PK columns", zTab
);
1235 for(nCol
=0; azCol
[nCol
]; nCol
++);
1237 /* Build and output the CREATE TABLE statement for the data_xxx table */
1238 strPrintf(&ct
, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab
);
1239 if( bOtaRowid
) strPrintf(&ct
, "rbu_rowid, ");
1240 strPrintfArray(&ct
, ", ", "%s", &azCol
[bOtaRowid
], -1);
1241 strPrintf(&ct
, ", rbu_control);");
1243 /* Get the SQL for the query to retrieve data from the two databases */
1244 getRbudiffQuery(zTab
, azCol
, nPK
, bOtaRowid
, &sql
);
1246 /* Build the first part of the INSERT statement output for each row
1247 ** in the data_xxx table. */
1248 strPrintf(&insert
, "INSERT INTO 'data_%q' (", zTab
);
1249 if( bOtaRowid
) strPrintf(&insert
, "rbu_rowid, ");
1250 strPrintfArray(&insert
, ", ", "%s", &azCol
[bOtaRowid
], -1);
1251 strPrintf(&insert
, ", rbu_control) VALUES(");
1253 pStmt
= db_prepare("%s", sql
.z
);
1255 while( sqlite3_step(pStmt
)==SQLITE_ROW
){
1257 /* If this is the first row output, print out the CREATE TABLE
1258 ** statement first. And then set ct.z to NULL so that it is not
1259 ** printed again. */
1261 fprintf(out
, "%s\n", ct
.z
);
1265 /* Output the first part of the INSERT statement */
1266 fprintf(out
, "%s", insert
.z
);
1269 if( sqlite3_column_type(pStmt
, nCol
)==SQLITE_INTEGER
){
1270 for(i
=0; i
<=nCol
; i
++){
1271 if( i
>0 ) fprintf(out
, ", ");
1272 printQuoted(out
, sqlite3_column_value(pStmt
, i
));
1276 int nOtaControl
= sqlite3_column_bytes(pStmt
, nCol
);
1278 zOtaControl
= (char*)sqlite3_malloc(nOtaControl
+1);
1279 memcpy(zOtaControl
, sqlite3_column_text(pStmt
, nCol
), nOtaControl
+1);
1281 for(i
=0; i
<nCol
; i
++){
1284 && sqlite3_column_type(pStmt
, i
)==SQLITE_BLOB
1285 && sqlite3_column_type(pStmt
, nCol
+1+i
)==SQLITE_BLOB
1287 const char *aSrc
= sqlite3_column_blob(pStmt
, nCol
+1+i
);
1288 int nSrc
= sqlite3_column_bytes(pStmt
, nCol
+1+i
);
1289 const char *aFinal
= sqlite3_column_blob(pStmt
, i
);
1290 int nFinal
= sqlite3_column_bytes(pStmt
, i
);
1294 aDelta
= sqlite3_malloc(nFinal
+ 60);
1295 nDelta
= rbuDeltaCreate(aSrc
, nSrc
, aFinal
, nFinal
, aDelta
);
1296 if( nDelta
<nFinal
){
1299 for(j
=0; j
<nDelta
; j
++) fprintf(out
, "%02x", (u8
)aDelta
[j
]);
1301 zOtaControl
[i
-bOtaRowid
] = 'f';
1304 sqlite3_free(aDelta
);
1308 printQuoted(out
, sqlite3_column_value(pStmt
, i
));
1312 fprintf(out
, "'%s'", zOtaControl
);
1313 sqlite3_free(zOtaControl
);
1316 /* And the closing bracket of the insert statement */
1317 fprintf(out
, ");\n");
1320 sqlite3_finalize(pStmt
);
1322 Str cnt
= {0, 0, 0};
1323 strPrintf(&cnt
, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab
, nRow
);
1324 fprintf(out
, "%s\n", cnt
.z
);
1334 ** Display a summary of differences between two versions of the same
1337 ** * Number of rows changed
1338 ** * Number of rows added
1339 ** * Number of rows deleted
1340 ** * Number of identical rows
1342 static void summarize_one_table(const char *zTab
, FILE *out
){
1343 char *zId
= safeId(zTab
); /* Name of table (translated for us in SQL) */
1344 char **az
= 0; /* Columns in main */
1345 char **az2
= 0; /* Columns in aux */
1346 int nPk
; /* Primary key columns in main */
1347 int nPk2
; /* Primary key columns in aux */
1348 int n
= 0; /* Number of columns in main */
1349 int n2
; /* Number of columns in aux */
1350 int i
; /* Loop counter */
1351 const char *zSep
; /* Separator string */
1352 Str sql
; /* Comparison query */
1353 sqlite3_stmt
*pStmt
; /* Query statement to do the diff */
1354 sqlite3_int64 nUpdate
; /* Number of updated rows */
1355 sqlite3_int64 nUnchanged
; /* Number of unmodified rows */
1356 sqlite3_int64 nDelete
; /* Number of deleted rows */
1357 sqlite3_int64 nInsert
; /* Number of inserted rows */
1360 if( sqlite3_table_column_metadata(g
.db
,"aux",zTab
,0,0,0,0,0,0) ){
1361 if( !sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
1362 /* Table missing from second database. */
1363 fprintf(out
, "%s: missing from second database\n", zTab
);
1365 goto end_summarize_one_table
;
1368 if( sqlite3_table_column_metadata(g
.db
,"main",zTab
,0,0,0,0,0,0) ){
1369 /* Table missing from source */
1370 fprintf(out
, "%s: missing from first database\n", zTab
);
1371 goto end_summarize_one_table
;
1374 az
= columnNames("main", zTab
, &nPk
, 0);
1375 az2
= columnNames("aux", zTab
, &nPk2
, 0);
1377 for(n
=0; az
[n
]; n
++){
1378 if( sqlite3_stricmp(az
[n
],az2
[n
])!=0 ) break;
1386 /* Schema mismatch */
1387 fprintf(out
, "%s: incompatible schema\n", zTab
);
1388 goto end_summarize_one_table
;
1391 /* Build the comparison query */
1392 for(n2
=n
; az
[n2
]; n2
++){}
1393 strPrintf(&sql
, "SELECT 1, count(*)");
1395 strPrintf(&sql
, ", 0\n");
1398 for(i
=nPk
; az
[i
]; i
++){
1399 strPrintf(&sql
, "%sA.%s IS NOT B.%s", zSep
, az
[i
], az
[i
]);
1402 strPrintf(&sql
, ")\n");
1404 strPrintf(&sql
, " FROM main.%s A, aux.%s B\n", zId
, zId
);
1406 for(i
=0; i
<nPk
; i
++){
1407 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1410 strPrintf(&sql
, " UNION ALL\n");
1411 strPrintf(&sql
, "SELECT 2, count(*), 0\n");
1412 strPrintf(&sql
, " FROM main.%s A\n", zId
);
1413 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId
);
1415 for(i
=0; i
<nPk
; i
++){
1416 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1419 strPrintf(&sql
, ")\n");
1420 strPrintf(&sql
, " UNION ALL\n");
1421 strPrintf(&sql
, "SELECT 3, count(*), 0\n");
1422 strPrintf(&sql
, " FROM aux.%s B\n", zId
);
1423 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId
);
1425 for(i
=0; i
<nPk
; i
++){
1426 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, az
[i
], az
[i
]);
1429 strPrintf(&sql
, ")\n ORDER BY 1;\n");
1431 if( (g
.fDebug
& DEBUG_DIFF_SQL
)!=0 ){
1432 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
1433 goto end_summarize_one_table
;
1436 /* Run the query and output difference summary */
1437 pStmt
= db_prepare("%s", sql
.z
);
1442 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1443 switch( sqlite3_column_int(pStmt
,0) ){
1445 nUpdate
= sqlite3_column_int64(pStmt
,2);
1446 nUnchanged
= sqlite3_column_int64(pStmt
,1) - nUpdate
;
1449 nDelete
= sqlite3_column_int64(pStmt
,1);
1452 nInsert
= sqlite3_column_int64(pStmt
,1);
1456 sqlite3_finalize(pStmt
);
1457 fprintf(out
, "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
1458 zTab
, nUpdate
, nInsert
, nDelete
, nUnchanged
);
1460 end_summarize_one_table
:
1469 ** Write a 64-bit signed integer as a varint onto out
1471 static void putsVarint(FILE *out
, sqlite3_uint64 v
){
1473 unsigned char p
[12];
1474 if( v
& (((sqlite3_uint64
)0xff000000)<<32) ){
1475 p
[8] = (unsigned char)v
;
1477 for(i
=7; i
>=0; i
--){
1478 p
[i
] = (unsigned char)((v
& 0x7f) | 0x80);
1481 fwrite(p
, 8, 1, out
);
1485 p
[n
--] = (unsigned char)((v
& 0x7f) | 0x80);
1489 fwrite(p
+n
+1, 9-n
, 1, out
);
1494 ** Write an SQLite value onto out.
1496 static void putValue(FILE *out
, sqlite3_stmt
*pStmt
, int k
){
1497 int iDType
= sqlite3_column_type(pStmt
, k
);
1505 case SQLITE_INTEGER
:
1506 iX
= sqlite3_column_int64(pStmt
, k
);
1507 memcpy(&uX
, &iX
, 8);
1508 for(j
=56; j
>=0; j
-=8) putc((uX
>>j
)&0xff, out
);
1511 rX
= sqlite3_column_double(pStmt
, k
);
1512 memcpy(&uX
, &rX
, 8);
1513 for(j
=56; j
>=0; j
-=8) putc((uX
>>j
)&0xff, out
);
1516 iX
= sqlite3_column_bytes(pStmt
, k
);
1517 putsVarint(out
, (sqlite3_uint64
)iX
);
1518 fwrite(sqlite3_column_text(pStmt
, k
),1,(size_t)iX
,out
);
1521 iX
= sqlite3_column_bytes(pStmt
, k
);
1522 putsVarint(out
, (sqlite3_uint64
)iX
);
1523 fwrite(sqlite3_column_blob(pStmt
, k
),1,(size_t)iX
,out
);
1531 ** Generate a CHANGESET for all differences from main.zTab to aux.zTab.
1533 static void changeset_one_table(const char *zTab
, FILE *out
){
1534 sqlite3_stmt
*pStmt
; /* SQL statment */
1535 char *zId
= safeId(zTab
); /* Escaped name of the table */
1536 char **azCol
= 0; /* List of escaped column names */
1537 int nCol
= 0; /* Number of columns */
1538 int *aiFlg
= 0; /* 0 if column is not part of PK */
1539 int *aiPk
= 0; /* Column numbers for each PK column */
1540 int nPk
= 0; /* Number of PRIMARY KEY columns */
1541 Str sql
; /* SQL for the diff query */
1542 int i
, k
; /* Loop counters */
1543 const char *zSep
; /* List separator */
1545 /* Check that the schemas of the two tables match. Exit early otherwise. */
1546 checkSchemasMatch(zTab
);
1548 pStmt
= db_prepare("PRAGMA main.table_info=%Q", zTab
);
1549 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1551 azCol
= sqlite3_realloc(azCol
, sizeof(char*)*nCol
);
1552 if( azCol
==0 ) runtimeError("out of memory");
1553 aiFlg
= sqlite3_realloc(aiFlg
, sizeof(int)*nCol
);
1554 if( aiFlg
==0 ) runtimeError("out of memory");
1555 azCol
[nCol
-1] = safeId((const char*)sqlite3_column_text(pStmt
,1));
1556 aiFlg
[nCol
-1] = i
= sqlite3_column_int(pStmt
,5);
1560 aiPk
= sqlite3_realloc(aiPk
, sizeof(int)*nPk
);
1561 if( aiPk
==0 ) runtimeError("out of memory");
1566 sqlite3_finalize(pStmt
);
1567 if( nPk
==0 ) goto end_changeset_one_table
;
1570 strPrintf(&sql
, "SELECT %d", SQLITE_UPDATE
);
1571 for(i
=0; i
<nCol
; i
++){
1573 strPrintf(&sql
, ",\n A.%s", azCol
[i
]);
1575 strPrintf(&sql
, ",\n A.%s IS NOT B.%s, A.%s, B.%s",
1576 azCol
[i
], azCol
[i
], azCol
[i
], azCol
[i
]);
1579 strPrintf(&sql
,"\n FROM main.%s A, aux.%s B\n", zId
, zId
);
1581 for(i
=0; i
<nPk
; i
++){
1582 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1586 for(i
=0; i
<nCol
; i
++){
1587 if( aiFlg
[i
] ) continue;
1588 strPrintf(&sql
, "%sA.%s IS NOT B.%s", zSep
, azCol
[i
], azCol
[i
]);
1591 strPrintf(&sql
,")\n UNION ALL\n");
1593 strPrintf(&sql
, "SELECT %d", SQLITE_DELETE
);
1594 for(i
=0; i
<nCol
; i
++){
1596 strPrintf(&sql
, ",\n A.%s", azCol
[i
]);
1598 strPrintf(&sql
, ",\n 1, A.%s, NULL", azCol
[i
]);
1601 strPrintf(&sql
, "\n FROM main.%s A\n", zId
);
1602 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId
);
1604 for(i
=0; i
<nPk
; i
++){
1605 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1608 strPrintf(&sql
, ")\n UNION ALL\n");
1609 strPrintf(&sql
, "SELECT %d", SQLITE_INSERT
);
1610 for(i
=0; i
<nCol
; i
++){
1612 strPrintf(&sql
, ",\n B.%s", azCol
[i
]);
1614 strPrintf(&sql
, ",\n 1, NULL, B.%s", azCol
[i
]);
1617 strPrintf(&sql
, "\n FROM aux.%s B\n", zId
);
1618 strPrintf(&sql
, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId
);
1620 for(i
=0; i
<nPk
; i
++){
1621 strPrintf(&sql
, "%s A.%s=B.%s", zSep
, azCol
[aiPk
[i
]], azCol
[aiPk
[i
]]);
1624 strPrintf(&sql
, ")\n");
1625 strPrintf(&sql
, " ORDER BY");
1627 for(i
=0; i
<nPk
; i
++){
1628 strPrintf(&sql
, "%s %d", zSep
, aiPk
[i
]+2);
1631 strPrintf(&sql
, ";\n");
1633 if( g
.fDebug
& DEBUG_DIFF_SQL
){
1634 printf("SQL for %s:\n%s\n", zId
, sql
.z
);
1635 goto end_changeset_one_table
;
1639 putsVarint(out
, (sqlite3_uint64
)nCol
);
1640 for(i
=0; i
<nCol
; i
++) putc(aiFlg
[i
], out
);
1641 fwrite(zTab
, 1, strlen(zTab
), out
);
1644 pStmt
= db_prepare("%s", sql
.z
);
1645 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1646 int iType
= sqlite3_column_int(pStmt
,0);
1649 switch( sqlite3_column_int(pStmt
,0) ){
1650 case SQLITE_UPDATE
: {
1651 for(k
=1, i
=0; i
<nCol
; i
++){
1653 putValue(out
, pStmt
, k
);
1655 }else if( sqlite3_column_int(pStmt
,k
) ){
1656 putValue(out
, pStmt
, k
+1);
1663 for(k
=1, i
=0; i
<nCol
; i
++){
1667 }else if( sqlite3_column_int(pStmt
,k
) ){
1668 putValue(out
, pStmt
, k
+2);
1677 case SQLITE_INSERT
: {
1678 for(k
=1, i
=0; i
<nCol
; i
++){
1680 putValue(out
, pStmt
, k
);
1683 putValue(out
, pStmt
, k
+2);
1689 case SQLITE_DELETE
: {
1690 for(k
=1, i
=0; i
<nCol
; i
++){
1692 putValue(out
, pStmt
, k
);
1695 putValue(out
, pStmt
, k
+1);
1703 sqlite3_finalize(pStmt
);
1705 end_changeset_one_table
:
1706 while( nCol
>0 ) sqlite3_free(azCol
[--nCol
]);
1707 sqlite3_free(azCol
);
1713 ** Extract the next SQL keyword or quoted string from buffer zIn and copy it
1714 ** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
1715 ** Return a pointer to the character within zIn immediately following
1716 ** the token or quoted string just extracted.
1718 const char *gobble_token(const char *zIn
, char *zBuf
, int nBuf
){
1719 const char *p
= zIn
;
1721 char *pEnd
= &pOut
[nBuf
-1];
1722 char q
= 0; /* quote character, if any */
1724 if( p
==0 ) return 0;
1725 while( *p
==' ' ) p
++;
1727 case '"': q
= '"'; break;
1728 case '\'': q
= '\''; break;
1729 case '`': q
= '`'; break;
1730 case '[': q
= ']'; break;
1735 while( *p
&& pOut
<pEnd
){
1740 if( pOut
<pEnd
) *pOut
++ = *p
;
1744 while( *p
&& *p
!=' ' && *p
!='(' ){
1745 if( pOut
<pEnd
) *pOut
++ = *p
;
1755 ** This function is the implementation of SQL scalar function "module_name":
1759 ** The only argument should be an SQL statement of the type that may appear
1760 ** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
1761 ** statement, then the value returned is the name of the module that it
1762 ** uses. Otherwise, if the statement is not a CVT, NULL is returned.
1764 static void module_name_func(
1765 sqlite3_context
*pCtx
,
1766 int nVal
, sqlite3_value
**apVal
1772 zSql
= (const char*)sqlite3_value_text(apVal
[0]);
1774 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1775 if( zSql
==0 || sqlite3_stricmp(zToken
, "create") ) return;
1776 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1777 if( zSql
==0 || sqlite3_stricmp(zToken
, "virtual") ) return;
1778 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1779 if( zSql
==0 || sqlite3_stricmp(zToken
, "table") ) return;
1780 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1781 if( zSql
==0 ) return;
1782 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1783 if( zSql
==0 || sqlite3_stricmp(zToken
, "using") ) return;
1784 zSql
= gobble_token(zSql
, zToken
, sizeof(zToken
));
1786 sqlite3_result_text(pCtx
, zToken
, -1, SQLITE_TRANSIENT
);
1790 ** Return the text of an SQL statement that itself returns the list of
1791 ** tables to process within the database.
1793 const char *all_tables_sql(){
1794 if( g
.bHandleVtab
){
1797 rc
= sqlite3_exec(g
.db
,
1798 "CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);"
1799 "INSERT INTO temp.tblmap VALUES"
1800 "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
1802 "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
1803 "('fts4', '_docsize'), ('fts4', '_stat'),"
1805 "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
1806 "('fts5', '_docsize'), ('fts5', '_config'),"
1808 "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
1811 assert( rc
==SQLITE_OK
);
1813 rc
= sqlite3_create_function(
1814 g
.db
, "module_name", 1, SQLITE_UTF8
, 0, module_name_func
, 0, 0
1816 assert( rc
==SQLITE_OK
);
1819 "SELECT name FROM main.sqlite_master\n"
1820 " WHERE type='table' AND (\n"
1821 " module_name(sql) IS NULL OR \n"
1822 " module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1823 " ) AND name NOT IN (\n"
1824 " SELECT a.name || b.postfix \n"
1825 "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
1826 "WHERE module_name(a.sql) = b.module\n"
1829 "SELECT name FROM aux.sqlite_master\n"
1830 " WHERE type='table' AND (\n"
1831 " module_name(sql) IS NULL OR \n"
1832 " module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1833 " ) AND name NOT IN (\n"
1834 " SELECT a.name || b.postfix \n"
1835 "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
1836 "WHERE module_name(a.sql) = b.module\n"
1841 "SELECT name FROM main.sqlite_master\n"
1842 " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1844 "SELECT name FROM aux.sqlite_master\n"
1845 " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1851 ** Print sketchy documentation for this utility program
1853 static void showHelp(void){
1854 printf("Usage: %s [options] DB1 DB2\n", g
.zArgv0
);
1856 "Output SQL text that would transform DB1 into DB2.\n"
1858 " --changeset FILE Write a CHANGESET into FILE\n"
1859 " -L|--lib LIBRARY Load an SQLite extension library\n"
1860 " --primarykey Use schema-defined PRIMARY KEYs\n"
1861 " --rbu Output SQL to create/populate RBU table(s)\n"
1862 " --schema Show only differences in the schema\n"
1863 " --summary Show only a summary of the differences\n"
1864 " --table TAB Show only differences in table TAB\n"
1865 " --transaction Show SQL output inside a transaction\n"
1866 " --vtab Handle fts3, fts4, fts5 and rtree tables\n"
1870 int main(int argc
, char **argv
){
1871 const char *zDb1
= 0;
1872 const char *zDb2
= 0;
1877 sqlite3_stmt
*pStmt
;
1880 void (*xDiff
)(const char*,FILE*) = diff_one_table
;
1881 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1885 int useTransaction
= 0;
1886 int neverUseTransaction
= 0;
1889 sqlite3_config(SQLITE_CONFIG_SINGLETHREAD
);
1890 for(i
=1; i
<argc
; i
++){
1891 const char *z
= argv
[i
];
1894 if( z
[0]=='-' ) z
++;
1895 if( strcmp(z
,"changeset")==0 ){
1896 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1897 out
= fopen(argv
[++i
], "wb");
1898 if( out
==0 ) cmdlineError("cannot open: %s", argv
[i
]);
1899 xDiff
= changeset_one_table
;
1900 neverUseTransaction
= 1;
1902 if( strcmp(z
,"debug")==0 ){
1903 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1904 g
.fDebug
= strtol(argv
[++i
], 0, 0);
1906 if( strcmp(z
,"help")==0 ){
1910 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1911 if( strcmp(z
,"lib")==0 || strcmp(z
,"L")==0 ){
1912 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1913 azExt
= realloc(azExt
, sizeof(azExt
[0])*(nExt
+1));
1914 if( azExt
==0 ) cmdlineError("out of memory");
1915 azExt
[nExt
++] = argv
[++i
];
1918 if( strcmp(z
,"primarykey")==0 ){
1921 if( strcmp(z
,"rbu")==0 ){
1922 xDiff
= rbudiff_one_table
;
1924 if( strcmp(z
,"schema")==0 ){
1927 if( strcmp(z
,"summary")==0 ){
1928 xDiff
= summarize_one_table
;
1930 if( strcmp(z
,"table")==0 ){
1931 if( i
==argc
-1 ) cmdlineError("missing argument to %s", argv
[i
]);
1934 if( strcmp(z
,"transaction")==0 ){
1937 if( strcmp(z
,"vtab")==0 ){
1941 cmdlineError("unknown option: %s", argv
[i
]);
1943 }else if( zDb1
==0 ){
1945 }else if( zDb2
==0 ){
1948 cmdlineError("unknown argument: %s", argv
[i
]);
1952 cmdlineError("two database arguments required");
1954 rc
= sqlite3_open(zDb1
, &g
.db
);
1956 cmdlineError("cannot open database file \"%s\"", zDb1
);
1958 rc
= sqlite3_exec(g
.db
, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg
);
1959 if( rc
|| zErrMsg
){
1960 cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1
);
1962 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1963 sqlite3_enable_load_extension(g
.db
, 1);
1964 for(i
=0; i
<nExt
; i
++){
1965 rc
= sqlite3_load_extension(g
.db
, azExt
[i
], 0, &zErrMsg
);
1966 if( rc
|| zErrMsg
){
1967 cmdlineError("error loading %s: %s", azExt
[i
], zErrMsg
);
1972 zSql
= sqlite3_mprintf("ATTACH %Q as aux;", zDb2
);
1973 rc
= sqlite3_exec(g
.db
, zSql
, 0, 0, &zErrMsg
);
1974 if( rc
|| zErrMsg
){
1975 cmdlineError("cannot attach database \"%s\"", zDb2
);
1977 rc
= sqlite3_exec(g
.db
, "SELECT * FROM aux.sqlite_master", 0, 0, &zErrMsg
);
1978 if( rc
|| zErrMsg
){
1979 cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2
);
1982 if( neverUseTransaction
) useTransaction
= 0;
1983 if( useTransaction
) fprintf(out
, "BEGIN TRANSACTION;\n");
1984 if( xDiff
==rbudiff_one_table
){
1985 fprintf(out
, "CREATE TABLE IF NOT EXISTS rbu_count"
1986 "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
1993 /* Handle tables one by one */
1994 pStmt
= db_prepare("%s", all_tables_sql() );
1995 while( SQLITE_ROW
==sqlite3_step(pStmt
) ){
1996 xDiff((const char*)sqlite3_column_text(pStmt
,0), out
);
1998 sqlite3_finalize(pStmt
);
2000 if( useTransaction
) printf("COMMIT;\n");
2002 /* TBD: Handle trigger differences */
2003 /* TBD: Handle view differences */
2004 sqlite3_close(g
.db
);