4 (C) by Vladimir Kladov, 2002-2004
6 These functions allows to create (MakeUpdates) a command stream
7 on base of two data streams: Stream1 and Stream2,
8 which later can be used to restore (ApplyUpdates) Stream2 on base of
9 the same source Stream1 and created by MakeUpdates command stream.
11 This technique used in my UpdateMaker and Updater applications
12 to provide patches to a set of (big) files which otherwise could
13 require a lot of files to download to get new version. This is great
14 for developers providing sources rather then compiled stuff.
19 first release of this unit: functions MakeUpdates and ApplyUpdates
20 extracted from UpdateMaker / Updater sources, with some sumplifications
21 in order to make possible to use streams rather then files, and to use
22 these two functions separately from UpdateMaker and Updater utilities.
25 version 2 of this unit, algorithm improved in comparison to current
26 version of UpdateMaker program and errors fixed (though new verion of
27 MakeUpdates creates command file still compatible with old version
28 of ApplyUpdates been corrected). Now plan to re-use these functions in
29 newer versions of Updater / UpdateMaker.
32 version 3 of this unit. Some errors fixed. Parameter added: OnProgress.
41 //{$DEFINE WRITE_LEN_CC}
44 TOnUpdatesProgress
= procedure( Percents
, TotalSize
, CurrentPosition
: Integer;
45 var Cancel
: Boolean ) of object;
47 function ApplyUpdates( OutStrm
, SourceStream
, CmdStrm
: PStream
;
48 OnProgress
: TOnUpdatesProgress
): Boolean;
49 function MakeUpdates( DestCommandStream
, LastVersion
, PrevVersion
: PStream
;
50 OnProgress
: TOnUpdatesProgress
): Boolean;
55 TCommand
= (cmdFWDandCOPY
, cmdBCKandCOPY
, cmdINSERT
, cmdNone
);
56 TDwordArray
= array[ 0..1000000 ] of DWORD
;
57 PDwordArray
= ^TDwordArray
;
59 function Compare4Bytes( A
: PDwordArray
; e1
, e2
: DWORD
): Integer;
88 procedure SwapIndxes( A
: PDwordArray
; e1
, e2
: DWORD
);
101 XCHG EBX, [EAX+EDX*4]
108 function MakePattern2(Data
: PStream
; Old
, New
: PChar
; OldLen
, NewLen
: Integer;
109 MaxSize
: Integer; var Equal
: Boolean; OnProgress
: TOnUpdatesProgress
): Boolean;
110 var HashTable1
: PChar
;
111 SortTable1
, AccTable1
: PDwordArray
;
113 function BytesEqual( OldPos
, NewPos
: Integer ): Integer;
116 Result
:= Min( OldLen
- OldPos
, NewLen
- NewPos
);
117 for I
:= 1 to Result
do
119 if Old
[ OldPos
] <> New
[ NewPos
] then
129 // çàïèñü îäíîãî áàéòà â âûõîäíîé ïîòîê
130 procedure WriteByte( B
: Byte );
135 // âîçâðàòèò Flag, åñëè âûïîëíåíî óñëîâèå Cond, èíà÷å 0.
136 function MakeFlag( Flag
: Byte; Cond
: Boolean ): Byte;
138 if Cond
then Result
:= Flag
142 // çàïèñü ÷èñëà â ôîðìàòå ñ ïåðåìåííîé äëèíîé. Ñòàðøèé áèò áàéòà îïðåäåëÿåò
143 // äëÿ ÷èòàòåëÿ ÷èñëà, ÷èòàòü ëè ñëåäóþùèé áàéò. 7 ðàçðÿäîâ õðàíÿò î÷åðåäíûå
144 // 7 áèò ÷èñëà. Äëÿ õðàíåíèÿ ÷èñëà îò 0 äî 127 äîñòàòî÷íî 1 áàéòà, îò 128 äî
145 // 16383 (14 áèò åäèíèö) - äâóõ áàéò, è ò.ä.
146 procedure WriteNum( N
: DWORD
);
149 WriteByte( N
and $7F or MakeFlag( $80, N
> 127 ) );
154 // Ïðåäâàðèòåëüíûé ïîäñ÷åò ÷èñëà â áàéòàõ
155 function CalcNumLen( N
: DWORD
): Integer;
165 var PrevCmd
: TCommand
;
167 // ôîðìèðóåò êîìàíäó ñ ïàðàìåòðîì "äëèíà" ïåðåìåííîé ðàçðÿäíîñòè. Ñì. îïèñàíèå
168 // âûøå. Ôîðìàò êîìàíäû çàâèñèò îò PrevCmd.
169 procedure WriteCommand( Cmd
: TCommand
; Len
: Integer );
177 WriteByte( Len
and $3F or $00 or MakeFlag( $40, Len
> 63 ) );
178 if Len
> 63 then WriteNum( Len
shr 6 );
182 WriteByte( Len
and $3F or $80 or MakeFlag( $40, Len
> 63 ) );
183 if Len
> 63 then WriteNum( Len
shr 6 );
187 ShowMessage( 'Error! BCKandCOPY at the beginning' );
196 WriteByte( Len
and $3F or $00 or MakeFlag( $40, Len
> 63 ) );
197 if Len
> 63 then WriteNum( Len
shr 6 );
201 WriteByte( Len
and $3F or $80 or MakeFlag( $40, Len
> 63 ) );
202 if Len
> 63 then WriteNum( Len
shr 6 );
206 ShowMessage( 'Error! INSERT after INSERT' );
211 cmdBCKandCOPY
, cmdFWDandCOPY
:
216 WriteByte( Len
and $3F or $80 or MakeFlag( $40, Len
> 63 ) );
217 if Len
> 63 then WriteNum( Len
shr 6 );
221 WriteByte( Len
and $1F or $00 or MakeFlag( $20, Len
> 31 ) );
222 if Len
> 31 then WriteNum( Len
shr 5 );
226 WriteByte( Len
and $1F or $40 or MakeFlag( $20, Len
> 31 ) );
227 if Len
> 31 then WriteNum( Len
shr 5 );
235 var OldPos
, NewPos
: Integer;
237 // Âû÷èñëÿåò ïðåäïîëîæèòåëüíûé ðàçìåð êîìàíäû BCKandCOPY èëè FWDandCOPY.
238 function CalcCmdLen( OldIdx
: Integer; LCopy
: Integer ): Integer;
242 if OldIdx
< OldPos
then
244 //Cmd := cmdBCKandCOPY;
245 Off
:= OldPos
-OldIdx
-1-LCopy
;
249 //Cmd := cmdFWDandCOPY;
250 Off
:= OldIdx
- OldPos
;
253 Result
:= 1 + CalcNumLen( Off
);
256 if LCopy
> 63 then Inc( Result
, CalcNumLen( LCopy
shr 6 ) );
257 cmdBCKandCOPY
, cmdFWDandCOPY
:
258 if LCopy
> 31 then Inc( Result
, CalcNumLen( LCopy
shr 5 ) );
262 // Âû÷èñëÿåò ïðåäïîëîæèòåëüíûé ðàçìåð êîìàíäû BCKandCOPY èëè FWDandCOPY.
263 function CalcCmdLen2( NewIdx
: Integer; LCopy
: Integer ): Integer;
267 if NewIdx
< NewPos
then
269 //Cmd := cmdBCKandCOPY;
270 Off
:= NewPos
-NewIdx
-1-LCopy
;
274 //Cmd := cmdFWDandCOPY;
275 Off
:= NewIdx
- NewPos
;
278 Result
:= 1 + CalcNumLen( Off
);
281 if LCopy
> 63 then Inc( Result
, CalcNumLen( LCopy
shr 6 ) );
282 cmdBCKandCOPY
, cmdFWDandCOPY
:
283 if LCopy
> 31 then Inc( Result
, CalcNumLen( LCopy
shr 5 ) );
287 // èçîáðàæåíèå ïðîãðåññà ïðîñìîòðà íîâîé âåðñèè ôàéëà.
288 function ShowProgress_Cancel
: Boolean;
291 Pr
:= NewPos
* 100 div NewLen
;
293 if Assigned( OnProgress
) then
294 OnProgress( Pr
, NewPos
, NewLen
, Result
);
297 // ïîèñê â ñòàðîì ôàéëå ïîñëåäîâàòåëüíîñòè áàéòîâ, ïî âîçìîæíîñòè áîëüøåé äëèíû,
298 // ñîâïàäàþùåé ñ áàéòàìè â íîâîé âåðñèè ôàéëà â ïîçèöèè NewIdx.
299 // Åñëè óäàåòñÿ íàéòè òàêóþ, âîçâðàùàåòñÿ ïîçèöèÿ â ñòàðîé âåðñèè ôàéëà
300 // è LenFound = äëèíà íàéäåííîé ïîñë-ñòè.
301 function SearchSimilar( NewIdx
: Integer; var LenFound
: Integer ): Integer;
302 function LexicographicCompare( X
, Y
: DWORD
): Integer;
311 Pos
, CmdLenFound
, CmdLen
: Integer;
316 Hash
:= PDWORD( @ New
[ NewIdx
] )^ and $FFFFFF;
317 I
:= AccTable1
[ Hash
and $FFFF ]; // èíäåêñ ïåðâîãî ýëåìåíòà â SortTable1,
318 // óêàçûâàþùåãî íà ïîñëåäîâàòåëüíîñòü â Old[],
319 // íà÷èíàþùóþñÿ ñ áàéòîâ New[ NewIdx ], New[ NewIdx + 1 ]
321 Exit
; // íåò òàêèõ ïîñëåäîâàòåëüíîñòåé èç 2õ áàéò â Old[]
322 for I
:= I
to OldLen
-4 do
324 Ptr
:= Pointer( SortTable1
[ I
] );
325 if (Ptr
^ and $FFFFFF) <> Hash
then
327 if LexicographicCompare( Ptr
^ and $FFFFFF, Hash
) > 0 then
329 Exit
; // âñ¸, âñå òàêèå ïîñëåäîâàòåëüíîñòè èç ïî êðàéíåé ìåðå òðåõ áàéò êîí÷èëèñü
334 Pos
:= Integer( Ptr
) - Integer( @ Old
[ 0 ] );
335 L
:= BytesEqual( Pos
, NewIdx
);
337 //Assert( L >= 3, '÷òî-òî íå òî' );
340 // ïðåäîòâðàòèì ïîÿâëåíèå îòðèöàòåëüíîãî ñìåùåíèÿ:
341 if (Pos
< OldPos
) and (Pos
+ L
>= OldPos
- 1) then
342 L
:= OldPos
- 1 - Pos
;
345 CmdLen
:= CalcCmdLen( Pos
, L
);
346 if L
+ CmdLen
> LenFound
+ CmdLenFound
then
348 CmdLenFound
:= CmdLen
;
359 var I
, J
, L
: Integer;
361 {$IFDEF WRITE_LEN_CC}
368 if (OldLen
= NewLen
) and (OldLen
<> 0) and (CompareMem( Old
, New
, OldLen
)) then
374 HashTable1
:= AllocMem( 1 shl 21 );
376 GetMem( SortTable1
, (OldLen
-3) * Sizeof( DWORD
) );
377 AccTable1
:= AllocMem( 65536 * Sizeof( DWORD
) );
379 if (HashTable1
= nil) or
380 (SortTable1
= nil) or (AccTable1
= nil) then
382 //ShowMessage( 'No memory to process (' + OldFile + '->' + NewFile + ').' );
385 // Èíèöèàëèçèðóåì õýø-òàáëèöû:
386 for I
:= 0 to OldLen
-3 do
388 J
:= PDWORD( @ Old
[ I
] )^ and $FFFFFF;
389 Byte( HashTable1
[ J
shr 3 ] ) := Byte( HashTable1
[ J
shr 3 ] ) or (1 shl (J
and 7));
391 // Ñòðîèì òàáëèöó áûñòðîãî ïîèñêà ïîñëåäîâàòåëüíîñòåé:
392 for I
:= 0 to OldLen
-4 do
393 SortTable1
[ I
] := DWORD( @ Old
[ I
] );
394 SortData( SortTable1
, OldLen
-3, @ Compare4Bytes
, @ SwapIndxes
);
395 for I
:= 0 to OldLen
-4 do
397 J
:= SortTable1
[ I
];
398 J
:= PDWORD( J
)^ and $FFFF;
399 if (AccTable1
[ J
] = 0) or (AccTable1
[ J
] > DWORD( I
)) then
400 AccTable1
[ J
] := I
; // AccTable1[ I ] = èíäåêñó ïåðâîãî âõîæäåíèÿ ïîñëåäîâàòåëüíîñòè
401 // íà÷èíàþùåéñÿ ñ 2õ áàéòîâ (J and $FF), (J shr 8)and $FF
403 // Ñòðîèì ôàéë ðàçëè÷èé:
407 {$IFDEF WRITE_LEN_CC}
409 // Ïîñ÷èòàåì êîíòðîëüíóþ ñóììó:
414 CC
:= ((CC
shl 1) or (CC
shr 31)) xor PDWORD(@Old
[ I
])^;
421 while (NewPos
< NewLen
) do
423 if ShowProgress_Cancel
then Exit
;
424 L
:= BytesEqual( OldPos
, NewPos
);
427 // ñîâïàäàåò ó÷àñòîê äîñòàòî÷íî õîðîøåé äëèíû, êîïèðóåì åãî:
428 WriteCommand( cmdFWDandCOPY
, L
);
429 WriteByte( 0 ); // ñìåùåíèå = 0
435 // èíà÷å èùåì ñëåäóþùèé ó÷àñòîê, êîòîðûé ìîæíî ñêîïèðîâàòü:
437 for I
:= NewPos
to NewLen
-6 do
439 //if (I and $1F) = 0 then
441 if ShowProgress_Cancel
then Exit
;
443 J
:= PDWORD( @ New
[ I
] )^ and $FFFFFF;
444 if ( Byte( HashTable1
[ J
shr 3 ] ) and (1 shl (J
and 7))) = 0 then continue
;
445 J
:= SearchSimilar( I
, L
);
451 WriteCommand( cmdINSERT
, I
-NewPos
);
452 Data
.Write( New
[ NewPos
], I
-NewPos
);
454 if J
< OldPos
then begin
455 WriteCommand( cmdBCKandCOPY
, L
);
456 WriteNum( OldPos
-J
-1-L
);
458 WriteCommand( cmdFWDandCOPY
, L
);
459 WriteNum( J
-OldPos
);
468 for I
:= NewPos
to NewLen
-6 do
470 //if (I and $1F) = 0 then
472 if ShowProgress_Cancel
then Exit
;
474 J
:= PDWORD( @ New
[ I
] )^ and $FFFFFF;
475 if ( Byte( HashTable1
[ J
shr 3 ] ) and (1 shl (J
and 7))) = 0 then continue
;
476 J
:= SearchSimilar( I
, L
);
480 WriteCommand( cmdINSERT
, I
-NewPos
);
481 Data
.Write( New
[ NewPos
], I
-NewPos
);
482 if J
< OldPos
then begin
483 WriteCommand( cmdBCKandCOPY
, L
);
484 WriteNum( OldPos
-J
-1-L
);
486 WriteCommand( cmdFWDandCOPY
, L
);
487 WriteNum( J
-OldPos
);
497 // Âîîáùå íåò áîëüøå ñîâïàäåíèé, îñòàòîê êîïèðóåì âñòàâêîé.
498 WriteCommand( cmdINSERT
, NewLen
-NewPos
);
499 Data
.Write( New
[ NewPos
], NewLen
-NewPos
);
507 FreeMem( HashTable1
);
508 //FreeMem( HashTable2 );
509 FreeMem( SortTable1
);
510 FreeMem( AccTable1
);
511 //FreeMem( SortTable2 );
512 //FreeMem( AccTable2 );
517 function ApplyUpdates( OutStrm
, SourceStream
, CmdStrm
: PStream
;
518 OnProgress
: TOnUpdatesProgress
): Boolean;
525 function ShowProgress_Cancel
: Boolean;
528 Pr
:= CmdStrm
.Position
* 100 div CmdStrm
.Size
;
530 if Assigned( OnProgress
) then
531 OnProgress( Pr
, CmdStrm
.Position
, CmdStrm
.Size
, Result
);
534 function ReadNum( Shft
: Integer = 0 ): Integer;
538 while CmdStrm
.Position
< CmdStrm
.Size
do
540 CmdStrm
.Read( B
, 1 );
541 Result
:= Result
or ((B
and $7F) shl Shft
);
543 if (B
and $80) = 0 then break
;
553 // íà÷èíàåì ïîñòðîåíèå âûõîäíîãî ôàéëà
555 GetMem( Src
, SourceStream
.Size
);
557 SourceStream
.Position
:= 0;
558 DataLen
:= CmdStrm
.Size
;
559 SourceStream
.Read( Src
^, SourceStream
.Size
);
561 Pos0
:= CmdStrm
.Position
;
562 while CmdStrm
.Position
< DWORD( Pos0
+ DataLen
) do
564 if ShowProgress_Cancel
then Exit
;
566 CmdStrm
.Read( B
, 1 );
570 if (B
and $80) = 0 then
575 if (B
and $40) <> 0 then
576 L
:= L
or ReadNum( 6 );
580 if (B
and $80) = 0 then
583 Cmd
:= cmdBCKandCOPY
;
585 if (B
and $40) <> 0 then
586 L
:= L
or ReadNum( 6 );
588 //cmdFWDandCOPY, cmdBCKandCOPY:
591 if (B
and $80) = 0 then
593 if (B
and $40) = 0 then
596 Cmd
:= cmdBCKandCOPY
;
598 if (B
and $20) <> 0 then
599 L
:= L
or ReadNum( 5 );
605 if (B
and $40) <> 0 then
606 L
:= L
or ReadNum( 6 );
612 cmdNONE
: Exit
;//Assert( Cmd in [cmdFWDandCOPY, cmdINSERT], '!' );
613 cmdINSERT
: Exit
; //Assert( Cmd in [cmdFWDandCOPY, cmdBCKandCOPY], '!' );
621 cmdBCKandCOPY
: Protocol
.Add( 'BCKandCOPY ' + Int2Str( L
) );
622 cmdFWDandCOPY
: Protocol
.Add( 'FWDandCOPY ' + Int2Str( L
) );
623 cmdINSERT
: Protocol
.Add( 'INSERT ' + Int2Str( L
) );
626 // âûïîëíÿåì êîìàíäó:
628 cmdBCKandCOPY
, // ñìåñòèòüñÿ íà N + L + 1 áàéò íàçàä
629 cmdFWDandCOPY
: // èëè âïåðåä è ñêîïèðîâàòü L áàéò èç èñõîäíîãî ôàéëà:
633 Protocol
.Add( 'OFFSET ' + Int2Str( I
) );
635 if Cmd
= cmdBCKandCOPY
then SrcPos
:= SrcPos
- 1 - I
- L
636 else SrcPos
:= SrcPos
+ I
;
638 Protocol
.Add( 'SRCPOS ' + Int2Str( SrcPos
) );
639 if (SrcPos
< 0) or (SrcPos
>= InLen
) then
641 //MsgBox( 'out of bounds', MB_OK );
646 OutStrm
.Write( Src
[ SrcPos
], L
);
650 Move( Src
[ SrcPos
+ L
- I
], S
[ 1 ], I
);
651 Protocol
.Add( 'LAST WRITTEN ARE: <' + S
+ '>' );
655 cmdINSERT
: // âñòàâèòü L áàéòîâ ïðÿìî èç êîìàíäíîãî ôàéëà:
657 Stream2Stream( OutStrm
, CmdStrm
, L
);
659 CmdStrm
.Position
:= CmdStrm
.Position
- DWORD( L
);
661 CmdStrm
.Read( S
[ 1 ], L
);
662 Protocol
.Add( 'DATA: ' + Copy( S
, 1, 100 ) );
673 function MakeUpdates( DestCommandStream
, LastVersion
, PrevVersion
: PStream
;
674 OnProgress
: TOnUpdatesProgress
): Boolean;
679 GetMem( Old
, PrevVersion
.Size
);
680 GetMem( New
, LastVersion
.Size
);
682 if LastVersion
.Size
> 0 then
684 LastVersion
.Position
:= 0;
685 LastVersion
.Read( New
^, LastVersion
.Size
);
687 if PrevVersion
.Size
> 0 then
689 PrevVersion
.Position
:= 0;
690 PrevVersion
.Read( Old
^, PrevVersion
.Size
);
692 if not MakePattern2( DestCommandStream
, Old
, New
, PrevVersion
.Size
,
693 LastVersion
.Size
, max( PrevVersion
.Size
, LastVersion
.Size
), Eq
,
694 OnProgress
) then Exit
;
698 if Old
<> nil then FreeMem( Old
);
699 if New
<> nil then FreeMem( New
);