2 * Copyright (c) 2006 Rene Scharfe
5 #define USE_THE_REPOSITORY_VARIABLE
7 #include "git-compat-util.h"
13 #include "streaming.h"
15 #include "object-store-ll.h"
18 #include "write-or-die.h"
19 #include "xdiff-interface.h"
25 /* We only care about the "buf" part here. */
26 static struct strbuf zip_dir
;
28 static uintmax_t zip_offset
;
29 static uint64_t zip_dir_entries
;
31 static unsigned int max_creator_version
;
33 #define ZIP_STREAM (1 << 3)
34 #define ZIP_UTF8 (1 << 11)
38 ZIP_METHOD_DEFLATE
= 8
41 struct zip_local_header
{
42 unsigned char magic
[4];
43 unsigned char version
[2];
44 unsigned char flags
[2];
45 unsigned char compression_method
[2];
46 unsigned char mtime
[2];
47 unsigned char mdate
[2];
48 unsigned char crc32
[4];
49 unsigned char compressed_size
[4];
50 unsigned char size
[4];
51 unsigned char filename_length
[2];
52 unsigned char extra_length
[2];
53 unsigned char _end
[1];
56 struct zip_data_desc
{
57 unsigned char magic
[4];
58 unsigned char crc32
[4];
59 unsigned char compressed_size
[4];
60 unsigned char size
[4];
61 unsigned char _end
[1];
64 struct zip64_data_desc
{
65 unsigned char magic
[4];
66 unsigned char crc32
[4];
67 unsigned char compressed_size
[8];
68 unsigned char size
[8];
69 unsigned char _end
[1];
72 struct zip_dir_trailer
{
73 unsigned char magic
[4];
74 unsigned char disk
[2];
75 unsigned char directory_start_disk
[2];
76 unsigned char entries_on_this_disk
[2];
77 unsigned char entries
[2];
78 unsigned char size
[4];
79 unsigned char offset
[4];
80 unsigned char comment_length
[2];
81 unsigned char _end
[1];
84 struct zip_extra_mtime
{
85 unsigned char magic
[2];
86 unsigned char extra_size
[2];
87 unsigned char flags
[1];
88 unsigned char mtime
[4];
89 unsigned char _end
[1];
93 unsigned char magic
[2];
94 unsigned char extra_size
[2];
95 unsigned char size
[8];
96 unsigned char compressed_size
[8];
97 unsigned char _end
[1];
100 struct zip64_dir_trailer
{
101 unsigned char magic
[4];
102 unsigned char record_size
[8];
103 unsigned char creator_version
[2];
104 unsigned char version
[2];
105 unsigned char disk
[4];
106 unsigned char directory_start_disk
[4];
107 unsigned char entries_on_this_disk
[8];
108 unsigned char entries
[8];
109 unsigned char size
[8];
110 unsigned char offset
[8];
111 unsigned char _end
[1];
114 struct zip64_dir_trailer_locator
{
115 unsigned char magic
[4];
116 unsigned char disk
[4];
117 unsigned char offset
[8];
118 unsigned char number_of_disks
[4];
119 unsigned char _end
[1];
123 * On ARM, padding is added at the end of the struct, so a simple
124 * sizeof(struct ...) reports two bytes more than the payload size
125 * we're interested in.
127 #define ZIP_LOCAL_HEADER_SIZE offsetof(struct zip_local_header, _end)
128 #define ZIP_DATA_DESC_SIZE offsetof(struct zip_data_desc, _end)
129 #define ZIP64_DATA_DESC_SIZE offsetof(struct zip64_data_desc, _end)
130 #define ZIP_DIR_HEADER_SIZE offsetof(struct zip_dir_header, _end)
131 #define ZIP_DIR_TRAILER_SIZE offsetof(struct zip_dir_trailer, _end)
132 #define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end)
133 #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
134 (ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
135 #define ZIP64_EXTRA_SIZE offsetof(struct zip64_extra, _end)
136 #define ZIP64_EXTRA_PAYLOAD_SIZE \
137 (ZIP64_EXTRA_SIZE - offsetof(struct zip64_extra, size))
138 #define ZIP64_DIR_TRAILER_SIZE offsetof(struct zip64_dir_trailer, _end)
139 #define ZIP64_DIR_TRAILER_RECORD_SIZE \
140 (ZIP64_DIR_TRAILER_SIZE - \
141 offsetof(struct zip64_dir_trailer, creator_version))
142 #define ZIP64_DIR_TRAILER_LOCATOR_SIZE \
143 offsetof(struct zip64_dir_trailer_locator, _end)
145 static void copy_le16(unsigned char *dest
, unsigned int n
)
148 dest
[1] = 0xff & (n
>> 010);
151 static void copy_le32(unsigned char *dest
, unsigned int n
)
154 dest
[1] = 0xff & (n
>> 010);
155 dest
[2] = 0xff & (n
>> 020);
156 dest
[3] = 0xff & (n
>> 030);
159 static void copy_le64(unsigned char *dest
, uint64_t n
)
162 dest
[1] = 0xff & (n
>> 010);
163 dest
[2] = 0xff & (n
>> 020);
164 dest
[3] = 0xff & (n
>> 030);
165 dest
[4] = 0xff & (n
>> 040);
166 dest
[5] = 0xff & (n
>> 050);
167 dest
[6] = 0xff & (n
>> 060);
168 dest
[7] = 0xff & (n
>> 070);
171 static uint64_t clamp_max(uint64_t n
, uint64_t max
, int *clamped
)
179 static void copy_le16_clamp(unsigned char *dest
, uint64_t n
, int *clamped
)
181 copy_le16(dest
, clamp_max(n
, 0xffff, clamped
));
184 static void copy_le32_clamp(unsigned char *dest
, uint64_t n
, int *clamped
)
186 copy_le32(dest
, clamp_max(n
, 0xffffffff, clamped
));
189 static int strbuf_add_le(struct strbuf
*sb
, size_t size
, uintmax_t n
)
192 strbuf_addch(sb
, n
& 0xff);
198 static uint32_t clamp32(uintmax_t n
)
200 const uintmax_t max
= 0xffffffff;
201 return (n
< max
) ? n
: max
;
204 static void *zlib_deflate_raw(void *data
, unsigned long size
,
205 int compression_level
,
206 unsigned long *compressed_size
)
209 unsigned long maxsize
;
213 git_deflate_init_raw(&stream
, compression_level
);
214 maxsize
= git_deflate_bound(&stream
, size
);
215 buffer
= xmalloc(maxsize
);
217 stream
.next_in
= data
;
218 stream
.avail_in
= size
;
219 stream
.next_out
= buffer
;
220 stream
.avail_out
= maxsize
;
223 result
= git_deflate(&stream
, Z_FINISH
);
224 } while (result
== Z_OK
);
226 if (result
!= Z_STREAM_END
) {
231 git_deflate_end(&stream
);
232 *compressed_size
= stream
.total_out
;
237 static void write_zip_data_desc(unsigned long size
,
238 unsigned long compressed_size
,
241 if (size
>= 0xffffffff || compressed_size
>= 0xffffffff) {
242 struct zip64_data_desc trailer
;
243 copy_le32(trailer
.magic
, 0x08074b50);
244 copy_le32(trailer
.crc32
, crc
);
245 copy_le64(trailer
.compressed_size
, compressed_size
);
246 copy_le64(trailer
.size
, size
);
247 write_or_die(1, &trailer
, ZIP64_DATA_DESC_SIZE
);
248 zip_offset
+= ZIP64_DATA_DESC_SIZE
;
250 struct zip_data_desc trailer
;
251 copy_le32(trailer
.magic
, 0x08074b50);
252 copy_le32(trailer
.crc32
, crc
);
253 copy_le32(trailer
.compressed_size
, compressed_size
);
254 copy_le32(trailer
.size
, size
);
255 write_or_die(1, &trailer
, ZIP_DATA_DESC_SIZE
);
256 zip_offset
+= ZIP_DATA_DESC_SIZE
;
260 static void set_zip_header_data_desc(struct zip_local_header
*header
,
262 unsigned long compressed_size
,
265 copy_le32(header
->crc32
, crc
);
266 copy_le32(header
->compressed_size
, compressed_size
);
267 copy_le32(header
->size
, size
);
270 static int has_only_ascii(const char *s
)
281 static int entry_is_binary(struct index_state
*istate
, const char *path
,
282 const void *buffer
, size_t size
)
284 struct userdiff_driver
*driver
= userdiff_find_by_path(istate
, path
);
286 driver
= userdiff_find_by_name("default");
287 if (driver
->binary
!= -1)
288 return driver
->binary
;
289 return buffer_is_binary(buffer
, size
);
292 #define STREAM_BUFFER_SIZE (1024 * 16)
294 static int write_zip_entry(struct archiver_args
*args
,
295 const struct object_id
*oid
,
296 const char *path
, size_t pathlen
,
298 void *buffer
, unsigned long size
)
300 struct zip_local_header header
;
301 uintmax_t offset
= zip_offset
;
302 struct zip_extra_mtime extra
;
303 struct zip64_extra extra64
;
304 size_t header_extra_size
= ZIP_EXTRA_MTIME_SIZE
;
305 int need_zip64_extra
= 0;
307 unsigned long compressed_size
;
309 enum zip_method method
;
311 void *deflated
= NULL
;
312 struct git_istream
*stream
= NULL
;
313 unsigned long flags
= 0;
315 const char *path_without_prefix
= path
+ args
->baselen
;
316 unsigned int creator_version
= 0;
317 unsigned int version_needed
= 10;
318 size_t zip_dir_extra_size
= ZIP_EXTRA_MTIME_SIZE
;
319 size_t zip64_dir_extra_payload_size
= 0;
321 crc
= crc32(0, NULL
, 0);
323 if (!has_only_ascii(path
)) {
327 warning(_("path is not valid UTF-8: %s"), path
);
330 if (pathlen
> 0xffff) {
331 return error(_("path too long (%d chars, SHA1: %s): %s"),
332 (int)pathlen
, oid_to_hex(oid
), path
);
335 if (S_ISDIR(mode
) || S_ISGITLINK(mode
)) {
336 method
= ZIP_METHOD_STORE
;
340 } else if (S_ISREG(mode
) || S_ISLNK(mode
)) {
341 method
= ZIP_METHOD_STORE
;
342 attr2
= S_ISLNK(mode
) ? ((mode
| 0777) << 16) :
343 (mode
& 0111) ? ((mode
) << 16) : 0;
344 if (S_ISLNK(mode
) || (mode
& 0111))
345 creator_version
= 0x0317;
346 if (S_ISREG(mode
) && args
->compression_level
!= 0 && size
> 0)
347 method
= ZIP_METHOD_DEFLATE
;
350 enum object_type type
;
351 stream
= open_istream(args
->repo
, oid
, &type
, &size
,
354 return error(_("cannot stream blob %s"),
359 crc
= crc32(crc
, buffer
, size
);
360 is_binary
= entry_is_binary(args
->repo
->index
,
365 compressed_size
= (method
== ZIP_METHOD_STORE
) ? size
: 0;
367 return error(_("unsupported file mode: 0%o (SHA1: %s)"), mode
,
371 if (creator_version
> max_creator_version
)
372 max_creator_version
= creator_version
;
374 if (buffer
&& method
== ZIP_METHOD_DEFLATE
) {
375 out
= deflated
= zlib_deflate_raw(buffer
, size
,
376 args
->compression_level
,
378 if (!out
|| compressed_size
>= size
) {
380 method
= ZIP_METHOD_STORE
;
381 compressed_size
= size
;
385 copy_le16(extra
.magic
, 0x5455);
386 copy_le16(extra
.extra_size
, ZIP_EXTRA_MTIME_PAYLOAD_SIZE
);
387 extra
.flags
[0] = 1; /* just mtime */
388 copy_le32(extra
.mtime
, args
->time
);
390 if (size
> 0xffffffff || compressed_size
> 0xffffffff)
391 need_zip64_extra
= 1;
392 if (stream
&& size
> 0x7fffffff)
393 need_zip64_extra
= 1;
395 if (need_zip64_extra
)
398 copy_le32(header
.magic
, 0x04034b50);
399 copy_le16(header
.version
, version_needed
);
400 copy_le16(header
.flags
, flags
);
401 copy_le16(header
.compression_method
, method
);
402 copy_le16(header
.mtime
, zip_time
);
403 copy_le16(header
.mdate
, zip_date
);
404 if (need_zip64_extra
) {
405 set_zip_header_data_desc(&header
, 0xffffffff, 0xffffffff, crc
);
406 header_extra_size
+= ZIP64_EXTRA_SIZE
;
408 set_zip_header_data_desc(&header
, size
, compressed_size
, crc
);
410 copy_le16(header
.filename_length
, pathlen
);
411 copy_le16(header
.extra_length
, header_extra_size
);
412 write_or_die(1, &header
, ZIP_LOCAL_HEADER_SIZE
);
413 zip_offset
+= ZIP_LOCAL_HEADER_SIZE
;
414 write_or_die(1, path
, pathlen
);
415 zip_offset
+= pathlen
;
416 write_or_die(1, &extra
, ZIP_EXTRA_MTIME_SIZE
);
417 zip_offset
+= ZIP_EXTRA_MTIME_SIZE
;
418 if (need_zip64_extra
) {
419 copy_le16(extra64
.magic
, 0x0001);
420 copy_le16(extra64
.extra_size
, ZIP64_EXTRA_PAYLOAD_SIZE
);
421 copy_le64(extra64
.size
, size
);
422 copy_le64(extra64
.compressed_size
, compressed_size
);
423 write_or_die(1, &extra64
, ZIP64_EXTRA_SIZE
);
424 zip_offset
+= ZIP64_EXTRA_SIZE
;
427 if (stream
&& method
== ZIP_METHOD_STORE
) {
428 unsigned char buf
[STREAM_BUFFER_SIZE
];
432 readlen
= read_istream(stream
, buf
, sizeof(buf
));
435 crc
= crc32(crc
, buf
, readlen
);
437 is_binary
= entry_is_binary(args
->repo
->index
,
440 write_or_die(1, buf
, readlen
);
442 close_istream(stream
);
446 compressed_size
= size
;
447 zip_offset
+= compressed_size
;
449 write_zip_data_desc(size
, compressed_size
, crc
);
450 } else if (stream
&& method
== ZIP_METHOD_DEFLATE
) {
451 unsigned char buf
[STREAM_BUFFER_SIZE
];
456 unsigned char compressed
[STREAM_BUFFER_SIZE
* 2];
458 git_deflate_init_raw(&zstream
, args
->compression_level
);
461 zstream
.next_out
= compressed
;
462 zstream
.avail_out
= sizeof(compressed
);
465 readlen
= read_istream(stream
, buf
, sizeof(buf
));
468 crc
= crc32(crc
, buf
, readlen
);
470 is_binary
= entry_is_binary(args
->repo
->index
,
474 zstream
.next_in
= buf
;
475 zstream
.avail_in
= readlen
;
476 result
= git_deflate(&zstream
, 0);
478 die(_("deflate error (%d)"), result
);
479 out_len
= zstream
.next_out
- compressed
;
482 write_or_die(1, compressed
, out_len
);
483 compressed_size
+= out_len
;
484 zstream
.next_out
= compressed
;
485 zstream
.avail_out
= sizeof(compressed
);
489 close_istream(stream
);
493 zstream
.next_in
= buf
;
494 zstream
.avail_in
= 0;
495 result
= git_deflate(&zstream
, Z_FINISH
);
496 if (result
!= Z_STREAM_END
)
497 die("deflate error (%d)", result
);
499 git_deflate_end(&zstream
);
500 out_len
= zstream
.next_out
- compressed
;
501 write_or_die(1, compressed
, out_len
);
502 compressed_size
+= out_len
;
503 zip_offset
+= compressed_size
;
505 write_zip_data_desc(size
, compressed_size
, crc
);
506 } else if (compressed_size
> 0) {
507 write_or_die(1, out
, compressed_size
);
508 zip_offset
+= compressed_size
;
513 if (compressed_size
> 0xffffffff || size
> 0xffffffff ||
514 offset
> 0xffffffff) {
515 if (compressed_size
>= 0xffffffff)
516 zip64_dir_extra_payload_size
+= 8;
517 if (size
>= 0xffffffff)
518 zip64_dir_extra_payload_size
+= 8;
519 if (offset
>= 0xffffffff)
520 zip64_dir_extra_payload_size
+= 8;
521 zip_dir_extra_size
+= 2 + 2 + zip64_dir_extra_payload_size
;
524 strbuf_add_le(&zip_dir
, 4, 0x02014b50); /* magic */
525 strbuf_add_le(&zip_dir
, 2, creator_version
);
526 strbuf_add_le(&zip_dir
, 2, version_needed
);
527 strbuf_add_le(&zip_dir
, 2, flags
);
528 strbuf_add_le(&zip_dir
, 2, method
);
529 strbuf_add_le(&zip_dir
, 2, zip_time
);
530 strbuf_add_le(&zip_dir
, 2, zip_date
);
531 strbuf_add_le(&zip_dir
, 4, crc
);
532 strbuf_add_le(&zip_dir
, 4, clamp32(compressed_size
));
533 strbuf_add_le(&zip_dir
, 4, clamp32(size
));
534 strbuf_add_le(&zip_dir
, 2, pathlen
);
535 strbuf_add_le(&zip_dir
, 2, zip_dir_extra_size
);
536 strbuf_add_le(&zip_dir
, 2, 0); /* comment length */
537 strbuf_add_le(&zip_dir
, 2, 0); /* disk */
538 strbuf_add_le(&zip_dir
, 2, !is_binary
);
539 strbuf_add_le(&zip_dir
, 4, attr2
);
540 strbuf_add_le(&zip_dir
, 4, clamp32(offset
));
541 strbuf_add(&zip_dir
, path
, pathlen
);
542 strbuf_add(&zip_dir
, &extra
, ZIP_EXTRA_MTIME_SIZE
);
543 if (zip64_dir_extra_payload_size
) {
544 strbuf_add_le(&zip_dir
, 2, 0x0001); /* magic */
545 strbuf_add_le(&zip_dir
, 2, zip64_dir_extra_payload_size
);
546 if (size
>= 0xffffffff)
547 strbuf_add_le(&zip_dir
, 8, size
);
548 if (compressed_size
>= 0xffffffff)
549 strbuf_add_le(&zip_dir
, 8, compressed_size
);
550 if (offset
>= 0xffffffff)
551 strbuf_add_le(&zip_dir
, 8, offset
);
558 static void write_zip64_trailer(void)
560 struct zip64_dir_trailer trailer64
;
561 struct zip64_dir_trailer_locator locator64
;
563 copy_le32(trailer64
.magic
, 0x06064b50);
564 copy_le64(trailer64
.record_size
, ZIP64_DIR_TRAILER_RECORD_SIZE
);
565 copy_le16(trailer64
.creator_version
, max_creator_version
);
566 copy_le16(trailer64
.version
, 45);
567 copy_le32(trailer64
.disk
, 0);
568 copy_le32(trailer64
.directory_start_disk
, 0);
569 copy_le64(trailer64
.entries_on_this_disk
, zip_dir_entries
);
570 copy_le64(trailer64
.entries
, zip_dir_entries
);
571 copy_le64(trailer64
.size
, zip_dir
.len
);
572 copy_le64(trailer64
.offset
, zip_offset
);
574 copy_le32(locator64
.magic
, 0x07064b50);
575 copy_le32(locator64
.disk
, 0);
576 copy_le64(locator64
.offset
, zip_offset
+ zip_dir
.len
);
577 copy_le32(locator64
.number_of_disks
, 1);
579 write_or_die(1, &trailer64
, ZIP64_DIR_TRAILER_SIZE
);
580 write_or_die(1, &locator64
, ZIP64_DIR_TRAILER_LOCATOR_SIZE
);
583 static void write_zip_trailer(const struct object_id
*oid
)
585 struct zip_dir_trailer trailer
;
588 copy_le32(trailer
.magic
, 0x06054b50);
589 copy_le16(trailer
.disk
, 0);
590 copy_le16(trailer
.directory_start_disk
, 0);
591 copy_le16_clamp(trailer
.entries_on_this_disk
, zip_dir_entries
,
593 copy_le16_clamp(trailer
.entries
, zip_dir_entries
, &clamped
);
594 copy_le32(trailer
.size
, zip_dir
.len
);
595 copy_le32_clamp(trailer
.offset
, zip_offset
, &clamped
);
596 copy_le16(trailer
.comment_length
, oid
? the_hash_algo
->hexsz
: 0);
598 write_or_die(1, zip_dir
.buf
, zip_dir
.len
);
600 write_zip64_trailer();
601 write_or_die(1, &trailer
, ZIP_DIR_TRAILER_SIZE
);
603 write_or_die(1, oid_to_hex(oid
), the_hash_algo
->hexsz
);
606 static void dos_time(timestamp_t
*timestamp
, int *dos_date
, int *dos_time
)
611 if (date_overflows(*timestamp
))
612 die(_("timestamp too large for this system: %"PRItime
),
614 time
= (time_t)*timestamp
;
615 localtime_r(&time
, &tm
);
618 *dos_date
= tm
.tm_mday
+ (tm
.tm_mon
+ 1) * 32 +
619 (tm
.tm_year
+ 1900 - 1980) * 512;
620 *dos_time
= tm
.tm_sec
/ 2 + tm
.tm_min
* 32 + tm
.tm_hour
* 2048;
623 static int archive_zip_config(const char *var
, const char *value
,
624 const struct config_context
*ctx UNUSED
,
627 return userdiff_config(var
, value
);
630 static int write_zip_archive(const struct archiver
*ar UNUSED
,
631 struct archiver_args
*args
)
635 git_config(archive_zip_config
, NULL
);
637 dos_time(&args
->time
, &zip_date
, &zip_time
);
639 strbuf_init(&zip_dir
, 0);
641 err
= write_archive_entries(args
, write_zip_entry
);
643 write_zip_trailer(args
->commit_oid
);
645 strbuf_release(&zip_dir
);
650 static struct archiver zip_archiver
= {
652 .write_archive
= write_zip_archive
,
653 .flags
= ARCHIVER_WANT_COMPRESSION_LEVELS
|ARCHIVER_REMOTE
,
656 void init_zip_archiver(void)
658 register_archiver(&zip_archiver
);