2 * Copyright (c) 2005, 2006 Rene Scharfe
5 #define USE_THE_REPOSITORY_VARIABLE
7 #include "git-compat-util.h"
14 #include "object-store-ll.h"
16 #include "streaming.h"
17 #include "run-command.h"
18 #include "write-or-die.h"
20 #define RECORDSIZE (512)
21 #define BLOCKSIZE (RECORDSIZE * 20)
23 static char block
[BLOCKSIZE
];
24 static unsigned long offset
;
26 static int tar_umask
= 002;
28 static int write_tar_filter_archive(const struct archiver
*ar
,
29 struct archiver_args
*args
);
32 * This is the max value that a ustar size header can specify, as it is fixed
33 * at 11 octal digits. POSIX specifies that we switch to extended headers at
36 * Likewise for the mtime (which happens to use a buffer of the same size).
38 #if ULONG_MAX == 0xFFFFFFFF
39 #define USTAR_MAX_SIZE ULONG_MAX
41 #define USTAR_MAX_SIZE 077777777777UL
43 #if TIME_MAX == 0xFFFFFFFF
44 #define USTAR_MAX_MTIME TIME_MAX
46 #define USTAR_MAX_MTIME 077777777777ULL
49 static void tar_write_block(const void *buf
)
51 write_or_die(1, buf
, BLOCKSIZE
);
54 static void (*write_block
)(const void *) = tar_write_block
;
56 /* writes out the whole block, but only if it is full */
57 static void write_if_needed(void)
59 if (offset
== BLOCKSIZE
) {
66 * queues up writes, so that all our write(2) calls write exactly one
67 * full block; pads writes to RECORDSIZE
69 static void do_write_blocked(const void *data
, unsigned long size
)
71 const char *buf
= data
;
74 unsigned long chunk
= BLOCKSIZE
- offset
;
77 memcpy(block
+ offset
, buf
, chunk
);
83 while (size
>= BLOCKSIZE
) {
89 memcpy(block
+ offset
, buf
, size
);
94 static void finish_record(void)
97 tail
= offset
% RECORDSIZE
;
99 memset(block
+ offset
, 0, RECORDSIZE
- tail
);
100 offset
+= RECORDSIZE
- tail
;
105 static void write_blocked(const void *data
, unsigned long size
)
107 do_write_blocked(data
, size
);
112 * The end of tar archives is marked by 2*512 nul bytes and after that
113 * follows the rest of the block (if any).
115 static void write_trailer(void)
117 int tail
= BLOCKSIZE
- offset
;
118 memset(block
+ offset
, 0, tail
);
120 if (tail
< 2 * RECORDSIZE
) {
121 memset(block
, 0, offset
);
127 * queues up writes, so that all our write(2) calls write exactly one
128 * full block; pads writes to RECORDSIZE
130 static int stream_blocked(struct repository
*r
, const struct object_id
*oid
)
132 struct git_istream
*st
;
133 enum object_type type
;
138 st
= open_istream(r
, oid
, &type
, &sz
, NULL
);
140 return error(_("cannot stream blob %s"), oid_to_hex(oid
));
142 readlen
= read_istream(st
, buf
, sizeof(buf
));
145 do_write_blocked(buf
, readlen
);
154 * pax extended header records have the format "%u %s=%s\n". %u contains
155 * the size of the whole string (including the %u), the first %s is the
156 * keyword, the second one is the value. This function constructs such a
157 * string and appends it to a struct strbuf.
159 static void strbuf_append_ext_header(struct strbuf
*sb
, const char *keyword
,
160 const char *value
, size_t valuelen
)
162 size_t orig_len
= sb
->len
;
166 len
= 1 + 1 + strlen(keyword
) + 1 + valuelen
+ 1;
167 for (tmp
= 1; len
/ 10 >= tmp
; tmp
*= 10)
170 strbuf_grow(sb
, len
);
171 strbuf_addf(sb
, "%"PRIuMAX
" %s=", (uintmax_t)len
, keyword
);
172 strbuf_add(sb
, value
, valuelen
);
173 strbuf_addch(sb
, '\n');
175 if (len
!= sb
->len
- orig_len
)
176 BUG("pax extended header length miscalculated as %"PRIuMAX
177 ", should be %"PRIuMAX
,
178 (uintmax_t)len
, (uintmax_t)(sb
->len
- orig_len
));
182 * Like strbuf_append_ext_header, but for numeric values.
184 static void strbuf_append_ext_header_uint(struct strbuf
*sb
,
188 char buf
[40]; /* big enough for 2^128 in decimal, plus NUL */
191 len
= xsnprintf(buf
, sizeof(buf
), "%"PRIuMAX
, value
);
192 strbuf_append_ext_header(sb
, keyword
, buf
, len
);
195 static unsigned int ustar_header_chksum(const struct ustar_header
*header
)
197 const unsigned char *p
= (const unsigned char *)header
;
198 unsigned int chksum
= 0;
199 while (p
< (const unsigned char *)header
->chksum
)
201 chksum
+= sizeof(header
->chksum
) * ' ';
202 p
+= sizeof(header
->chksum
);
203 while (p
< (const unsigned char *)header
+ sizeof(struct ustar_header
))
208 static size_t get_path_prefix(const char *path
, size_t pathlen
, size_t maxlen
)
211 if (i
> 1 && path
[i
- 1] == '/')
217 } while (i
> 0 && path
[i
] != '/');
221 static void prepare_header(struct archiver_args
*args
,
222 struct ustar_header
*header
,
223 unsigned int mode
, unsigned long size
)
225 xsnprintf(header
->mode
, sizeof(header
->mode
), "%07o", mode
& 07777);
226 xsnprintf(header
->size
, sizeof(header
->size
), "%011"PRIoMAX
, S_ISREG(mode
) ? (uintmax_t)size
: (uintmax_t)0);
227 xsnprintf(header
->mtime
, sizeof(header
->mtime
), "%011lo", (unsigned long) args
->time
);
229 xsnprintf(header
->uid
, sizeof(header
->uid
), "%07o", 0);
230 xsnprintf(header
->gid
, sizeof(header
->gid
), "%07o", 0);
231 strlcpy(header
->uname
, "root", sizeof(header
->uname
));
232 strlcpy(header
->gname
, "root", sizeof(header
->gname
));
233 xsnprintf(header
->devmajor
, sizeof(header
->devmajor
), "%07o", 0);
234 xsnprintf(header
->devminor
, sizeof(header
->devminor
), "%07o", 0);
236 memcpy(header
->magic
, "ustar", 6);
237 memcpy(header
->version
, "00", 2);
239 xsnprintf(header
->chksum
, sizeof(header
->chksum
), "%07o", ustar_header_chksum(header
));
242 static void write_extended_header(struct archiver_args
*args
,
243 const struct object_id
*oid
,
244 const void *buffer
, unsigned long size
)
246 struct ustar_header header
;
248 memset(&header
, 0, sizeof(header
));
249 *header
.typeflag
= TYPEFLAG_EXT_HEADER
;
251 xsnprintf(header
.name
, sizeof(header
.name
), "%s.paxheader", oid_to_hex(oid
));
252 prepare_header(args
, &header
, mode
, size
);
253 write_blocked(&header
, sizeof(header
));
254 write_blocked(buffer
, size
);
257 static int write_tar_entry(struct archiver_args
*args
,
258 const struct object_id
*oid
,
259 const char *path
, size_t pathlen
,
261 void *buffer
, unsigned long size
)
263 struct ustar_header header
;
264 struct strbuf ext_header
= STRBUF_INIT
;
265 unsigned long size_in_header
;
268 memset(&header
, 0, sizeof(header
));
270 if (S_ISDIR(mode
) || S_ISGITLINK(mode
)) {
271 *header
.typeflag
= TYPEFLAG_DIR
;
272 mode
= (mode
| 0777) & ~tar_umask
;
273 } else if (S_ISLNK(mode
)) {
274 *header
.typeflag
= TYPEFLAG_LNK
;
276 } else if (S_ISREG(mode
)) {
277 *header
.typeflag
= TYPEFLAG_REG
;
278 mode
= (mode
| ((mode
& 0100) ? 0777 : 0666)) & ~tar_umask
;
280 return error(_("unsupported file mode: 0%o (SHA1: %s)"),
281 mode
, oid_to_hex(oid
));
283 if (pathlen
> sizeof(header
.name
)) {
284 size_t plen
= get_path_prefix(path
, pathlen
,
285 sizeof(header
.prefix
));
286 size_t rest
= pathlen
- plen
- 1;
287 if (plen
> 0 && rest
<= sizeof(header
.name
)) {
288 memcpy(header
.prefix
, path
, plen
);
289 memcpy(header
.name
, path
+ plen
+ 1, rest
);
291 xsnprintf(header
.name
, sizeof(header
.name
), "%s.data",
293 strbuf_append_ext_header(&ext_header
, "path",
297 memcpy(header
.name
, path
, pathlen
);
300 if (size
> sizeof(header
.linkname
)) {
301 xsnprintf(header
.linkname
, sizeof(header
.linkname
),
302 "see %s.paxheader", oid_to_hex(oid
));
303 strbuf_append_ext_header(&ext_header
, "linkpath",
306 memcpy(header
.linkname
, buffer
, size
);
309 size_in_header
= size
;
310 if (S_ISREG(mode
) && size
> USTAR_MAX_SIZE
) {
312 strbuf_append_ext_header_uint(&ext_header
, "size", size
);
315 prepare_header(args
, &header
, mode
, size_in_header
);
317 if (ext_header
.len
> 0) {
318 write_extended_header(args
, oid
, ext_header
.buf
,
321 strbuf_release(&ext_header
);
322 write_blocked(&header
, sizeof(header
));
323 if (S_ISREG(mode
) && size
> 0) {
325 write_blocked(buffer
, size
);
327 err
= stream_blocked(args
->repo
, oid
);
332 static void write_global_extended_header(struct archiver_args
*args
)
334 const struct object_id
*oid
= args
->commit_oid
;
335 struct strbuf ext_header
= STRBUF_INIT
;
336 struct ustar_header header
;
340 strbuf_append_ext_header(&ext_header
, "comment",
342 the_hash_algo
->hexsz
);
343 if (args
->time
> USTAR_MAX_MTIME
) {
344 strbuf_append_ext_header_uint(&ext_header
, "mtime",
346 args
->time
= USTAR_MAX_MTIME
;
352 memset(&header
, 0, sizeof(header
));
353 *header
.typeflag
= TYPEFLAG_GLOBAL_HEADER
;
355 xsnprintf(header
.name
, sizeof(header
.name
), "pax_global_header");
356 prepare_header(args
, &header
, mode
, ext_header
.len
);
357 write_blocked(&header
, sizeof(header
));
358 write_blocked(ext_header
.buf
, ext_header
.len
);
359 strbuf_release(&ext_header
);
362 static struct archiver
**tar_filters
;
363 static int nr_tar_filters
;
364 static int alloc_tar_filters
;
366 static struct archiver
*find_tar_filter(const char *name
, size_t len
)
369 for (i
= 0; i
< nr_tar_filters
; i
++) {
370 struct archiver
*ar
= tar_filters
[i
];
371 if (!xstrncmpz(ar
->name
, name
, len
))
377 static int tar_filter_config(const char *var
, const char *value
,
385 if (parse_config_key(var
, "tar", &name
, &namelen
, &type
) < 0 || !name
)
388 ar
= find_tar_filter(name
, namelen
);
391 ar
->name
= xmemdupz(name
, namelen
);
392 ar
->write_archive
= write_tar_filter_archive
;
393 ar
->flags
= ARCHIVER_WANT_COMPRESSION_LEVELS
|
394 ARCHIVER_HIGH_COMPRESSION_LEVELS
;
395 ALLOC_GROW(tar_filters
, nr_tar_filters
+ 1, alloc_tar_filters
);
396 tar_filters
[nr_tar_filters
++] = ar
;
399 if (!strcmp(type
, "command")) {
401 return config_error_nonbool(var
);
402 free(ar
->filter_command
);
403 ar
->filter_command
= xstrdup(value
);
406 if (!strcmp(type
, "remote")) {
407 if (git_config_bool(var
, value
))
408 ar
->flags
|= ARCHIVER_REMOTE
;
410 ar
->flags
&= ~ARCHIVER_REMOTE
;
417 static int git_tar_config(const char *var
, const char *value
,
418 const struct config_context
*ctx
, void *cb
)
420 if (!strcmp(var
, "tar.umask")) {
421 if (value
&& !strcmp(value
, "user")) {
422 tar_umask
= umask(0);
425 tar_umask
= git_config_int(var
, value
, ctx
->kvi
);
430 return tar_filter_config(var
, value
, cb
);
433 static int write_tar_archive(const struct archiver
*ar UNUSED
,
434 struct archiver_args
*args
)
438 write_global_extended_header(args
);
439 err
= write_archive_entries(args
, write_tar_entry
);
445 static git_zstream gzstream
;
446 static unsigned char outbuf
[16384];
448 static void tgz_deflate(int flush
)
450 while (gzstream
.avail_in
|| flush
== Z_FINISH
) {
451 int status
= git_deflate(&gzstream
, flush
);
452 if (!gzstream
.avail_out
|| status
== Z_STREAM_END
) {
453 write_or_die(1, outbuf
, gzstream
.next_out
- outbuf
);
454 gzstream
.next_out
= outbuf
;
455 gzstream
.avail_out
= sizeof(outbuf
);
456 if (status
== Z_STREAM_END
)
459 if (status
!= Z_OK
&& status
!= Z_BUF_ERROR
)
460 die(_("deflate error (%d)"), status
);
464 static void tgz_write_block(const void *data
)
466 gzstream
.next_in
= (void *)data
;
467 gzstream
.avail_in
= BLOCKSIZE
;
468 tgz_deflate(Z_NO_FLUSH
);
471 static const char internal_gzip_command
[] = "git archive gzip";
473 static int write_tar_filter_archive(const struct archiver
*ar
,
474 struct archiver_args
*args
)
476 #if ZLIB_VERNUM >= 0x1221
477 struct gz_header_s gzhead
= { .os
= 3 }; /* Unix, for reproducibility */
479 struct strbuf cmd
= STRBUF_INIT
;
480 struct child_process filter
= CHILD_PROCESS_INIT
;
483 if (!ar
->filter_command
)
484 BUG("tar-filter archiver called with no filter defined");
486 if (!strcmp(ar
->filter_command
, internal_gzip_command
)) {
487 write_block
= tgz_write_block
;
488 git_deflate_init_gzip(&gzstream
, args
->compression_level
);
489 #if ZLIB_VERNUM >= 0x1221
490 if (deflateSetHeader(&gzstream
.z
, &gzhead
) != Z_OK
)
491 BUG("deflateSetHeader() called too late");
493 gzstream
.next_out
= outbuf
;
494 gzstream
.avail_out
= sizeof(outbuf
);
496 r
= write_tar_archive(ar
, args
);
498 tgz_deflate(Z_FINISH
);
499 git_deflate_end(&gzstream
);
503 strbuf_addstr(&cmd
, ar
->filter_command
);
504 if (args
->compression_level
>= 0)
505 strbuf_addf(&cmd
, " -%d", args
->compression_level
);
507 strvec_push(&filter
.args
, cmd
.buf
);
508 filter
.use_shell
= 1;
510 filter
.silent_exec_failure
= 1;
512 if (start_command(&filter
) < 0)
513 die_errno(_("unable to start '%s' filter"), cmd
.buf
);
515 if (dup2(filter
.in
, 1) < 0)
516 die_errno(_("unable to redirect descriptor"));
519 r
= write_tar_archive(ar
, args
);
522 if (finish_command(&filter
) != 0)
523 die(_("'%s' filter reported error"), cmd
.buf
);
525 strbuf_release(&cmd
);
529 static struct archiver tar_archiver
= {
531 .write_archive
= write_tar_archive
,
532 .flags
= ARCHIVER_REMOTE
,
535 void init_tar_archiver(void)
538 register_archiver(&tar_archiver
);
540 tar_filter_config("tar.tgz.command", internal_gzip_command
, NULL
);
541 tar_filter_config("tar.tgz.remote", "true", NULL
);
542 tar_filter_config("tar.tar.gz.command", internal_gzip_command
, NULL
);
543 tar_filter_config("tar.tar.gz.remote", "true", NULL
);
544 git_config(git_tar_config
, NULL
);
545 for (i
= 0; i
< nr_tar_filters
; i
++) {
546 /* omit any filters that never had a command configured */
547 if (tar_filters
[i
]->filter_command
)
548 register_archiver(tar_filters
[i
]);