1 package Git
::SVN
::Fetcher
;
2 use vars qw
/@ISA $_ignore_regex $_include_regex $_preserve_empty_dirs
3 $_placeholder_filename @deleted_gpath %added_placeholder
9 use File
::Basename qw
/dirname/;
11 use Git qw
/command command_oneline command_noisy command_output_pipe
12 command_input_pipe command_close_pipe
13 command_bidi_pipe command_close_bidi_pipe
/;
15 @ISA = qw(SVN::Delta::Editor);
18 # file baton members: path, mode_a, mode_b, pool, fh, blob, base
20 my ($class, $git_svn, $switch_path) = @_;
21 my $self = SVN
::Delta
::Editor
->new;
23 if (exists $git_svn->{last_commit
}) {
24 $self->{c
} = $git_svn->{last_commit
};
25 $self->{empty_symlinks
} =
26 _mark_empty_symlinks
($git_svn, $switch_path);
29 # some options are read globally, but can be overridden locally
30 # per [svn-remote "..."] section. Command-line options will *NOT*
31 # override options set in an [svn-remote "..."] section
32 $repo_id = $git_svn->{repo_id
};
33 my $k = "svn-remote.$repo_id.ignore-paths";
34 my $v = eval { command_oneline
('config', '--get', $k) };
35 $self->{ignore_regex
} = $v;
37 $k = "svn-remote.$repo_id.include-paths";
38 $v = eval { command_oneline
('config', '--get', $k) };
39 $self->{include_regex
} = $v;
41 $k = "svn-remote.$repo_id.preserve-empty-dirs";
42 $v = eval { command_oneline
('config', '--get', '--bool', $k) };
43 if ($v && $v eq 'true') {
44 $_preserve_empty_dirs = 1;
45 $k = "svn-remote.$repo_id.placeholder-filename";
46 $v = eval { command_oneline
('config', '--get', $k) };
47 $_placeholder_filename = $v;
50 # Load the list of placeholder files added during previous invocations.
51 $k = "svn-remote.$repo_id.added-placeholder";
52 $v = eval { command_oneline
('config', '--get-all', $k) };
53 if ($_preserve_empty_dirs && $v) {
54 # command() prints errors to stderr, so we only call it if
55 # command_oneline() succeeded.
56 my @v = command
('config', '--get-all', $k);
57 $added_placeholder{ dirname
($_) } = $_ foreach @v;
61 $self->{dir_prop
} = {};
62 $self->{file_prop
} = {};
63 $self->{absent_dir
} = {};
64 $self->{absent_file
} = {};
65 require Git
::IndexInfo
;
66 $self->{gii
} = $git_svn->tmp_index_do(sub { Git
::IndexInfo
->new });
67 $self->{pathnameencoding
} = Git
::config
('svn.pathnameencoding');
71 # this uses the Ra object, so it must be called before do_{switch,update},
72 # not inside them (when the Git::SVN::Fetcher object is passed) to
74 sub _mark_empty_symlinks
{
75 my ($git_svn, $switch_path) = @_;
76 my $bool = Git
::config_bool
('svn.brokenSymlinkWorkaround');
77 return {} if (!defined($bool)) || (defined($bool) && ! $bool);
80 my ($rev, $cmt) = $git_svn->last_rev_commit;
81 return {} unless ($rev && $cmt);
83 # allow the warning to be printed for each revision we fetch to
84 # ensure the user sees it. The user can also disable the workaround
85 # on the repository even while git svn is running and the next
86 # revision fetched will skip this expensive function.
88 chomp(my $empty_blob = `git hash-object -t blob --stdin < /dev/null`);
89 my ($ls, $ctx) = command_output_pipe
(qw
/ls-tree -r -z/, $cmt);
91 my $pfx = defined($switch_path) ?
$switch_path : $git_svn->path;
92 $pfx .= '/' if length($pfx);
95 s/\A100644 blob $empty_blob\t//o or next;
96 unless ($printed_warning) {
97 print STDERR
"Scanning for empty symlinks, ",
98 "this may take a while if you have ",
100 "You may disable this with `",
101 "git config svn.brokenSymlinkWorkaround ",
103 "This may be done in a different ",
104 "terminal without restarting ",
106 $printed_warning = 1;
110 $git_svn->ra->get_file($pfx.$path, $rev, undef);
111 if ($props->{'svn:special'}) {
115 command_close_pipe
($ls, $ctx);
119 # returns true if a given path is inside a ".git" directory
121 $_[0] =~ m{(?:^|/)\.git(?:/|$)};
124 # return value: 0 -- don't ignore, 1 -- ignore
125 # This will also check whether the path is explicitly included
126 sub is_path_ignored
{
127 my ($self, $path) = @_;
128 return 1 if in_dot_git
($path);
129 return 1 if defined($self->{ignore_regex
}) &&
130 $path =~ m!$self->{ignore_regex}!;
131 return 0 if defined($self->{include_regex
}) &&
132 $path =~ m!$self->{include_regex}!;
133 return 0 if defined($_include_regex) &&
134 $path =~ m!$_include_regex!;
135 return 1 if defined($self->{include_regex
});
136 return 1 if defined($_include_regex);
137 return 0 unless defined($_ignore_regex);
138 return 1 if $path =~ m!$_ignore_regex!o;
143 my ($self, $path) = @_;
144 $self->{path_strip
} = qr/^\Q$path\E(\/|$)/ if length $path;
152 my ($self, $path, $pb, $rev) = @_;
157 my ($self, $path) = @_;
158 if (my $enc = $self->{pathnameencoding
}) {
160 Encode
::from_to
($path, 'UTF-8', $enc);
162 if ($self->{path_strip
}) {
163 $path =~ s!$self->{path_strip}!! or
164 die "Failed to strip path '$path' ($self->{path_strip})\n";
170 my ($self, $path, $rev, $pb) = @_;
171 return undef if $self->is_path_ignored($path);
173 my $gpath = $self->git_path($path);
174 return undef if ($gpath eq '');
176 # remove entire directories.
177 my ($tree) = (command
('ls-tree', '-z', $self->{c
}, "./$gpath")
178 =~ /\A040000 tree ([a-f\d]{40})\t\Q$gpath\E\0/);
180 my ($ls, $ctx) = command_output_pipe
(qw
/ls
-tree
186 my $rmpath = "$gpath/$_";
187 $self->{gii
}->remove($rmpath);
188 print "\tD\t$rmpath\n" unless $::_q
;
190 print "\tD\t$gpath/\n" unless $::_q
;
191 command_close_pipe
($ls, $ctx);
193 $self->{gii
}->remove($gpath);
194 print "\tD\t$gpath\n" unless $::_q
;
196 # Don't add to @deleted_gpath if we're deleting a placeholder file.
197 push @deleted_gpath, $gpath unless $added_placeholder{dirname
($path)};
198 $self->{empty
}->{$path} = 0;
203 my ($self, $path, $pb, $rev) = @_;
206 goto out
if $self->is_path_ignored($path);
208 my $gpath = $self->git_path($path);
209 ($mode, $blob) = (command
('ls-tree', '-z', $self->{c
}, "./$gpath")
210 =~ /\A(\d{6}) blob ([a-f\d]{40})\t\Q$gpath\E\0/);
211 unless (defined $mode && defined $blob) {
212 die "$path was not found in commit $self->{c} (r$rev)\n";
214 if ($mode eq '100644' && $self->{empty_symlinks
}->{$path}) {
218 { path
=> $path, mode_a
=> $mode, mode_b
=> $mode, blob
=> $blob,
219 pool
=> SVN
::Pool
->new, action
=> 'M' };
223 my ($self, $path, $pb, $cp_path, $cp_rev) = @_;
226 if (!$self->is_path_ignored($path)) {
227 my ($dir, $file) = ($path =~ m
#^(.*?)/?([^/]+)$#);
228 delete $self->{empty
}->{$dir};
231 if ($added_placeholder{$dir}) {
232 # Remove our placeholder file, if we created one.
233 delete_entry
($self, $added_placeholder{$dir})
234 unless $path eq $added_placeholder{$dir};
235 delete $added_placeholder{$dir}
239 { path
=> $path, mode_a
=> $mode, mode_b
=> $mode,
240 pool
=> SVN
::Pool
->new, action
=> 'A' };
244 my ($self, $path, $cp_path, $cp_rev) = @_;
245 goto out
if $self->is_path_ignored($path);
246 my $gpath = $self->git_path($path);
248 my ($ls, $ctx) = command_output_pipe
(qw
/ls
-tree
254 $self->{gii
}->remove($_);
255 print "\tD\t$_\n" unless $::_q
;
256 push @deleted_gpath, $gpath;
258 command_close_pipe
($ls, $ctx);
259 $self->{empty
}->{$path} = 0;
261 my ($dir, $file) = ($path =~ m
#^(.*?)/?([^/]+)$#);
262 delete $self->{empty
}->{$dir};
263 $self->{empty
}->{$path} = 1;
265 if ($added_placeholder{$dir}) {
266 # Remove our placeholder file, if we created one.
267 delete_entry
($self, $added_placeholder{$dir});
268 delete $added_placeholder{$dir}
275 sub change_dir_prop
{
276 my ($self, $db, $prop, $value) = @_;
277 return undef if $self->is_path_ignored($db->{path
});
278 $self->{dir_prop
}->{$db->{path
}} ||= {};
279 $self->{dir_prop
}->{$db->{path
}}->{$prop} = $value;
283 sub absent_directory
{
284 my ($self, $path, $pb) = @_;
285 return undef if $self->is_path_ignored($path);
286 $self->{absent_dir
}->{$pb->{path
}} ||= [];
287 push @
{$self->{absent_dir
}->{$pb->{path
}}}, $path;
292 my ($self, $path, $pb) = @_;
293 return undef if $self->is_path_ignored($path);
294 $self->{absent_file
}->{$pb->{path
}} ||= [];
295 push @
{$self->{absent_file
}->{$pb->{path
}}}, $path;
299 sub change_file_prop
{
300 my ($self, $fb, $prop, $value) = @_;
301 return undef if $self->is_path_ignored($fb->{path
});
302 if ($prop eq 'svn:executable') {
303 if ($fb->{mode_b
} != 120000) {
304 $fb->{mode_b
} = defined $value ?
100755 : 100644;
306 } elsif ($prop eq 'svn:special') {
307 $fb->{mode_b
} = defined $value ?
120000 : 100644;
309 $self->{file_prop
}->{$fb->{path
}} ||= {};
310 $self->{file_prop
}->{$fb->{path
}}->{$prop} = $value;
315 sub apply_textdelta
{
316 my ($self, $fb, $exp) = @_;
317 return undef if $self->is_path_ignored($fb->{path
});
319 ++$suffix while $::_repository
->temp_is_locked("svn_delta_${$}_$suffix");
320 my $fh = $::_repository
->temp_acquire("svn_delta_${$}_$suffix");
321 # $fh gets auto-closed() by SVN::TxDelta::apply(),
322 # (but $base does not,) so dup() it for reading in close_file
323 open my $dup, '<&', $fh or croak
$!;
324 my $base = $::_repository
->temp_acquire("git_blob_${$}_$suffix");
327 my ($base_is_link, $size);
329 if ($fb->{mode_a
} eq '120000' &&
330 ! $self->{empty_symlinks
}->{$fb->{path
}}) {
331 print $base 'link ' or die "print $!\n";
335 $size = $::_repository
->cat_blob($fb->{blob
}, $base);
336 die "Failed to read object $fb->{blob}" if ($size < 0);
339 seek $base, 0, 0 or croak
$!;
340 my $got = ::md5sum
($base);
342 my $err = "Checksum mismatch: ".
343 "$fb->{path} $fb->{blob}\n" .
348 "Retrying... (possibly ",
349 "a bad symlink from SVN)\n";
350 $::_repository
->temp_reset($base);
358 seek $base, 0, 0 or croak
$!;
361 [ SVN
::TxDelta
::apply
($base, $dup, undef, $fb->{path
}, $fb->{pool
}) ];
365 my ($self, $fb, $exp) = @_;
366 return undef if $self->is_path_ignored($fb->{path
});
369 my $path = $self->git_path($fb->{path
});
370 if (my $fh = $fb->{fh
}) {
372 seek($fh, 0, 0) or croak
$!;
373 my $got = ::md5sum
($fh);
375 die "Checksum mismatch: $path\n",
376 "expected: $exp\n got: $got\n";
379 if ($fb->{mode_b
} == 120000) {
380 sysseek($fh, 0, 0) or croak
$!;
381 my $rd = sysread($fh, my $buf, 5);
384 croak
"sysread: $!\n";
386 warn "$path has mode 120000",
387 " but it points to nothing\n",
388 "converting to an empty file with mode",
390 $fb->{mode_b
} = '100644';
391 } elsif ($buf ne 'link ') {
392 warn "$path has mode 120000",
393 " but is not a link\n";
395 my $tmp_fh = $::_repository
->temp_acquire(
398 while ($res = sysread($fh, my $str, 1024)) {
399 my $out = syswrite($tmp_fh, $str, $res);
400 defined($out) && $out == $res
402 Git
::temp_path
($tmp_fh),
405 defined $res or croak
$!;
407 ($fh, $tmp_fh) = ($tmp_fh, $fh);
408 Git
::temp_release
($tmp_fh, 1);
412 $hash = $::_repository
->hash_and_insert_object(
413 Git
::temp_path
($fh));
414 $hash =~ /^[a-f\d]{40}$/ or die "not a sha1: $hash\n";
416 Git
::temp_release
($fb->{base
}, 1);
417 Git
::temp_release
($fh, 1);
419 $hash = $fb->{blob
} or die "no blob information\n";
422 $self->{gii
}->update($fb->{mode_b
}, $hash, $path) or croak
$!;
423 print "\t$fb->{action}\t$path\n" if $fb->{action
} && ! $::_q
;
429 $self->{nr
} = $self->{gii
}->{nr
};
431 $self->SUPER::abort_edit
(@_);
437 if ($_preserve_empty_dirs) {
440 # Any entry flagged as empty that also has an associated
441 # dir_prop represents a newly created empty directory.
442 foreach my $i (keys %{$self->{empty
}}) {
443 push @empty_dirs, $i if exists $self->{dir_prop
}->{$i};
446 # Search for directories that have become empty due subsequent
448 push @empty_dirs, $self->find_empty_directories();
450 # Finally, add a placeholder file to each empty directory.
451 $self->add_placeholder_file($_) foreach (@empty_dirs);
453 $self->stash_placeholder_list();
456 $self->{git_commit_ok
} = 1;
457 $self->{nr
} = $self->{gii
}->{nr
};
459 $self->SUPER::close_edit
(@_);
462 sub find_empty_directories
{
465 my %dirs = map { dirname
($_) => 1 } @deleted_gpath;
467 foreach my $dir (sort keys %dirs) {
470 # If there have been any additions to this directory, there is
471 # no reason to check if it is empty.
473 foreach my $t (qw
/dir_prop file_prop/) {
474 foreach my $path (keys %{ $self->{$t} }) {
475 if (exists $self->{$t}->{dirname
($path)}) {
484 # Use `git ls-tree` to get the filenames of this directory
485 # that existed prior to this particular commit.
486 my $ls = command
('ls-tree', '-z', '--name-only',
487 $self->{c
}, "$dir/");
488 my %files = map { $_ => 1 } split(/\0/, $ls);
490 # Remove the filenames that were deleted during this commit.
491 delete $files{$_} foreach (@deleted_gpath);
493 # Report the directory if there are no filenames left.
494 push @empty_dirs, $dir unless (scalar %files);
499 sub add_placeholder_file
{
500 my ($self, $dir) = @_;
501 my $path = "$dir/$_placeholder_filename";
502 my $gpath = $self->git_path($path);
504 my $fh = $::_repository
->temp_acquire($gpath);
505 my $hash = $::_repository
->hash_and_insert_object(Git
::temp_path
($fh));
506 Git
::temp_release
($fh, 1);
507 $self->{gii
}->update('100644', $hash, $gpath) or croak
$!;
509 # The directory should no longer be considered empty.
510 delete $self->{empty
}->{$dir} if exists $self->{empty
}->{$dir};
512 # Keep track of any placeholder files we create.
513 $added_placeholder{$dir} = $path;
516 sub stash_placeholder_list
{
518 my $k = "svn-remote.$repo_id.added-placeholder";
519 my $v = eval { command_oneline
('config', '--get-all', $k) };
520 command_noisy
('config', '--unset-all', $k) if $v;
521 foreach (values %added_placeholder) {
522 command_noisy
('config', '--add', $k, $_);
531 Git::SVN::Fetcher - tree delta consumer for "git svn fetch"
538 use Git::SVN::Fetcher;
541 my $gs = Git::SVN->find_by_url($url);
542 my $ra = SVN::Ra->new(url => $url);
543 my $editor = Git::SVN::Fetcher->new($gs);
544 my $reporter = $ra->do_update($SVN::Core::INVALID_REVNUM, '',
546 $reporter->set_path('', $old_rev, 0);
547 $reporter->finish_report;
548 my $tree = $gs->tmp_index_do(sub { command_oneline('write-tree') });
550 foreach my $path (keys %{$editor->{dir_prop}) {
551 my $props = $editor->{dir_prop}{$path};
552 foreach my $prop (keys %$props) {
553 print "property $prop at $path changed to $props->{$prop}\n";
556 foreach my $path (keys %{$editor->{empty}) {
557 my $action = $editor->{empty}{$path} ? 'added' : 'removed';
558 print "empty directory $path $action\n";
560 foreach my $path (keys %{$editor->{file_prop}) { ... }
561 foreach my $parent (keys %{$editor->{absent_dir}}) {
562 my @children = @{$editor->{abstent_dir}{$parent}};
563 print "cannot fetch directory $parent/$_: not authorized?\n"
566 foreach my $parent (keys %{$editor->{absent_file}) { ... }
570 This is a subclass of C<SVN::Delta::Editor>, which means it implements
571 callbacks to act as a consumer of Subversion tree deltas. This
572 particular implementation of those callbacks is meant to store
573 information about the resulting content which B<git svn fetch> could
574 use to populate new commits and new entries for F<unhandled.log>.
579 =item * Additions, removals, and modifications of files are propagated
580 to git-svn's index file F<$GIT_DIR/svn/$refname/index> using
583 =item * Changes in Subversion path properties are recorded in the
584 C<dir_prop> and C<file_prop> fields (which are hashes).
586 =item * Addition and removal of empty directories are indicated by
587 entries with value 1 and 0 respectively in the C<empty> hash.
589 =item * Paths that are present but cannot be conveyed (presumably due
590 to permissions) are recorded in the C<absent_file> and
591 C<absent_dirs> hashes. For each key, the corresponding value is
592 a list of paths under that directory that were present but
593 could not be conveyed.
597 The interface is unstable. Do not use this module unless you are
602 L<SVN::Delta> from the Subversion perl bindings,
603 the core L<Carp>, L<File::Basename>, and L<IO::File> modules,
604 and git's L<Git> helper module.
606 C<Git::SVN::Fetcher> has not been tested using callers other than
614 =head1 INCOMPATIBILITIES