5 darcs2git -- Darcs to git converter.
7 Copyright (c) 2007 Han-Wen Nienhuys <hanwen@xs4all.nl>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2, or (at your option)
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
32 # - use binary search to find from-patch in case of conflict.
36 import distutils
.version
41 import xml
.dom
.minidom
43 import gdbm
as dbmodule
46 from email
.utils
import parsedate_tz
48 ################################################################
54 mail_to_name_dict
= {}
59 ################################################################
62 class PullConflict (Exception):
64 class CommandFailed (Exception):
68 sys
.stderr
.write (s
+ '\n')
70 def get_cli_options ():
71 class MyOP(optparse
.OptionParser
):
73 optparse
.OptionParser
.print_help (self
)
77 This tool is a conversion utility for Darcs repositories, importing
78 them in chronological order. It requires a Git version that has
79 git-fast-import. It does not support incremental updating.
83 * repositories with skewed timestamps, or different patches with
84 equal timestamps will confuse darcs2git.
85 * does not respect file modes or time zones.
86 * too slow. See source code for instructions to speed it up.
87 * probably doesn\'t work on partial repositories
89 Report new bugs to hanwen@xs4all.nl
93 Copyright (c) 2007 Han-Wen Nienhuys <hanwen@xs4all.nl>.
94 Distributed under terms of the GNU General Public License
95 This program comes with NO WARRANTY.
100 p
.usage
='''darcs2git [OPTIONS] DARCS-REPO'''
101 p
.description
='''Convert darcs repo to git.'''
103 def update_map (option
, opt
, value
, parser
):
104 for l
in open (value
).readlines ():
105 (mail
, name
) = tuple (l
.strip ().split ('='))
106 mail_to_name_dict
[mail
] = name
108 p
.add_option ('-a', '--authors', action
='callback',
112 help='read a text file, containing EMAIL=NAME lines')
114 p
.add_option ('--checkpoint-frequency', action
='store',
115 dest
='checkpoint_frequency',
118 help='how often should the git importer be synced?\n'
119 'Default is 0 (no limit)'
122 p
.add_option ('-d', '--destination', action
='store',
125 dest
='target_git_repo',
126 help='where to put the resulting Git repo.')
128 p
.add_option ('--verbose', action
='store_true',
131 help='show commands as they are invoked')
133 p
.add_option ('--history-window', action
='store',
134 dest
='history_window',
137 help='Look back this many patches as conflict ancestors.\n'
138 'Default is 0 (no limit)')
140 p
.add_option ('--debug', action
='store_true',
143 help="""add patch numbers to commit messages;
144 don\'t clean conversion repo;
148 options
, args
= p
.parse_args ()
153 if len(urlparse
.urlparse(args
[0])) == 0:
154 raise NotImplementedError,"We support local DARCS repos only."
156 git_version
= distutils
.version
.LooseVersion(os
.popen("git --version","r").read().strip().split(" ")[-1])
157 ideal_version
= distutils
.version
.LooseVersion("1.5.0")
158 if git_version
<ideal_version
:
159 raise RuntimeError,"You need git >= 1.5.0 for this."
161 options
.basename
= os
.path
.basename (os
.path
.normpath (args
[0])).replace ('.darcs', '')
162 if not options
.target_git_repo
:
163 options
.target_git_repo
= options
.basename
+ '.git'
167 name
= options
.target_git_repo
.replace ('.git', '.log')
168 if name
== options
.target_git_repo
:
171 progress ("Shell log to %s" % name
)
172 log_file
= open (name
, 'w')
174 return (options
, args
)
176 def read_pipe (cmd
, ignore_errors
=False):
178 progress ('pipe %s' % cmd
)
179 pipe
= os
.popen (cmd
)
182 if pipe
.close () and not ignore_errors
:
183 raise CommandFailed ("Pipe failed: %s" % cmd
)
187 def system (c
, ignore_error
=0, timed
=0):
194 log_file
.write ('%s\n' % c
)
197 if os
.system (c
) and not ignore_error
:
198 raise CommandFailed ("Command failed: %s" % c
)
200 def darcs_date_to_git (x
):
201 t
= time
.strptime (x
, '%Y%m%d%H%M%S')
202 return '%d' % int (time
.mktime (t
))
204 def darcs_timezone (x
) :
205 tz
= parsedate_tz(x
)[9] / 60
206 return "%+03d%02d" % (tz
/ 60, tz
% 60)
208 ################################################################
211 class DarcsConversionRepo
:
212 """Representation of a Darcs repo.
214 The repo is thought to be ordered, and supports methods for
215 going back (obliterate) and forward (pull).
219 def __init__ (self
, dir, patches
):
220 self
.dir = os
.path
.abspath (dir)
221 self
.patches
= patches
222 self
._current
_number
= -1
224 self
._inventory
_dict
= None
226 self
._short
_id
_dict
= dict ((p
.short_id (), p
) for p
in patches
)
229 if not options
.debug
:
230 system ('rm -fr %s' % self
.dir)
232 def is_contiguous (self
):
233 return (len (self
.inventory_dict ()) == self
._current
_number
+1
234 and self
.contains_contiguous (self
._current
_number
))
236 def contains_contiguous (self
, num
):
237 if not self
._is
_valid
:
240 darcs_dir
= self
.dir + '/_darcs'
241 if not os
.path
.exists (darcs_dir
):
244 for p
in self
.patches
[:num
+ 1]:
245 if not self
.has_patch (p
):
250 def has_patch (self
, p
):
251 assert self
._is
_valid
253 return self
.inventory_dict ().has_key (p
.short_id ())
255 def pristine_tree (self
):
256 return self
.dir + '/_darcs/pristine'
258 def go_back_to (self
, dest
):
260 # at 4, len = 5, go to 2: count == 2
261 count
= len (self
.inventory_dict()) - dest
- 1
263 assert self
._is
_valid
269 progress ('Rewinding %d patches' % count
)
270 system ('cd %(dir)s && echo ay|darcs obliterate --ignore-times --last %(count)d' % locals ())
271 d
= self
.inventory_dict ()
272 for p
in self
.patches
[dest
+1:self
._current
_number
+1]:
278 self
._current
_number
= dest
281 system ('rm -rf %s' % self
.dir)
285 system ('rsync -a %(dir)s/_darcs/pristine/ %(dir)s/' % locals ())
287 def pull (self
, patch
):
288 id = patch
.attributes
['hash']
289 source_repo
= patch
.dir
292 progress ('Pull patch %d' % patch
.number
)
293 system ('cd %(dir)s && darcs pull --ignore-times --quiet --all --match "hash %(id)s" %(source_repo)s ' % locals ())
295 self
._current
_number
= patch
.number
297 ## must reread: the pull may have pulled in others.
298 self
._inventory
_dict
= None
300 def go_forward_to (self
, num
):
301 d
= self
.inventory_dict ()
306 for p
in self
.patches
[0:num
+1]:
307 if not d
.has_key (p
.short_id ()):
311 pull_str
= ' || '.join (['hash %s' % p
.id () for p
in pull_me
])
313 src
= self
.patches
[0].dir
315 progress ('Pulling %d patches to go to %d' % (len (pull_me
), num
))
316 system ('darcs pull --all --repo %(dir)s --match "%(pull_str)s" %(src)s' % locals ())
318 def create_fresh (self
):
320 system ('rm -rf %(dir)s && mkdir %(dir)s && darcs init --repo %(dir)s'
322 self
._is
_valid
= True
323 self
._current
_number
= -1
324 self
._inventory
_dict
= {}
326 def inventory (self
):
327 darcs_dir
= self
.dir + '/_darcs'
329 for f
in [darcs_dir
+ '/inventory'] + glob
.glob (darcs_dir
+ '/inventories/*'):
330 i
+= open (f
).read ()
333 def inventory_dict (self
):
334 if type (self
._inventory
_dict
) != type ({}):
335 self
._inventory
_dict
= {}
338 self
._inventory
_dict
[m
.group (1)] = self
._short
_id
_dict
[m
.group(1)]
340 re
.sub (r
'\n([^*\n]+\*[*-][0-9]+)', note_patch
, self
.inventory ())
341 return self
._inventory
_dict
343 def start_at (self
, num
):
345 """Move the repo to NUM.
347 This uses the fishy technique of writing the inventory and
348 constructing the pristine tree with 'darcs repair'
350 progress ('Starting afresh at %d' % num
)
354 iv
= open (dir + '/_darcs/inventory', 'w')
356 log_file
.write ("# messing with _darcs/inventory")
358 for p
in self
.patches
[:num
+1]:
359 os
.link (p
.filename (), dir + '/_darcs/patches/' + os
.path
.basename (p
.filename ()))
360 iv
.write (p
.header ())
361 self
._inventory
_dict
[p
.short_id ()] = p
364 system ('darcs repair --repo %(dir)s --quiet' % locals ())
366 self
._current
_number
= num
367 self
._is
_valid
= True
369 def go_to (self
, dest
):
370 contiguous
= self
.is_contiguous ()
372 if not self
._is
_valid
:
374 elif dest
== self
._current
_number
and contiguous
:
376 elif (self
.contains_contiguous (dest
)):
377 self
.go_back_to (dest
)
378 elif dest
- len (self
.inventory_dict ()) < dest
/ 100:
379 self
.go_forward_to (dest
)
384 def go_from_to (self
, from_patch
, to_patch
):
386 """Move the repo to FROM_PATCH, then go to TO_PATCH. Raise
387 PullConflict if conflict is detected
390 progress ('Trying %s -> %s' % (from_patch
, to_patch
))
392 source
= to_patch
.dir
395 self
.go_to (from_patch
.number
)
401 success
= 'No conflicts to resolve' in read_pipe ('cd %(dir)s && echo y|darcs resolve' % locals ())
402 except CommandFailed
:
403 self
._is
_valid
= False
404 raise PullConflict ()
407 raise PullConflict ()
411 return 'patch %d' % self
.number
413 def __init__ (self
, xml
, dir):
418 self
._contents
= None
419 for (nm
, value
) in xml
.attributes
.items():
420 self
.attributes
[nm
] = value
422 # fixme: ugh attributes vs. methods.
423 self
.extract_author ()
424 self
.extract_message ()
428 return self
.attributes
['hash']
432 if self
.attributes
['inverted'] == 'True':
435 return '%s*%s%s' % (self
.attributes
['author'], inv
, self
.attributes
['hash'].split ('-')[0])
438 return self
.dir + '/_darcs/patches/' + self
.attributes
['hash']
441 if type (self
._contents
) != type (''):
442 f
= gzip
.open (self
.filename ())
443 self
._contents
= f
.read ()
445 return self
._contents
448 lines
= self
.contents ().split ('\n')
451 committer
= lines
[1] + '\n'
452 committer
= re
.sub ('] {\n$', ']\n', committer
)
453 committer
= re
.sub ('] *\n$', ']\n', committer
)
455 if not committer
.endswith (']\n'):
462 header
= name
+ '\n' + committer
466 assert header
[-1] == '\n'
469 def extract_author (self
):
470 mail
= self
.attributes
['author']
472 m
= re
.search ("^(.*) <(.*)>$", mail
)
479 name
= mail_to_name_dict
[mail
]
481 name
= mail
.split ('@')[0]
483 self
.author_name
= name
484 self
.author_mail
= mail
486 def extract_time (self
):
487 self
.date
= darcs_date_to_git (self
.attributes
['date']) + ' ' + darcs_timezone (self
.attributes
['local_date'])
490 patch_name
= '(no comment)'
492 name_elt
= self
.xml
.getElementsByTagName ('name')[0]
493 patch_name
= name_elt
.childNodes
[0].data
498 def extract_message (self
):
499 patch_name
= self
.name ()
500 comment_elts
= self
.xml
.getElementsByTagName ('comment')
503 comment
= comment_elts
[0].childNodes
[0].data
505 if self
.attributes
['inverted'] == 'True':
506 patch_name
= 'UNDO: ' + patch_name
508 self
.message
= '%s\n\n%s' % (patch_name
, comment
)
511 patch_name
= self
.name ()
512 if patch_name
.startswith ("TAG "):
514 tag
= re
.sub (r
'\s', '_', tag
).strip ()
515 tag
= re
.sub (r
':', '_', tag
).strip ()
519 def get_darcs_patches (darcs_repo
):
520 progress ('reading patches.')
522 xml_string
= read_pipe ('darcs changes --xml --reverse --repo ' + darcs_repo
)
524 dom
= xml
.dom
.minidom
.parseString(xml_string
)
525 xmls
= dom
.documentElement
.getElementsByTagName('patch')
527 patches
= [DarcsPatch (x
, darcs_repo
) for x
in xmls
]
536 ################################################################
540 def __init__ (self
, parent
, darcs_patch
):
542 self
.darcs_patch
= darcs_patch
544 self
.depth
= parent
.depth
+ 1
549 return self
.darcs_patch
.number
551 def parent_patch (self
):
553 return self
.parent
.darcs_patch
557 def common_ancestor (a
, b
):
559 if a
.depth
< b
.depth
:
561 elif a
.depth
> b
.depth
:
575 def export_checkpoint (gfi
):
576 gfi
.write ('checkpoint\n\n')
578 def export_tree (tree
, gfi
):
579 tree
= os
.path
.normpath (tree
)
580 gfi
.write ('deleteall\n')
581 for (root
, dirs
, files
) in os
.walk (tree
):
583 rf
= os
.path
.normpath (os
.path
.join (root
, f
))
584 s
= open (rf
).read ()
585 rf
= rf
.replace (tree
+ '/', '')
587 gfi
.write ('M 644 inline %s\n' % rf
)
588 gfi
.write ('data %d\n%s\n' % (len (s
), s
))
592 def export_commit (repo
, patch
, last_patch
, gfi
):
593 gfi
.write ('commit refs/heads/darcstmp%d\n' % patch
.number
)
594 gfi
.write ('mark :%d\n' % (patch
.number
+ 1))
595 gfi
.write ('committer %s <%s> %s\n' % (patch
.author_name
,
601 msg
+= '\n\n#%d\n' % patch
.number
603 gfi
.write ('data %d\n%s\n' % (len (msg
), msg
))
607 for (n
, p
) in pending_patches
.items ():
608 if repo
.has_patch (p
):
610 del pending_patches
[n
]
614 and git_commits
.has_key (last_patch
.number
)):
615 mergers
= [last_patch
.number
]
618 gfi
.write ('from :%d\n' % (mergers
[0] + 1))
619 for m
in mergers
[1:]:
620 gfi
.write ('merge :%d\n' % (m
+ 1))
622 pending_patches
[patch
.number
] = patch
623 export_tree (repo
.pristine_tree (), gfi
)
628 n
= last_patch
.number
629 git_commits
[patch
.number
] = GitCommit (git_commits
.get (n
, None),
632 def export_pending (gfi
):
633 if len (pending_patches
.items ()) == 1:
634 gfi
.write ('reset refs/heads/master\n')
635 gfi
.write ('from :%d\n\n' % (pending_patches
.values()[0].number
+1))
637 progress ("Creating branch master")
640 for (n
, p
) in pending_patches
.items ():
641 gfi
.write ('reset refs/heads/master%d\n' % n
)
642 gfi
.write ('from :%d\n\n' % (n
+1))
644 progress ("Creating branch master%d" % n
)
646 patches
= pending_patches
.values()
648 gfi
.write ('commit refs/heads/master\n')
649 gfi
.write ('committer %s <%s> %s\n' % (patch
.author_name
,
653 gfi
.write ('data %d\n%s\n' % (len(msg
), msg
))
654 gfi
.write ('from :%d\n' % (patch
.number
+ 1))
655 for p
in patches
[1:]:
656 gfi
.write ('merge :%d\n' % (p
.number
+ 1))
659 def export_tag (patch
, gfi
):
660 gfi
.write ('tag %s\n' % patch
.tag_name ())
661 gfi
.write ('from :%d\n' % (patch
.number
+ 1))
662 gfi
.write ('tagger %s <%s> %s\n' % (patch
.author_name
,
665 gfi
.write ('data %d\n%s\n' % (len (patch
.message
),
668 ################################################################
671 def test_conversion (darcs_repo
, git_repo
):
672 pristine
= '%(darcs_repo)s/_darcs/pristine' % locals ()
673 if not os
.path
.exists (pristine
):
674 progress ("darcs repository does not contain pristine tree?!")
677 gd
= options
.basename
+ '.checkouttmp.git'
678 system ('rm -rf %(gd)s && git clone %(git_repo)s %(gd)s' % locals ())
679 diff_cmd
= 'diff --exclude .git -urN %(gd)s %(pristine)s' % locals ()
680 system ('rm -rf %(gd)s' % locals ())
682 diff
= read_pipe (diff_cmd
, ignore_errors
=True)
684 if len (diff
) > 1024:
685 diff
= diff
[:512] + '\n...\n' + diff
[512:]
687 progress ("Conversion introduced changes: %s" % diff
)
689 progress ("Checkout matches pristine darcs tree.")
692 (options
, args
) = get_cli_options ()
694 darcs_repo
= os
.path
.abspath (args
[0])
695 git_repo
= os
.path
.abspath (options
.target_git_repo
)
697 if os
.path
.exists (git_repo
):
698 system ('rm -rf %(git_repo)s' % locals ())
700 system ('mkdir %(git_repo)s && cd %(git_repo)s && git --bare init' % locals ())
701 system ('git --git-dir %(git_repo)s repo-config core.logAllRefUpdates false' % locals ())
703 os
.environ
['GIT_DIR'] = git_repo
705 gfi
= os
.popen ('git-fast-import --quiet', 'w')
707 patches
= get_darcs_patches (darcs_repo
)
708 conv_repo
= DarcsConversionRepo (options
.basename
+ ".tmpdarcs", patches
)
709 conv_repo
.start_at (-1)
716 combinations
= [(v
, w
) for v
in pending_patches
.values ()
717 for w
in pending_patches
.values ()]
718 candidates
= [common_ancestor (git_commits
[c
[0].number
], git_commits
[c
[1].number
]) for c
in combinations
]
719 candidates
= sorted ([(-a
.darcs_patch
.number
, a
) for a
in candidates
])
720 for (depth
, c
) in candidates
:
723 conv_repo
.go_from_to (q
, p
)
726 parent_number
= q
.number
727 progress ('Found existing common parent as predecessor')
733 ## no branches found where we could attach.
734 ## try previous commits one by one.
736 parent_number
= p
.number
- 2
738 if parent_number
>= 0:
739 parent_patch
= patches
[parent_number
]
742 conv_repo
.go_from_to (parent_patch
, p
)
746 ## simplistic, may not be enough.
747 progress ('conflict, going one back')
750 if parent_number
< 0:
753 if (options
.history_window
754 and parent_number
< p
.number
- options
.history_window
):
759 if parent_number
>= 0 or p
.number
== 0:
760 progress ('Export %d -> %d (total %d)' % (parent_number
,
761 p
.number
, len (patches
)))
762 export_commit (conv_repo
, p
, parent_patch
, gfi
)
766 if options
.checkpoint_frequency
and p
.number
% options
.checkpoint_frequency
== 0:
767 export_checkpoint (gfi
)
769 progress ("Can't import patch %d, need conflict resolution patch?" % p
.number
)
774 for f
in glob
.glob ('%(git_repo)s/refs/heads/darcstmp*' % locals ()):
777 test_conversion (darcs_repo
, git_repo
)
779 if not options
.debug
: