cvs2svn_lib/dvcs_common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2007-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Miscellaneous utility code common to DVCS backends (like
  18 Git, Mercurial, or Bazaar).
  19 """
  20
  21 import os, sys
  22
  23 from cvs2svn_lib import config
  24 from cvs2svn_lib.common import FatalError
  25 from cvs2svn_lib.common import InternalError
  26 from cvs2svn_lib.run_options import RunOptions
  27 from cvs2svn_lib.log import logger
  28 from cvs2svn_lib.common import error_prefix
  29 from cvs2svn_lib.context import Ctx
  30 from cvs2svn_lib.artifact_manager import artifact_manager
  31 from cvs2svn_lib.project import Project
  32 from cvs2svn_lib.cvs_item import CVSRevisionAdd
  33 from cvs2svn_lib.cvs_item import CVSRevisionChange
  34 from cvs2svn_lib.cvs_item import CVSRevisionDelete
  35 from cvs2svn_lib.cvs_item import CVSRevisionNoop
  36 from cvs2svn_lib.svn_revision_range import RevisionScores
  37 from cvs2svn_lib.openings_closings import SymbolingsReader
  38 from cvs2svn_lib.repository_mirror import RepositoryMirror
  39 from cvs2svn_lib.output_option import OutputOption
  40 from cvs2svn_lib.property_setters import FilePropertySetter
  41
  42
  43 class KeywordHandlingPropertySetter(FilePropertySetter):
  44   """Set property _keyword_handling to a specified value.
  45
  46   This keyword is used to tell the RevisionReader whether it has to
  47   collapse/expand RCS keywords when generating the fulltext or leave
  48   them alone."""
  49
  50   propname = '_keyword_handling'
  51
  52   def __init__(self, value):
  53     if value not in ['collapsed', 'expanded', 'untouched', None]:
  54       raise FatalError(
  55           'Value for %s must be "collapsed", "expanded", or "untouched"'
  56           % (self.propname,)
  57           )
  58     self.value = value
  59
  60   def set_properties(self, cvs_file):
  61     self.maybe_set_property(cvs_file, self.propname, self.value)
  62
  63
  64 class DVCSRunOptions(RunOptions):
  65   """Dumping ground for whatever is common to GitRunOptions,
  66   HgRunOptions, and BzrRunOptions."""
  67
  68   def __init__(self, progname, cmd_args, pass_manager):
  69     Ctx().cross_project_commits = False
  70     Ctx().cross_branch_commits = False
  71     if Ctx().username is None:
  72       Ctx().username = self.DEFAULT_USERNAME
  73     RunOptions.__init__(self, progname, cmd_args, pass_manager)
  74
  75   def set_project(
  76         self,
  77         project_cvs_repos_path,
  78         symbol_transforms=None,
  79         symbol_strategy_rules=[],
  80         exclude_paths=[],
  81         ):
  82     """Set the project to be converted.
  83
  84     If a project had already been set, overwrite it.
  85
  86     Most arguments are passed straight through to the Project
  87     constructor.  SYMBOL_STRATEGY_RULES is an iterable of
  88     SymbolStrategyRules that will be applied to symbols in this
  89     project."""
  90
  91     symbol_strategy_rules = list(symbol_strategy_rules)
  92
  93     project = Project(
  94         0,
  95         project_cvs_repos_path,
  96         symbol_transforms=symbol_transforms,
  97         exclude_paths=exclude_paths,
  98         )
  99
 100     self.projects = [project]
 101     self.project_symbol_strategy_rules = [symbol_strategy_rules]
 102
 103   def process_property_setter_options(self):
 104     RunOptions.process_property_setter_options(self)
 105
 106     # Property setters for internal use:
 107     Ctx().file_property_setters.append(
 108         KeywordHandlingPropertySetter('collapsed')
 109         )
 110
 111   def process_options(self):
 112     # Consistency check for options and arguments.
 113     if len(self.args) == 0:
 114       # Default to using '.' as the source repository path
 115       self.args.append(os.getcwd())
 116
 117     if len(self.args) > 1:
 118       logger.error(error_prefix + ": must pass only one CVS repository.\n")
 119       self.usage()
 120       sys.exit(1)
 121
 122     cvsroot = self.args[0]
 123
 124     self.process_extraction_options()
 125     self.process_output_options()
 126     self.process_symbol_strategy_options()
 127     self.process_property_setter_options()
 128
 129     # Create the project:
 130     self.set_project(
 131         cvsroot,
 132         symbol_transforms=self.options.symbol_transforms,
 133         symbol_strategy_rules=self.options.symbol_strategy_rules,
 134         )
 135
 136
 137 class DVCSOutputOption(OutputOption):
 138   def __init__(self):
 139     self._mirror = RepositoryMirror()
 140     self._symbolings_reader = None
 141
 142   def normalize_author_transforms(self, author_transforms):
 143     """Convert AUTHOR_TRANSFORMS into author strings.
 144
 145     AUTHOR_TRANSFORMS is a dict { CVSAUTHOR : DVCSAUTHOR } where
 146     CVSAUTHOR is the CVS author and DVCSAUTHOR is either:
 147
 148     * a tuple (NAME, EMAIL) where NAME and EMAIL are strings.  Such
 149       entries are converted into a UTF-8 string of the form 'name
 150       <email>'.
 151
 152     * a string already in the form 'name <email>'.
 153
 154     Return a similar dict { CVSAUTHOR : DVCSAUTHOR } where all keys
 155     and values are UTF-8-encoded strings.
 156
 157     Any of the input strings may be Unicode strings (in which case
 158     they are encoded to UTF-8) or 8-bit strings (in which case they
 159     are used as-is).  Also turns None into the empty dict."""
 160
 161     result = {}
 162     if author_transforms is not None:
 163       for (cvsauthor, dvcsauthor) in author_transforms.iteritems():
 164         cvsauthor = to_utf8(cvsauthor)
 165         if isinstance(dvcsauthor, basestring):
 166           dvcsauthor = to_utf8(dvcsauthor)
 167         else:
 168           (name, email,) = dvcsauthor
 169           name = to_utf8(name)
 170           email = to_utf8(email)
 171           dvcsauthor = "%s <%s>" % (name, email,)
 172         result[cvsauthor] = dvcsauthor
 173     return result
 174
 175   def register_artifacts(self, which_pass):
 176     # These artifacts are needed for SymbolingsReader:
 177     artifact_manager.register_temp_file_needed(
 178         config.SYMBOL_OPENINGS_CLOSINGS_SORTED, which_pass
 179         )
 180     artifact_manager.register_temp_file_needed(
 181         config.SYMBOL_OFFSETS_DB, which_pass
 182         )
 183     self._mirror.register_artifacts(which_pass)
 184
 185   def check(self):
 186     if Ctx().cross_project_commits:
 187       raise FatalError(
 188           '%s output is not supported with cross-project commits' % self.name
 189           )
 190     if Ctx().cross_branch_commits:
 191       raise FatalError(
 192           '%s output is not supported with cross-branch commits' % self.name
 193           )
 194     if Ctx().username is None:
 195       raise FatalError(
 196           '%s output requires a default commit username' % self.name
 197           )
 198
 199   def setup(self, svn_rev_count):
 200     self._symbolings_reader = SymbolingsReader()
 201     self._mirror.open()
 202
 203   def cleanup(self):
 204     self._mirror.close()
 205     self._symbolings_reader.close()
 206     del self._symbolings_reader
 207
 208   def _get_source_groups(self, svn_commit):
 209     """Return groups of sources for SVN_COMMIT.
 210
 211     SVN_COMMIT is an instance of SVNSymbolCommit.  Return a list of tuples
 212     (svn_revnum, source_lod, cvs_symbols) where svn_revnum is the revision
 213     that should serve as a source, source_lod is the CVS line of
 214     development, and cvs_symbols is a list of CVSSymbolItems that can be
 215     copied from that source.  The list is in arbitrary order."""
 216
 217     # Get a map {CVSSymbol : SVNRevisionRange}:
 218     range_map = self._symbolings_reader.get_range_map(svn_commit)
 219
 220     # range_map, split up into one map per LOD; i.e., {LOD :
 221     # {CVSSymbol : SVNRevisionRange}}:
 222     lod_range_maps = {}
 223
 224     for (cvs_symbol, range) in range_map.iteritems():
 225       lod_range_map = lod_range_maps.get(range.source_lod)
 226       if lod_range_map is None:
 227         lod_range_map = {}
 228         lod_range_maps[range.source_lod] = lod_range_map
 229       lod_range_map[cvs_symbol] = range
 230
 231     # Sort the sources so that the branch that serves most often as
 232     # parent is processed first:
 233     lod_ranges = lod_range_maps.items()
 234     lod_ranges.sort(
 235         lambda (lod1,lod_range_map1),(lod2,lod_range_map2):
 236         -cmp(len(lod_range_map1), len(lod_range_map2)) or cmp(lod1, lod2)
 237         )
 238
 239     source_groups = []
 240     for (lod, lod_range_map) in lod_ranges:
 241       while lod_range_map:
 242         revision_scores = RevisionScores(lod_range_map.values())
 243         (source_lod, revnum, score) = revision_scores.get_best_revnum()
 244         assert source_lod == lod
 245         cvs_symbols = []
 246         for (cvs_symbol, range) in lod_range_map.items():
 247           if revnum in range:
 248             cvs_symbols.append(cvs_symbol)
 249             del lod_range_map[cvs_symbol]
 250         source_groups.append((revnum, lod, cvs_symbols))
 251
 252     return source_groups
 253
 254   def _is_simple_copy(self, svn_commit, source_groups):
 255     """Return True iff SVN_COMMIT can be created as a simple copy.
 256
 257     SVN_COMMIT is an SVNTagCommit.  Return True iff it can be created
 258     as a simple copy from an existing revision (i.e., if the fixup
 259     branch can be avoided for this tag creation)."""
 260
 261     # The first requirement is that there be exactly one source:
 262     if len(source_groups) != 1:
 263       return False
 264
 265     (svn_revnum, source_lod, cvs_symbols) = source_groups[0]
 266
 267     # The second requirement is that the destination LOD not already
 268     # exist:
 269     try:
 270       self._mirror.get_current_lod_directory(svn_commit.symbol)
 271     except KeyError:
 272       # The LOD doesn't already exist.  This is good.
 273       pass
 274     else:
 275       # The LOD already exists.  It cannot be created by a copy.
 276       return False
 277
 278     # The third requirement is that the source LOD contains exactly
 279     # the same files as we need to add to the symbol:
 280     try:
 281       source_node = self._mirror.get_old_lod_directory(source_lod, svn_revnum)
 282     except KeyError:
 283       raise InternalError('Source %r does not exist' % (source_lod,))
 284     return (
 285         set([cvs_symbol.cvs_file for cvs_symbol in cvs_symbols])
 286         == set(self._get_all_files(source_node))
 287         )
 288
 289   def _get_all_files(self, node):
 290     """Generate all of the CVSFiles under NODE."""
 291
 292     for cvs_path in node:
 293       subnode = node[cvs_path]
 294       if subnode is None:
 295         yield cvs_path
 296       else:
 297         for sub_cvs_path in self._get_all_files(subnode):
 298           yield sub_cvs_path
 299
 300
 301 class ExpectedDirectoryError(Exception):
 302   """A file was found where a directory was expected."""
 303
 304   pass
 305
 306
 307 class ExpectedFileError(Exception):
 308   """A directory was found where a file was expected."""
 309
 310   pass
 311
 312
 313 class MirrorUpdater(object):
 314   def register_artifacts(self, which_pass):
 315     pass
 316
 317   def start(self, mirror):
 318     self._mirror = mirror
 319
 320   def _mkdir_p(self, cvs_directory, lod):
 321     """Make sure that CVS_DIRECTORY exists in LOD.
 322
 323     If not, create it.  Return the node for CVS_DIRECTORY."""
 324
 325     try:
 326       node = self._mirror.get_current_lod_directory(lod)
 327     except KeyError:
 328       node = self._mirror.add_lod(lod)
 329
 330     for sub_path in cvs_directory.get_ancestry()[1:]:
 331       try:
 332         node = node[sub_path]
 333       except KeyError:
 334         node = node.mkdir(sub_path)
 335       if node is None:
 336         raise ExpectedDirectoryError(
 337             'File found at \'%s\' where directory was expected.' % (sub_path,)
 338             )
 339
 340     return node
 341
 342   def add_file(self, cvs_rev, post_commit):
 343     cvs_file = cvs_rev.cvs_file
 344     if post_commit:
 345       lod = cvs_file.project.get_trunk()
 346     else:
 347       lod = cvs_rev.lod
 348     parent_node = self._mkdir_p(cvs_file.parent_directory, lod)
 349     parent_node.add_file(cvs_file)
 350
 351   def modify_file(self, cvs_rev, post_commit):
 352     cvs_file = cvs_rev.cvs_file
 353     if post_commit:
 354       lod = cvs_file.project.get_trunk()
 355     else:
 356       lod = cvs_rev.lod
 357     if self._mirror.get_current_path(cvs_file, lod) is not None:
 358       raise ExpectedFileError(
 359           'Directory found at \'%s\' where file was expected.' % (cvs_file,)
 360           )
 361
 362   def delete_file(self, cvs_rev, post_commit):
 363     cvs_file = cvs_rev.cvs_file
 364     if post_commit:
 365       lod = cvs_file.project.get_trunk()
 366     else:
 367       lod = cvs_rev.lod
 368     parent_node = self._mirror.get_current_path(
 369         cvs_file.parent_directory, lod
 370         )
 371     if parent_node[cvs_file] is not None:
 372       raise ExpectedFileError(
 373           'Directory found at \'%s\' where file was expected.' % (cvs_file,)
 374           )
 375     del parent_node[cvs_file]
 376
 377   def process_revision(self, cvs_rev, post_commit):
 378     if isinstance(cvs_rev, CVSRevisionAdd):
 379       self.add_file(cvs_rev, post_commit)
 380     elif isinstance(cvs_rev, CVSRevisionChange):
 381       self.modify_file(cvs_rev, post_commit)
 382     elif isinstance(cvs_rev, CVSRevisionDelete):
 383       self.delete_file(cvs_rev, post_commit)
 384     elif isinstance(cvs_rev, CVSRevisionNoop):
 385       pass
 386     else:
 387       raise InternalError('Unexpected CVSRevision type: %s' % (cvs_rev,))
 388
 389   def branch_file(self, cvs_symbol):
 390     cvs_file = cvs_symbol.cvs_file
 391     parent_node = self._mkdir_p(cvs_file.parent_directory, cvs_symbol.symbol)
 392     parent_node.add_file(cvs_file)
 393
 394   def finish(self):
 395     del self._mirror
 396
 397
 398 def to_utf8(s):
 399   if isinstance(s, unicode):
 400     return s.encode('utf8')
 401   else:
 402     return s
 403
 404