darcs-fast-import

   1 #!/usr/bin/env python
   2
   3 """
   4
   5     darcs-fast-export - darcs backend for fast data exporters
   6
   7     Copyright (c) 2008 Miklos Vajna <vmiklos@frugalware.org>
   8     Copyright (c) 2008 Matthias Andree <matthias.andree@gmx.de>
   9
  10     This program is free software; you can redistribute it and/or modify
  11     it under the terms of the GNU General Public License as published by
  12     the Free Software Foundation; either version 2, or (at your option)
  13     any later version.
  14
  15     This program is distributed in the hope that it will be useful,
  16     but WITHOUT ANY WARRANTY; without even the implied warranty of
  17     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18     GNU General Public License for more details.
  19
  20     You should have received a copy of the GNU General Public License
  21     along with this program; if not, write to the Free Software
  22     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 """
  25
  26 import sys
  27 import os
  28 import re
  29 import time
  30 import shutil
  31 import optparse
  32 import subprocess
  33
  34 class Handler:
  35         def __init__(self):
  36                 self.marks = {}
  37                 self.files = []
  38                 self.prevfiles = None
  39                 self.ch = None
  40                 self.line = None
  41                 self.unread_line = False
  42                 self.eof = False
  43                 self.debug = False
  44                 self.export_marks = []
  45                 self.import_marks = []
  46
  47         def read_next_line(self):
  48                 if self.unread_line:
  49                         self.unread_line = False
  50                         return
  51                 self.line = ""
  52                 if self.eof:
  53                         return
  54                 if self.ch:
  55                         self.line += self.ch
  56                         self.ch = None
  57                 buf = sys.stdin.readline()
  58                 if not len(buf):
  59                         self.eof = True
  60                 else:
  61                         self.line += buf
  62                 if self.debug:
  63                         print "read_next_line: '%s'" % self.line
  64
  65         def read(self, length):
  66                 buf = ""
  67                 if self.ch:
  68                         buf += self.ch
  69                         self.ch = None
  70                 buf += sys.stdin.read(length)
  71                 if self.debug:
  72                         print "read: '%s'" % buf
  73                 return buf
  74
  75         def skip_optional_lf(self):
  76                 self.ch = self.read(1)
  77                 if self.ch == "\n":
  78                         self.ch = None
  79
  80         def bug(self, s):
  81                 raise Exception(s)
  82
  83         def get_date(self, ts, tz):
  84                 # first fix the case when tz is higher than +1200, as
  85                 # darcs won't accept it
  86                 if int(tz[:3]) > 12:
  87                         ts = str(int(ts) + 60*60*24)
  88                         tz = str(int(tz[:3])-24) + tz[3:]
  89                 # int(ts) is seconds since epoch. Since we're trying to
  90                 # capture both the absolute time of the commit and the
  91                 # localtime in the timezone of the committer, we need to turn
  92                 # the (seconds-since-epoch, committer-timezone-offset) pair
  93                 # that we get from the git-fast-export stream format into a
  94                 # localized-time-plus-timezone-marker string that darcs will
  95                 # accept. Therefore, we parse the timezone-offset (which
  96                 # looks like +0500 or +0000 or -0730 or something) and add it
  97                 # to seconds-since-epoch before calling gmtime().
  98                 mo = re.search(r'^([\+\-])(\d\d)(\d\d)$', tz)
  99                 offset = 60*60*int(mo.group(2)) + 60*int(mo.group(3))
 100                 if mo.group(1) == "-":
 101                         offset = -offset
 102                 offset_time = int(ts) + offset
 103                 s = time.strftime("%a %b %d %H:%M:%S %Y", time.gmtime(offset_time))
 104                 items = s.split(' ')
 105                 return " ".join(items[:-1]) + " " + tz + " " + items[-1]
 106
 107         def handle_mark(self):
 108                 if self.line.startswith("mark :"):
 109                         self.mark_num = int(self.line[6:-1])
 110                         self.read_next_line()
 111
 112         def handle_data(self):
 113                 if not self.line.startswith("data "):
 114                         self.bug("Expected 'data n' command, found: '%s'" % self.line[:-1])
 115                 length = int(self.line[5:-1])
 116                 self.buf = self.read(length)
 117                 self.skip_optional_lf()
 118
 119         def handle_blob(self):
 120                 self.read_next_line()
 121                 self.handle_mark()
 122                 self.handle_data()
 123                 self.marks[self.mark_num] = self.buf
 124
 125         def handle_ident(self, s):
 126                 items = s.split(' ')
 127                 self.ident = " ".join(items[:-2])
 128                 self.date = self.get_date(items[-2], items[-1])
 129
 130         def handle_msg(self):
 131                 items = self.buf.split('\n')
 132                 self.short = items[0]
 133                 self.long = "\n".join(items[1:])
 134
 135         def handle_tag(self):
 136                 version = self.line[:-1].split(' ')[1]
 137                 self.read_next_line()
 138                 if self.line.startswith("from "):
 139                         self.read_next_line()
 140                 if self.line.startswith("tagger "):
 141                         self.handle_ident(self.line[7:-1])
 142                         self.read_next_line()
 143                 self.handle_data()
 144                 self.skip_optional_lf()
 145                 sock = subprocess.Popen(["darcs", "tag", "--pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 146                 buf = [self.date, self.ident, version]
 147                 sock.stdin.write("\n".join(buf))
 148                 sock.stdin.close()
 149                 self.log("Tagging %s:\n%s" % (version, sock.stdout.read()))
 150                 sock.stdout.close()
 151
 152         def handle_commit(self):
 153                 if not self.prevfiles and self.options.import_marks:
 154                         # first commit in an incremental continued
 155                         # import
 156                         for (root, dirs, files) in os.walk("."):
 157                                 for i in files:
 158                                         path = os.path.normpath(os.path.join(root, i))
 159                                         if path.startswith("_darcs") or "-darcs-backup" in path:
 160                                                 continue
 161                                         self.files.append(path)
 162                 self.prevfiles = self.files[:]
 163                 adds = []
 164
 165                 self.read_next_line()
 166                 self.handle_mark()
 167                 if self.line.startswith("author "):
 168                         self.handle_ident(self.line[7:-1])
 169                         self.read_next_line()
 170                 if self.line.startswith("committer "):
 171                         self.handle_ident(self.line[10:-1])
 172                         self.read_next_line()
 173                 self.handle_data()
 174                 self.skip_optional_lf()
 175                 self.handle_msg()
 176                 self.read_next_line()
 177                 if self.line.startswith("from "):
 178                         self.read_next_line()
 179                 while self.line.startswith("merge "):
 180                         self.read_next_line()
 181                 while len(self.line) > 0:
 182                         if self.line.startswith("deleteall"):
 183                                 path = self.line[2:-1]
 184                                 for path in self.files:
 185                                         os.unlink(path)
 186                                 self.files = []
 187                         elif self.line.startswith("D "):
 188                                 path = self.line[2:-1]
 189                                 if os.path.exists(path):
 190                                         os.unlink(path)
 191                                 if path in self.files:
 192                                         self.files.remove(path)
 193                         elif self.line.startswith("R "):
 194                                 os.system("darcs mv %s" % self.line[2:])
 195                         elif self.line.startswith("C "):
 196                                 src, dest = self.line[:-1].split(' ')[1:]
 197                                 shutil.copy(src.strip('"'), dest.strip('"'))
 198                                 os.system("darcs add %s" % dest)
 199                         elif self.line.startswith("M "):
 200                                 items = self.line.split(' ')
 201                                 path = items[3][:-1]
 202                                 dir = os.path.split(path)[0]
 203                                 if len(dir) and not os.path.exists(dir):
 204                                         os.makedirs(dir)
 205                                 sock = open(path, "w")
 206                                 if items[2] != "inline":
 207                                         idx = int(items[2][1:])
 208                                         sock.write(self.marks[idx])
 209                                         del self.marks[idx]
 210                                 else:
 211                                         self.read_next_line()
 212                                         self.handle_data()
 213                                         sock.write(self.buf)
 214                                 sock.close()
 215                                 if path not in self.prevfiles:
 216                                         adds.append(path)
 217                                 if path not in self.files:
 218                                         self.files.append(path)
 219                         else:
 220                                 self.unread_line = True
 221                                 break
 222                         self.read_next_line()
 223                         if not len(self.line):
 224                                 break
 225
 226                 for i in adds:
 227                         os.system("darcs add %s" % i)
 228                 sock = subprocess.Popen(["darcs", "record", "--ignore-times", "-a", "--pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 229                 buf = [self.date, self.ident, self.short, self.long]
 230                 sock.stdin.write("\n".join(buf))
 231                 sock.stdin.close()
 232                 self.log("Recording :%s:\n%s" % (self.mark_num, sock.stdout.read()))
 233                 sock.stdout.close()
 234
 235                 if self.options.export_marks:
 236                         # yeah, an xml parser would be better, but
 237                         # should we mess with encodings just because of
 238                         # this? i hope not
 239                         sock = os.popen("darcs changes --last=1 --xml", "r")
 240                         buf = sock.read()
 241                         sock.close()
 242                         hash = buf.split('\n')[1].split("'")[-2]
 243                         self.export_marks.append(":%s %s" % (self.mark_num, hash))
 244
 245         def handle_progress(self, s):
 246                 print "progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s.strip())
 247                 sys.stdout.flush()
 248
 249         def handle_opts(self):
 250                 # Option Parser
 251                 usage="%prog [options]"
 252                 opp = optparse.OptionParser(usage=usage)
 253                 opp.add_option("--import-marks", metavar="IFILE",
 254                         help="read state for incremental imports from IFILE")
 255                 opp.add_option("--export-marks", metavar="OFILE",
 256                         help="write state for incremental imports to OFILE")
 257                 opp.add_option("--logfile", metavar="L",
 258                         help="log file which contains the output of external programs invoked during the conversion")
 259                 (self.options, args) = opp.parse_args()
 260
 261                 if self.options.logfile:
 262                         logfile = self.options.logfile
 263                 else:
 264                         logfile = "_darcs/import.log"
 265                 self.logsock = open(os.path.abspath(logfile), "a")
 266
 267         def log(self, s):
 268                 self.logsock.write("[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s))
 269                 self.logsock.flush()
 270
 271         def handle_export_marks(self):
 272                 if self.options.export_marks:
 273                         sock = open(self.options.export_marks, 'w')
 274                         sock.write("\n".join(self.export_marks))
 275                         sock.write("\n")
 276                         sock.close()
 277
 278         def handle_import_marks(self):
 279                 if self.options.import_marks:
 280                         sock = open(self.options.import_marks)
 281                         for i in sock.readlines():
 282                                 line = i.strip()
 283                                 if not len(line):
 284                                         continue
 285                                 self.import_marks.append(line.split(' ')[1])
 286                                 self.export_marks.append(line)
 287                         sock.close()
 288
 289         def handle(self):
 290                 self.handle_opts()
 291                 self.handle_import_marks()
 292
 293                 while not self.eof:
 294                         self.read_next_line()
 295                         if not len(self.line[:-1]):
 296                                 pass
 297                         elif self.line.startswith("blob"):
 298                                 self.handle_blob()
 299                         elif self.line.startswith("commit"):
 300                                 self.handle_commit()
 301                         elif self.line.startswith("tag"):
 302                                 self.handle_tag()
 303                         elif self.line.startswith("reset"):
 304                                 self.read_next_line()
 305                                 if not self.line.startswith("from "):
 306                                         self.unread_line = True
 307                         elif self.line.startswith("checkpoint"):
 308                                 pass
 309                         elif self.line.startswith("progress"):
 310                                 self.handle_progress(self.line[9:])
 311                         else:
 312                                 self.bug("'%s': invalid command" % self.line[:-1])
 313
 314                 self.handle_export_marks()
 315
 316 if __name__ == "__main__":
 317         h = Handler()
 318         h.handle()