third_party/re2/lib/codereview/codereview.py

   1 # coding=utf-8
   2 # (The line above is necessary so that I can use 世界 in the
   3 # *comment* below without Python getting all bent out of shape.)
   4
   5 # Copyright 2007-2009 Google Inc.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #       http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 '''Mercurial interface to codereview.appspot.com.
  20
  21 To configure, set the following options in
  22 your repository's .hg/hgrc file.
  23
  24         [extensions]
  25         codereview = /path/to/codereview.py
  26
  27         [codereview]
  28         server = codereview.appspot.com
  29
  30 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
  31
  32 In addition to the new commands, this extension introduces
  33 the file pattern syntax @nnnnnn, where nnnnnn is a change list
  34 number, to mean the files included in that change list, which
  35 must be associated with the current client.
  36
  37 For example, if change 123456 contains the files x.go and y.go,
  38 "hg diff @123456" is equivalent to"hg diff x.go y.go".
  39 '''
  40
  41 import sys
  42
  43 if __name__ == "__main__":
  44         print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
  45         sys.exit(2)
  46
  47 # We require Python 2.6 for the json package.
  48 if sys.version < '2.6':
  49         print >>sys.stderr, "The codereview extension requires Python 2.6 or newer."
  50         print >>sys.stderr, "You are running Python " + sys.version
  51         sys.exit(2)
  52
  53 import json
  54 import os
  55 import re
  56 import stat
  57 import subprocess
  58 import threading
  59 import time
  60
  61 from mercurial import commands as hg_commands
  62 from mercurial import util as hg_util
  63
  64 defaultcc = None
  65 codereview_disabled = None
  66 real_rollback = None
  67 releaseBranch = None
  68 server = "codereview.appspot.com"
  69 server_url_base = None
  70
  71 #######################################################################
  72 # Normally I would split this into multiple files, but it simplifies
  73 # import path headaches to keep it all in one file.  Sorry.
  74 # The different parts of the file are separated by banners like this one.
  75
  76 #######################################################################
  77 # Helpers
  78
  79 def RelativePath(path, cwd):
  80         n = len(cwd)
  81         if path.startswith(cwd) and path[n] == '/':
  82                 return path[n+1:]
  83         return path
  84
  85 def Sub(l1, l2):
  86         return [l for l in l1 if l not in l2]
  87
  88 def Add(l1, l2):
  89         l = l1 + Sub(l2, l1)
  90         l.sort()
  91         return l
  92
  93 def Intersect(l1, l2):
  94         return [l for l in l1 if l in l2]
  95
  96 #######################################################################
  97 # RE: UNICODE STRING HANDLING
  98 #
  99 # Python distinguishes between the str (string of bytes)
 100 # and unicode (string of code points) types.  Most operations
 101 # work on either one just fine, but some (like regexp matching)
 102 # require unicode, and others (like write) require str.
 103 #
 104 # As befits the language, Python hides the distinction between
 105 # unicode and str by converting between them silently, but
 106 # *only* if all the bytes/code points involved are 7-bit ASCII.
 107 # This means that if you're not careful, your program works
 108 # fine on "hello, world" and fails on "hello, 世界".  And of course,
 109 # the obvious way to be careful - use static types - is unavailable.
 110 # So the only way is trial and error to find where to put explicit
 111 # conversions.
 112 #
 113 # Because more functions do implicit conversion to str (string of bytes)
 114 # than do implicit conversion to unicode (string of code points),
 115 # the convention in this module is to represent all text as str,
 116 # converting to unicode only when calling a unicode-only function
 117 # and then converting back to str as soon as possible.
 118
 119 def typecheck(s, t):
 120         if type(s) != t:
 121                 raise hg_util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t))
 122
 123 # If we have to pass unicode instead of str, ustr does that conversion clearly.
 124 def ustr(s):
 125         typecheck(s, str)
 126         return s.decode("utf-8")
 127
 128 # Even with those, Mercurial still sometimes turns unicode into str
 129 # and then tries to use it as ascii.  Change Mercurial's default.
 130 def set_mercurial_encoding_to_utf8():
 131         from mercurial import encoding
 132         encoding.encoding = 'utf-8'
 133
 134 set_mercurial_encoding_to_utf8()
 135
 136 # Even with those we still run into problems.
 137 # I tried to do things by the book but could not convince
 138 # Mercurial to let me check in a change with UTF-8 in the
 139 # CL description or author field, no matter how many conversions
 140 # between str and unicode I inserted and despite changing the
 141 # default encoding.  I'm tired of this game, so set the default
 142 # encoding for all of Python to 'utf-8', not 'ascii'.
 143 def default_to_utf8():
 144         import sys
 145         stdout, __stdout__ = sys.stdout, sys.__stdout__
 146         reload(sys)  # site.py deleted setdefaultencoding; get it back
 147         sys.stdout, sys.__stdout__ = stdout, __stdout__
 148         sys.setdefaultencoding('utf-8')
 149
 150 default_to_utf8()
 151
 152 #######################################################################
 153 # Status printer for long-running commands
 154
 155 global_status = None
 156
 157 def set_status(s):
 158         # print >>sys.stderr, "\t", time.asctime(), s
 159         global global_status
 160         global_status = s
 161
 162 class StatusThread(threading.Thread):
 163         def __init__(self):
 164                 threading.Thread.__init__(self)
 165         def run(self):
 166                 # pause a reasonable amount of time before
 167                 # starting to display status messages, so that
 168                 # most hg commands won't ever see them.
 169                 time.sleep(30)
 170
 171                 # now show status every 15 seconds
 172                 while True:
 173                         time.sleep(15 - time.time() % 15)
 174                         s = global_status
 175                         if s is None:
 176                                 continue
 177                         if s == "":
 178                                 s = "(unknown status)"
 179                         print >>sys.stderr, time.asctime(), s
 180
 181 def start_status_thread():
 182         t = StatusThread()
 183         t.setDaemon(True)  # allowed to exit if t is still running
 184         t.start()
 185
 186 #######################################################################
 187 # Change list parsing.
 188 #
 189 # Change lists are stored in .hg/codereview/cl.nnnnnn
 190 # where nnnnnn is the number assigned by the code review server.
 191 # Most data about a change list is stored on the code review server
 192 # too: the description, reviewer, and cc list are all stored there.
 193 # The only thing in the cl.nnnnnn file is the list of relevant files.
 194 # Also, the existence of the cl.nnnnnn file marks this repository
 195 # as the one where the change list lives.
 196
 197 emptydiff = """Index: ~rietveld~placeholder~
 198 ===================================================================
 199 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
 200 new file mode 100644
 201 """
 202
 203 class CL(object):
 204         def __init__(self, name):
 205                 typecheck(name, str)
 206                 self.name = name
 207                 self.desc = ''
 208                 self.files = []
 209                 self.reviewer = []
 210                 self.cc = []
 211                 self.url = ''
 212                 self.local = False
 213                 self.web = False
 214                 self.copied_from = None # None means current user
 215                 self.mailed = False
 216                 self.private = False
 217                 self.lgtm = []
 218
 219         def DiskText(self):
 220                 cl = self
 221                 s = ""
 222                 if cl.copied_from:
 223                         s += "Author: " + cl.copied_from + "\n\n"
 224                 if cl.private:
 225                         s += "Private: " + str(self.private) + "\n"
 226                 s += "Mailed: " + str(self.mailed) + "\n"
 227                 s += "Description:\n"
 228                 s += Indent(cl.desc, "\t")
 229                 s += "Files:\n"
 230                 for f in cl.files:
 231                         s += "\t" + f + "\n"
 232                 typecheck(s, str)
 233                 return s
 234
 235         def EditorText(self):
 236                 cl = self
 237                 s = _change_prolog
 238                 s += "\n"
 239                 if cl.copied_from:
 240                         s += "Author: " + cl.copied_from + "\n"
 241                 if cl.url != '':
 242                         s += 'URL: ' + cl.url + '       # cannot edit\n\n'
 243                 if cl.private:
 244                         s += "Private: True\n"
 245                 s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
 246                 s += "CC: " + JoinComma(cl.cc) + "\n"
 247                 s += "\n"
 248                 s += "Description:\n"
 249                 if cl.desc == '':
 250                         s += "\t<enter description here>\n"
 251                 else:
 252                         s += Indent(cl.desc, "\t")
 253                 s += "\n"
 254                 if cl.local or cl.name == "new":
 255                         s += "Files:\n"
 256                         for f in cl.files:
 257                                 s += "\t" + f + "\n"
 258                         s += "\n"
 259                 typecheck(s, str)
 260                 return s
 261
 262         def PendingText(self, quick=False):
 263                 cl = self
 264                 s = cl.name + ":" + "\n"
 265                 s += Indent(cl.desc, "\t")
 266                 s += "\n"
 267                 if cl.copied_from:
 268                         s += "\tAuthor: " + cl.copied_from + "\n"
 269                 if not quick:
 270                         s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
 271                         for (who, line) in cl.lgtm:
 272                                 s += "\t\t" + who + ": " + line + "\n"
 273                         s += "\tCC: " + JoinComma(cl.cc) + "\n"
 274                 s += "\tFiles:\n"
 275                 for f in cl.files:
 276                         s += "\t\t" + f + "\n"
 277                 typecheck(s, str)
 278                 return s
 279
 280         def Flush(self, ui, repo):
 281                 if self.name == "new":
 282                         self.Upload(ui, repo, gofmt_just_warn=True, creating=True)
 283                 dir = CodeReviewDir(ui, repo)
 284                 path = dir + '/cl.' + self.name
 285                 f = open(path+'!', "w")
 286                 f.write(self.DiskText())
 287                 f.close()
 288                 if sys.platform == "win32" and os.path.isfile(path):
 289                         os.remove(path)
 290                 os.rename(path+'!', path)
 291                 if self.web and not self.copied_from:
 292                         EditDesc(self.name, desc=self.desc,
 293                                 reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc),
 294                                 private=self.private)
 295
 296         def Delete(self, ui, repo):
 297                 dir = CodeReviewDir(ui, repo)
 298                 os.unlink(dir + "/cl." + self.name)
 299
 300         def Subject(self):
 301                 s = line1(self.desc)
 302                 if len(s) > 60:
 303                         s = s[0:55] + "..."
 304                 if self.name != "new":
 305                         s = "code review %s: %s" % (self.name, s)
 306                 typecheck(s, str)
 307                 return s
 308
 309         def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False):
 310                 if not self.files and not creating:
 311                         ui.warn("no files in change list\n")
 312                 if ui.configbool("codereview", "force_gofmt", True) and gofmt:
 313                         CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn)
 314                 set_status("uploading CL metadata + diffs")
 315                 os.chdir(repo.root)
 316                 form_fields = [
 317                         ("content_upload", "1"),
 318                         ("reviewers", JoinComma(self.reviewer)),
 319                         ("cc", JoinComma(self.cc)),
 320                         ("description", self.desc),
 321                         ("base_hashes", ""),
 322                 ]
 323
 324                 if self.name != "new":
 325                         form_fields.append(("issue", self.name))
 326                 vcs = None
 327                 # We do not include files when creating the issue,
 328                 # because we want the patch sets to record the repository
 329                 # and base revision they are diffs against.  We use the patch
 330                 # set message for that purpose, but there is no message with
 331                 # the first patch set.  Instead the message gets used as the
 332                 # new CL's overall subject.  So omit the diffs when creating
 333                 # and then we'll run an immediate upload.
 334                 # This has the effect that every CL begins with an empty "Patch set 1".
 335                 if self.files and not creating:
 336                         vcs = MercurialVCS(upload_options, ui, repo)
 337                         data = vcs.GenerateDiff(self.files)
 338                         files = vcs.GetBaseFiles(data)
 339                         if len(data) > MAX_UPLOAD_SIZE:
 340                                 uploaded_diff_file = []
 341                                 form_fields.append(("separate_patches", "1"))
 342                         else:
 343                                 uploaded_diff_file = [("data", "data.diff", data)]
 344                 else:
 345                         uploaded_diff_file = [("data", "data.diff", emptydiff)]
 346
 347                 if vcs and self.name != "new":
 348                         form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + ui.expandpath("default")))
 349                 else:
 350                         # First upload sets the subject for the CL itself.
 351                         form_fields.append(("subject", self.Subject()))
 352                 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
 353                 response_body = MySend("/upload", body, content_type=ctype)
 354                 patchset = None
 355                 msg = response_body
 356                 lines = msg.splitlines()
 357                 if len(lines) >= 2:
 358                         msg = lines[0]
 359                         patchset = lines[1].strip()
 360                         patches = [x.split(" ", 1) for x in lines[2:]]
 361                 if response_body.startswith("Issue updated.") and quiet:
 362                         pass
 363                 else:
 364                         ui.status(msg + "\n")
 365                 set_status("uploaded CL metadata + diffs")
 366                 if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
 367                         raise hg_util.Abort("failed to update issue: " + response_body)
 368                 issue = msg[msg.rfind("/")+1:]
 369                 self.name = issue
 370                 if not self.url:
 371                         self.url = server_url_base + self.name
 372                 if not uploaded_diff_file:
 373                         set_status("uploading patches")
 374                         patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
 375                 if vcs:
 376                         set_status("uploading base files")
 377                         vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
 378                 if send_mail:
 379                         set_status("sending mail")
 380                         MySend("/" + issue + "/mail", payload="")
 381                 self.web = True
 382                 set_status("flushing changes to disk")
 383                 self.Flush(ui, repo)
 384                 return
 385
 386         def Mail(self, ui, repo):
 387                 pmsg = "Hello " + JoinComma(self.reviewer)
 388                 if self.cc:
 389                         pmsg += " (cc: %s)" % (', '.join(self.cc),)
 390                 pmsg += ",\n"
 391                 pmsg += "\n"
 392                 repourl = ui.expandpath("default")
 393                 if not self.mailed:
 394                         pmsg += "I'd like you to review this change to\n" + repourl + "\n"
 395                 else:
 396                         pmsg += "Please take another look.\n"
 397                 typecheck(pmsg, str)
 398                 PostMessage(ui, self.name, pmsg, subject=self.Subject())
 399                 self.mailed = True
 400                 self.Flush(ui, repo)
 401
 402 def GoodCLName(name):
 403         typecheck(name, str)
 404         return re.match("^[0-9]+$", name)
 405
 406 def ParseCL(text, name):
 407         typecheck(text, str)
 408         typecheck(name, str)
 409         sname = None
 410         lineno = 0
 411         sections = {
 412                 'Author': '',
 413                 'Description': '',
 414                 'Files': '',
 415                 'URL': '',
 416                 'Reviewer': '',
 417                 'CC': '',
 418                 'Mailed': '',
 419                 'Private': '',
 420         }
 421         for line in text.split('\n'):
 422                 lineno += 1
 423                 line = line.rstrip()
 424                 if line != '' and line[0] == '#':
 425                         continue
 426                 if line == '' or line[0] == ' ' or line[0] == '\t':
 427                         if sname == None and line != '':
 428                                 return None, lineno, 'text outside section'
 429                         if sname != None:
 430                                 sections[sname] += line + '\n'
 431                         continue
 432                 p = line.find(':')
 433                 if p >= 0:
 434                         s, val = line[:p].strip(), line[p+1:].strip()
 435                         if s in sections:
 436                                 sname = s
 437                                 if val != '':
 438                                         sections[sname] += val + '\n'
 439                                 continue
 440                 return None, lineno, 'malformed section header'
 441
 442         for k in sections:
 443                 sections[k] = StripCommon(sections[k]).rstrip()
 444
 445         cl = CL(name)
 446         if sections['Author']:
 447                 cl.copied_from = sections['Author']
 448         cl.desc = sections['Description']
 449         for line in sections['Files'].split('\n'):
 450                 i = line.find('#')
 451                 if i >= 0:
 452                         line = line[0:i].rstrip()
 453                 line = line.strip()
 454                 if line == '':
 455                         continue
 456                 cl.files.append(line)
 457         cl.reviewer = SplitCommaSpace(sections['Reviewer'])
 458         cl.cc = SplitCommaSpace(sections['CC'])
 459         cl.url = sections['URL']
 460         if sections['Mailed'] != 'False':
 461                 # Odd default, but avoids spurious mailings when
 462                 # reading old CLs that do not have a Mailed: line.
 463                 # CLs created with this update will always have
 464                 # Mailed: False on disk.
 465                 cl.mailed = True
 466         if sections['Private'] in ('True', 'true', 'Yes', 'yes'):
 467                 cl.private = True
 468         if cl.desc == '<enter description here>':
 469                 cl.desc = ''
 470         return cl, 0, ''
 471
 472 def SplitCommaSpace(s):
 473         typecheck(s, str)
 474         s = s.strip()
 475         if s == "":
 476                 return []
 477         return re.split(", *", s)
 478
 479 def CutDomain(s):
 480         typecheck(s, str)
 481         i = s.find('@')
 482         if i >= 0:
 483                 s = s[0:i]
 484         return s
 485
 486 def JoinComma(l):
 487         for s in l:
 488                 typecheck(s, str)
 489         return ", ".join(l)
 490
 491 def ExceptionDetail():
 492         s = str(sys.exc_info()[0])
 493         if s.startswith("<type '") and s.endswith("'>"):
 494                 s = s[7:-2]
 495         elif s.startswith("<class '") and s.endswith("'>"):
 496                 s = s[8:-2]
 497         arg = str(sys.exc_info()[1])
 498         if len(arg) > 0:
 499                 s += ": " + arg
 500         return s
 501
 502 def IsLocalCL(ui, repo, name):
 503         return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
 504
 505 # Load CL from disk and/or the web.
 506 def LoadCL(ui, repo, name, web=True):
 507         typecheck(name, str)
 508         set_status("loading CL " + name)
 509         if not GoodCLName(name):
 510                 return None, "invalid CL name"
 511         dir = CodeReviewDir(ui, repo)
 512         path = dir + "cl." + name
 513         if os.access(path, 0):
 514                 ff = open(path)
 515                 text = ff.read()
 516                 ff.close()
 517                 cl, lineno, err = ParseCL(text, name)
 518                 if err != "":
 519                         return None, "malformed CL data: "+err
 520                 cl.local = True
 521         else:
 522                 cl = CL(name)
 523         if web:
 524                 set_status("getting issue metadata from web")
 525                 d = JSONGet(ui, "/api/" + name + "?messages=true")
 526                 set_status(None)
 527                 if d is None:
 528                         return None, "cannot load CL %s from server" % (name,)
 529                 if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name:
 530                         return None, "malformed response loading CL data from code review server"
 531                 cl.dict = d
 532                 cl.reviewer = d.get('reviewers', [])
 533                 cl.cc = d.get('cc', [])
 534                 if cl.local and cl.copied_from and cl.desc:
 535                         # local copy of CL written by someone else
 536                         # and we saved a description.  use that one,
 537                         # so that committers can edit the description
 538                         # before doing hg submit.
 539                         pass
 540                 else:
 541                         cl.desc = d.get('description', "")
 542                 cl.url = server_url_base + name
 543                 cl.web = True
 544                 cl.private = d.get('private', False) != False
 545                 cl.lgtm = []
 546                 for m in d.get('messages', []):
 547                         if m.get('approval', False) == True:
 548                                 who = re.sub('@.*', '', m.get('sender', ''))
 549                                 text = re.sub("\n(.|\n)*", '', m.get('text', ''))
 550                                 cl.lgtm.append((who, text))
 551
 552         set_status("loaded CL " + name)
 553         return cl, ''
 554
 555 class LoadCLThread(threading.Thread):
 556         def __init__(self, ui, repo, dir, f, web):
 557                 threading.Thread.__init__(self)
 558                 self.ui = ui
 559                 self.repo = repo
 560                 self.dir = dir
 561                 self.f = f
 562                 self.web = web
 563                 self.cl = None
 564         def run(self):
 565                 cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
 566                 if err != '':
 567                         self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
 568                         return
 569                 self.cl = cl
 570
 571 # Load all the CLs from this repository.
 572 def LoadAllCL(ui, repo, web=True):
 573         dir = CodeReviewDir(ui, repo)
 574         m = {}
 575         files = [f for f in os.listdir(dir) if f.startswith('cl.')]
 576         if not files:
 577                 return m
 578         active = []
 579         first = True
 580         for f in files:
 581                 t = LoadCLThread(ui, repo, dir, f, web)
 582                 t.start()
 583                 if web and first:
 584                         # first request: wait in case it needs to authenticate
 585                         # otherwise we get lots of user/password prompts
 586                         # running in parallel.
 587                         t.join()
 588                         if t.cl:
 589                                 m[t.cl.name] = t.cl
 590                         first = False
 591                 else:
 592                         active.append(t)
 593         for t in active:
 594                 t.join()
 595                 if t.cl:
 596                         m[t.cl.name] = t.cl
 597         return m
 598
 599 # Find repository root.  On error, ui.warn and return None
 600 def RepoDir(ui, repo):
 601         url = repo.url();
 602         if not url.startswith('file:'):
 603                 ui.warn("repository %s is not in local file system\n" % (url,))
 604                 return None
 605         url = url[5:]
 606         if url.endswith('/'):
 607                 url = url[:-1]
 608         typecheck(url, str)
 609         return url
 610
 611 # Find (or make) code review directory.  On error, ui.warn and return None
 612 def CodeReviewDir(ui, repo):
 613         dir = RepoDir(ui, repo)
 614         if dir == None:
 615                 return None
 616         dir += '/.hg/codereview/'
 617         if not os.path.isdir(dir):
 618                 try:
 619                         os.mkdir(dir, 0700)
 620                 except:
 621                         ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
 622                         return None
 623         typecheck(dir, str)
 624         return dir
 625
 626 # Turn leading tabs into spaces, so that the common white space
 627 # prefix doesn't get confused when people's editors write out
 628 # some lines with spaces, some with tabs.  Only a heuristic
 629 # (some editors don't use 8 spaces either) but a useful one.
 630 def TabsToSpaces(line):
 631         i = 0
 632         while i < len(line) and line[i] == '\t':
 633                 i += 1
 634         return ' '*(8*i) + line[i:]
 635
 636 # Strip maximal common leading white space prefix from text
 637 def StripCommon(text):
 638         typecheck(text, str)
 639         ws = None
 640         for line in text.split('\n'):
 641                 line = line.rstrip()
 642                 if line == '':
 643                         continue
 644                 line = TabsToSpaces(line)
 645                 white = line[:len(line)-len(line.lstrip())]
 646                 if ws == None:
 647                         ws = white
 648                 else:
 649                         common = ''
 650                         for i in range(min(len(white), len(ws))+1):
 651                                 if white[0:i] == ws[0:i]:
 652                                         common = white[0:i]
 653                         ws = common
 654                 if ws == '':
 655                         break
 656         if ws == None:
 657                 return text
 658         t = ''
 659         for line in text.split('\n'):
 660                 line = line.rstrip()
 661                 line = TabsToSpaces(line)
 662                 if line.startswith(ws):
 663                         line = line[len(ws):]
 664                 if line == '' and t == '':
 665                         continue
 666                 t += line + '\n'
 667         while len(t) >= 2 and t[-2:] == '\n\n':
 668                 t = t[:-1]
 669         typecheck(t, str)
 670         return t
 671
 672 # Indent text with indent.
 673 def Indent(text, indent):
 674         typecheck(text, str)
 675         typecheck(indent, str)
 676         t = ''
 677         for line in text.split('\n'):
 678                 t += indent + line + '\n'
 679         typecheck(t, str)
 680         return t
 681
 682 # Return the first line of l
 683 def line1(text):
 684         typecheck(text, str)
 685         return text.split('\n')[0]
 686
 687 _change_prolog = """# Change list.
 688 # Lines beginning with # are ignored.
 689 # Multi-line values should be indented.
 690 """
 691
 692 desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)'
 693
 694 desc_msg = '''Your CL description appears not to use the standard form.
 695
 696 The first line of your change description is conventionally a
 697 one-line summary of the change, prefixed by the primary affected package,
 698 and is used as the subject for code review mail; the rest of the description
 699 elaborates.
 700
 701 Examples:
 702
 703         encoding/rot13: new package
 704
 705         math: add IsInf, IsNaN
 706
 707         net: fix cname in LookupHost
 708
 709         unicode: update to Unicode 5.0.2
 710
 711 '''
 712
 713 def promptyesno(ui, msg):
 714         if hgversion >= "2.7":
 715                 return ui.promptchoice(msg + " $$ &yes $$ &no", 0) == 0
 716         else:
 717                 return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
 718
 719 def promptremove(ui, repo, f):
 720         if promptyesno(ui, "hg remove %s (y/n)?" % (f,)):
 721                 if hg_commands.remove(ui, repo, 'path:'+f) != 0:
 722                         ui.warn("error removing %s" % (f,))
 723
 724 def promptadd(ui, repo, f):
 725         if promptyesno(ui, "hg add %s (y/n)?" % (f,)):
 726                 if hg_commands.add(ui, repo, 'path:'+f) != 0:
 727                         ui.warn("error adding %s" % (f,))
 728
 729 def EditCL(ui, repo, cl):
 730         set_status(None)        # do not show status
 731         s = cl.EditorText()
 732         while True:
 733                 s = ui.edit(s, ui.username())
 734
 735                 # We can't trust Mercurial + Python not to die before making the change,
 736                 # so, by popular demand, just scribble the most recent CL edit into
 737                 # $(hg root)/last-change so that if Mercurial does die, people
 738                 # can look there for their work.
 739                 try:
 740                         f = open(repo.root+"/last-change", "w")
 741                         f.write(s)
 742                         f.close()
 743                 except:
 744                         pass
 745
 746                 clx, line, err = ParseCL(s, cl.name)
 747                 if err != '':
 748                         if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
 749                                 return "change list not modified"
 750                         continue
 751
 752                 # Check description.
 753                 if clx.desc == '':
 754                         if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"):
 755                                 continue
 756                 elif re.search('<enter reason for undo>', clx.desc):
 757                         if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"):
 758                                 continue
 759                 elif not re.match(desc_re, clx.desc.split('\n')[0]):
 760                         if promptyesno(ui, desc_msg + "re-edit (y/n)?"):
 761                                 continue
 762
 763                 # Check file list for files that need to be hg added or hg removed
 764                 # or simply aren't understood.
 765                 pats = ['path:'+f for f in clx.files]
 766                 changed = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
 767                 deleted = hg_matchPattern(ui, repo, *pats, deleted=True)
 768                 unknown = hg_matchPattern(ui, repo, *pats, unknown=True)
 769                 ignored = hg_matchPattern(ui, repo, *pats, ignored=True)
 770                 clean = hg_matchPattern(ui, repo, *pats, clean=True)
 771                 files = []
 772                 for f in clx.files:
 773                         if f in changed:
 774                                 files.append(f)
 775                                 continue
 776                         if f in deleted:
 777                                 promptremove(ui, repo, f)
 778                                 files.append(f)
 779                                 continue
 780                         if f in unknown:
 781                                 promptadd(ui, repo, f)
 782                                 files.append(f)
 783                                 continue
 784                         if f in ignored:
 785                                 ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,))
 786                                 continue
 787                         if f in clean:
 788                                 ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,))
 789                                 files.append(f)
 790                                 continue
 791                         p = repo.root + '/' + f
 792                         if os.path.isfile(p):
 793                                 ui.warn("warning: %s is a file but not known to hg\n" % (f,))
 794                                 files.append(f)
 795                                 continue
 796                         if os.path.isdir(p):
 797                                 ui.warn("error: %s is a directory, not a file; omitting\n" % (f,))
 798                                 continue
 799                         ui.warn("error: %s does not exist; omitting\n" % (f,))
 800                 clx.files = files
 801
 802                 cl.desc = clx.desc
 803                 cl.reviewer = clx.reviewer
 804                 cl.cc = clx.cc
 805                 cl.files = clx.files
 806                 cl.private = clx.private
 807                 break
 808         return ""
 809
 810 # For use by submit, etc. (NOT by change)
 811 # Get change list number or list of files from command line.
 812 # If files are given, make a new change list.
 813 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
 814         if len(pats) > 0 and GoodCLName(pats[0]):
 815                 if len(pats) != 1:
 816                         return None, "cannot specify change number and file names"
 817                 if opts.get('message'):
 818                         return None, "cannot use -m with existing CL"
 819                 cl, err = LoadCL(ui, repo, pats[0], web=True)
 820                 if err != "":
 821                         return None, err
 822         else:
 823                 cl = CL("new")
 824                 cl.local = True
 825                 cl.files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
 826                 if not cl.files:
 827                         return None, "no files changed"
 828         if opts.get('reviewer'):
 829                 cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
 830         if opts.get('cc'):
 831                 cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
 832         if defaultcc:
 833                 cl.cc = Add(cl.cc, defaultcc)
 834         if cl.name == "new":
 835                 if opts.get('message'):
 836                         cl.desc = opts.get('message')
 837                 else:
 838                         err = EditCL(ui, repo, cl)
 839                         if err != '':
 840                                 return None, err
 841         return cl, ""
 842
 843 #######################################################################
 844 # Change list file management
 845
 846 # Return list of changed files in repository that match pats.
 847 # The patterns came from the command line, so we warn
 848 # if they have no effect or cannot be understood.
 849 def ChangedFiles(ui, repo, pats, taken=None):
 850         taken = taken or {}
 851         # Run each pattern separately so that we can warn about
 852         # patterns that didn't do anything useful.
 853         for p in pats:
 854                 for f in hg_matchPattern(ui, repo, p, unknown=True):
 855                         promptadd(ui, repo, f)
 856                 for f in hg_matchPattern(ui, repo, p, removed=True):
 857                         promptremove(ui, repo, f)
 858                 files = hg_matchPattern(ui, repo, p, modified=True, added=True, removed=True)
 859                 for f in files:
 860                         if f in taken:
 861                                 ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name))
 862                 if not files:
 863                         ui.warn("warning: %s did not match any modified files\n" % (p,))
 864
 865         # Again, all at once (eliminates duplicates)
 866         l = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
 867         l.sort()
 868         if taken:
 869                 l = Sub(l, taken.keys())
 870         return l
 871
 872 # Return list of changed files in repository that match pats and still exist.
 873 def ChangedExistingFiles(ui, repo, pats, opts):
 874         l = hg_matchPattern(ui, repo, *pats, modified=True, added=True)
 875         l.sort()
 876         return l
 877
 878 # Return list of files claimed by existing CLs
 879 def Taken(ui, repo):
 880         all = LoadAllCL(ui, repo, web=False)
 881         taken = {}
 882         for _, cl in all.items():
 883                 for f in cl.files:
 884                         taken[f] = cl
 885         return taken
 886
 887 # Return list of changed files that are not claimed by other CLs
 888 def DefaultFiles(ui, repo, pats):
 889         return ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
 890
 891 #######################################################################
 892 # File format checking.
 893
 894 def CheckFormat(ui, repo, files, just_warn=False):
 895         set_status("running gofmt")
 896         CheckGofmt(ui, repo, files, just_warn)
 897         CheckTabfmt(ui, repo, files, just_warn)
 898
 899 # Check that gofmt run on the list of files does not change them
 900 def CheckGofmt(ui, repo, files, just_warn):
 901         files = gofmt_required(files)
 902         if not files:
 903                 return
 904         cwd = os.getcwd()
 905         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
 906         files = [f for f in files if os.access(f, 0)]
 907         if not files:
 908                 return
 909         try:
 910                 cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32")
 911                 cmd.stdin.close()
 912         except:
 913                 raise hg_util.Abort("gofmt: " + ExceptionDetail())
 914         data = cmd.stdout.read()
 915         errors = cmd.stderr.read()
 916         cmd.wait()
 917         set_status("done with gofmt")
 918         if len(errors) > 0:
 919                 ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
 920                 return
 921         if len(data) > 0:
 922                 msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
 923                 if just_warn:
 924                         ui.warn("warning: " + msg + "\n")
 925                 else:
 926                         raise hg_util.Abort(msg)
 927         return
 928
 929 # Check that *.[chys] files indent using tabs.
 930 def CheckTabfmt(ui, repo, files, just_warn):
 931         files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f) and not re.search(r"\.tab\.[ch]$", f)]
 932         if not files:
 933                 return
 934         cwd = os.getcwd()
 935         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
 936         files = [f for f in files if os.access(f, 0)]
 937         badfiles = []
 938         for f in files:
 939                 try:
 940                         for line in open(f, 'r'):
 941                                 # Four leading spaces is enough to complain about,
 942                                 # except that some Plan 9 code uses four spaces as the label indent,
 943                                 # so allow that.
 944                                 if line.startswith('    ') and not re.match('    [A-Za-z0-9_]+:', line):
 945                                         badfiles.append(f)
 946                                         break
 947                 except:
 948                         # ignore cannot open file, etc.
 949                         pass
 950         if len(badfiles) > 0:
 951                 msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles)
 952                 if just_warn:
 953                         ui.warn("warning: " + msg + "\n")
 954                 else:
 955                         raise hg_util.Abort(msg)
 956         return
 957
 958 #######################################################################
 959 # CONTRIBUTORS file parsing
 960
 961 contributorsCache = None
 962 contributorsURL = None
 963
 964 def ReadContributors(ui, repo):
 965         global contributorsCache
 966         if contributorsCache is not None:
 967                 return contributorsCache
 968
 969         try:
 970                 if contributorsURL is not None:
 971                         opening = contributorsURL
 972                         f = urllib2.urlopen(contributorsURL)
 973                 else:
 974                         opening = repo.root + '/CONTRIBUTORS'
 975                         f = open(repo.root + '/CONTRIBUTORS', 'r')
 976         except:
 977                 ui.write("warning: cannot open %s: %s\n" % (opening, ExceptionDetail()))
 978                 return
 979
 980         contributors = {}
 981         for line in f:
 982                 # CONTRIBUTORS is a list of lines like:
 983                 #       Person <email>
 984                 #       Person <email> <alt-email>
 985                 # The first email address is the one used in commit logs.
 986                 if line.startswith('#'):
 987                         continue
 988                 m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
 989                 if m:
 990                         name = m.group(1)
 991                         email = m.group(2)[1:-1]
 992                         contributors[email.lower()] = (name, email)
 993                         for extra in m.group(3).split():
 994                                 contributors[extra[1:-1].lower()] = (name, email)
 995
 996         contributorsCache = contributors
 997         return contributors
 998
 999 def CheckContributor(ui, repo, user=None):
1000         set_status("checking CONTRIBUTORS file")
1001         user, userline = FindContributor(ui, repo, user, warn=False)
1002         if not userline:
1003                 raise hg_util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1004         return userline
1005
1006 def FindContributor(ui, repo, user=None, warn=True):
1007         if not user:
1008                 user = ui.config("ui", "username")
1009                 if not user:
1010                         raise hg_util.Abort("[ui] username is not configured in .hgrc")
1011         user = user.lower()
1012         m = re.match(r".*<(.*)>", user)
1013         if m:
1014                 user = m.group(1)
1015
1016         contributors = ReadContributors(ui, repo)
1017         if user not in contributors:
1018                 if warn:
1019                         ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1020                 return user, None
1021
1022         user, email = contributors[user]
1023         return email, "%s <%s>" % (user, email)
1024
1025 #######################################################################
1026 # Mercurial helper functions.
1027 # Read http://mercurial.selenic.com/wiki/MercurialApi before writing any of these.
1028 # We use the ui.pushbuffer/ui.popbuffer + hg_commands.xxx tricks for all interaction
1029 # with Mercurial.  It has proved the most stable as they make changes.
1030
1031 hgversion = hg_util.version()
1032
1033 # We require Mercurial 1.9 and suggest Mercurial 2.0.
1034 # The details of the scmutil package changed then,
1035 # so allowing earlier versions would require extra band-aids below.
1036 # Ubuntu 11.10 ships with Mercurial 1.9.1 as the default version.
1037 hg_required = "1.9"
1038 hg_suggested = "2.0"
1039
1040 old_message = """
1041
1042 The code review extension requires Mercurial """+hg_required+""" or newer.
1043 You are using Mercurial """+hgversion+""".
1044
1045 To install a new Mercurial, use
1046
1047         sudo easy_install mercurial=="""+hg_suggested+"""
1048
1049 or visit http://mercurial.selenic.com/downloads/.
1050 """
1051
1052 linux_message = """
1053 You may need to clear your current Mercurial installation by running:
1054
1055         sudo apt-get remove mercurial mercurial-common
1056         sudo rm -rf /etc/mercurial
1057 """
1058
1059 if hgversion < hg_required:
1060         msg = old_message
1061         if os.access("/etc/mercurial", 0):
1062                 msg += linux_message
1063         raise hg_util.Abort(msg)
1064
1065 from mercurial.hg import clean as hg_clean
1066 from mercurial import cmdutil as hg_cmdutil
1067 from mercurial import error as hg_error
1068 from mercurial import match as hg_match
1069 from mercurial import node as hg_node
1070
1071 class uiwrap(object):
1072         def __init__(self, ui):
1073                 self.ui = ui
1074                 ui.pushbuffer()
1075                 self.oldQuiet = ui.quiet
1076                 ui.quiet = True
1077                 self.oldVerbose = ui.verbose
1078                 ui.verbose = False
1079         def output(self):
1080                 ui = self.ui
1081                 ui.quiet = self.oldQuiet
1082                 ui.verbose = self.oldVerbose
1083                 return ui.popbuffer()
1084
1085 def to_slash(path):
1086         if sys.platform == "win32":
1087                 return path.replace('\\', '/')
1088         return path
1089
1090 def hg_matchPattern(ui, repo, *pats, **opts):
1091         w = uiwrap(ui)
1092         hg_commands.status(ui, repo, *pats, **opts)
1093         text = w.output()
1094         ret = []
1095         prefix = to_slash(os.path.realpath(repo.root))+'/'
1096         for line in text.split('\n'):
1097                 f = line.split()
1098                 if len(f) > 1:
1099                         if len(pats) > 0:
1100                                 # Given patterns, Mercurial shows relative to cwd
1101                                 p = to_slash(os.path.realpath(f[1]))
1102                                 if not p.startswith(prefix):
1103                                         print >>sys.stderr, "File %s not in repo root %s.\n" % (p, prefix)
1104                                 else:
1105                                         ret.append(p[len(prefix):])
1106                         else:
1107                                 # Without patterns, Mercurial shows relative to root (what we want)
1108                                 ret.append(to_slash(f[1]))
1109         return ret
1110
1111 def hg_heads(ui, repo):
1112         w = uiwrap(ui)
1113         hg_commands.heads(ui, repo)
1114         return w.output()
1115
1116 noise = [
1117         "",
1118         "resolving manifests",
1119         "searching for changes",
1120         "couldn't find merge tool hgmerge",
1121         "adding changesets",
1122         "adding manifests",
1123         "adding file changes",
1124         "all local heads known remotely",
1125 ]
1126
1127 def isNoise(line):
1128         line = str(line)
1129         for x in noise:
1130                 if line == x:
1131                         return True
1132         return False
1133
1134 def hg_incoming(ui, repo):
1135         w = uiwrap(ui)
1136         ret = hg_commands.incoming(ui, repo, force=False, bundle="")
1137         if ret and ret != 1:
1138                 raise hg_util.Abort(ret)
1139         return w.output()
1140
1141 def hg_log(ui, repo, **opts):
1142         for k in ['date', 'keyword', 'rev', 'user']:
1143                 if not opts.has_key(k):
1144                         opts[k] = ""
1145         w = uiwrap(ui)
1146         ret = hg_commands.log(ui, repo, **opts)
1147         if ret:
1148                 raise hg_util.Abort(ret)
1149         return w.output()
1150
1151 def hg_outgoing(ui, repo, **opts):
1152         w = uiwrap(ui)
1153         ret = hg_commands.outgoing(ui, repo, **opts)
1154         if ret and ret != 1:
1155                 raise hg_util.Abort(ret)
1156         return w.output()
1157
1158 def hg_pull(ui, repo, **opts):
1159         w = uiwrap(ui)
1160         ui.quiet = False
1161         ui.verbose = True  # for file list
1162         err = hg_commands.pull(ui, repo, **opts)
1163         for line in w.output().split('\n'):
1164                 if isNoise(line):
1165                         continue
1166                 if line.startswith('moving '):
1167                         line = 'mv ' + line[len('moving '):]
1168                 if line.startswith('getting ') and line.find(' to ') >= 0:
1169                         line = 'mv ' + line[len('getting '):]
1170                 if line.startswith('getting '):
1171                         line = '+ ' + line[len('getting '):]
1172                 if line.startswith('removing '):
1173                         line = '- ' + line[len('removing '):]
1174                 ui.write(line + '\n')
1175         return err
1176
1177 def hg_push(ui, repo, **opts):
1178         w = uiwrap(ui)
1179         ui.quiet = False
1180         ui.verbose = True
1181         err = hg_commands.push(ui, repo, **opts)
1182         for line in w.output().split('\n'):
1183                 if not isNoise(line):
1184                         ui.write(line + '\n')
1185         return err
1186
1187 def hg_commit(ui, repo, *pats, **opts):
1188         return hg_commands.commit(ui, repo, *pats, **opts)
1189
1190 #######################################################################
1191 # Mercurial precommit hook to disable commit except through this interface.
1192
1193 commit_okay = False
1194
1195 def precommithook(ui, repo, **opts):
1196         if commit_okay:
1197                 return False  # False means okay.
1198         ui.write("\ncodereview extension enabled; use mail, upload, or submit instead of commit\n\n")
1199         return True
1200
1201 #######################################################################
1202 # @clnumber file pattern support
1203
1204 # We replace scmutil.match with the MatchAt wrapper to add the @clnumber pattern.
1205
1206 match_repo = None
1207 match_ui = None
1208 match_orig = None
1209
1210 def InstallMatch(ui, repo):
1211         global match_repo
1212         global match_ui
1213         global match_orig
1214
1215         match_ui = ui
1216         match_repo = repo
1217
1218         from mercurial import scmutil
1219         match_orig = scmutil.match
1220         scmutil.match = MatchAt
1221
1222 def MatchAt(ctx, pats=None, opts=None, globbed=False, default='relpath'):
1223         taken = []
1224         files = []
1225         pats = pats or []
1226         opts = opts or {}
1227
1228         for p in pats:
1229                 if p.startswith('@'):
1230                         taken.append(p)
1231                         clname = p[1:]
1232                         if clname == "default":
1233                                 files = DefaultFiles(match_ui, match_repo, [])
1234                         else:
1235                                 if not GoodCLName(clname):
1236                                         raise hg_util.Abort("invalid CL name " + clname)
1237                                 cl, err = LoadCL(match_repo.ui, match_repo, clname, web=False)
1238                                 if err != '':
1239                                         raise hg_util.Abort("loading CL " + clname + ": " + err)
1240                                 if not cl.files:
1241                                         raise hg_util.Abort("no files in CL " + clname)
1242                                 files = Add(files, cl.files)
1243         pats = Sub(pats, taken) + ['path:'+f for f in files]
1244
1245         # work-around for http://selenic.com/hg/rev/785bbc8634f8
1246         if not hasattr(ctx, 'match'):
1247                 ctx = ctx[None]
1248         return match_orig(ctx, pats=pats, opts=opts, globbed=globbed, default=default)
1249
1250 #######################################################################
1251 # Commands added by code review extension.
1252
1253 # As of Mercurial 2.1 the commands are all required to return integer
1254 # exit codes, whereas earlier versions allowed returning arbitrary strings
1255 # to be printed as errors.  We wrap the old functions to make sure we
1256 # always return integer exit codes now.  Otherwise Mercurial dies
1257 # with a TypeError traceback (unsupported operand type(s) for &: 'str' and 'int').
1258 # Introduce a Python decorator to convert old functions to the new
1259 # stricter convention.
1260
1261 def hgcommand(f):
1262         def wrapped(ui, repo, *pats, **opts):
1263                 err = f(ui, repo, *pats, **opts)
1264                 if type(err) is int:
1265                         return err
1266                 if not err:
1267                         return 0
1268                 raise hg_util.Abort(err)
1269         wrapped.__doc__ = f.__doc__
1270         return wrapped
1271
1272 #######################################################################
1273 # hg change
1274
1275 @hgcommand
1276 def change(ui, repo, *pats, **opts):
1277         """create, edit or delete a change list
1278
1279         Create, edit or delete a change list.
1280         A change list is a group of files to be reviewed and submitted together,
1281         plus a textual description of the change.
1282         Change lists are referred to by simple alphanumeric names.
1283
1284         Changes must be reviewed before they can be submitted.
1285
1286         In the absence of options, the change command opens the
1287         change list for editing in the default editor.
1288
1289         Deleting a change with the -d or -D flag does not affect
1290         the contents of the files listed in that change.  To revert
1291         the files listed in a change, use
1292
1293                 hg revert @123456
1294
1295         before running hg change -d 123456.
1296         """
1297
1298         if codereview_disabled:
1299                 return codereview_disabled
1300
1301         dirty = {}
1302         if len(pats) > 0 and GoodCLName(pats[0]):
1303                 name = pats[0]
1304                 if len(pats) != 1:
1305                         return "cannot specify CL name and file patterns"
1306                 pats = pats[1:]
1307                 cl, err = LoadCL(ui, repo, name, web=True)
1308                 if err != '':
1309                         return err
1310                 if not cl.local and (opts["stdin"] or not opts["stdout"]):
1311                         return "cannot change non-local CL " + name
1312         else:
1313                 name = "new"
1314                 cl = CL("new")
1315                 if repo[None].branch() != "default":
1316                         return "cannot create CL outside default branch; switch with 'hg update default'"
1317                 dirty[cl] = True
1318                 files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
1319
1320         if opts["delete"] or opts["deletelocal"]:
1321                 if opts["delete"] and opts["deletelocal"]:
1322                         return "cannot use -d and -D together"
1323                 flag = "-d"
1324                 if opts["deletelocal"]:
1325                         flag = "-D"
1326                 if name == "new":
1327                         return "cannot use "+flag+" with file patterns"
1328                 if opts["stdin"] or opts["stdout"]:
1329                         return "cannot use "+flag+" with -i or -o"
1330                 if not cl.local:
1331                         return "cannot change non-local CL " + name
1332                 if opts["delete"]:
1333                         if cl.copied_from:
1334                                 return "original author must delete CL; hg change -D will remove locally"
1335                         PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed)
1336                         EditDesc(cl.name, closed=True, private=cl.private)
1337                 cl.Delete(ui, repo)
1338                 return
1339
1340         if opts["stdin"]:
1341                 s = sys.stdin.read()
1342                 clx, line, err = ParseCL(s, name)
1343                 if err != '':
1344                         return "error parsing change list: line %d: %s" % (line, err)
1345                 if clx.desc is not None:
1346                         cl.desc = clx.desc;
1347                         dirty[cl] = True
1348                 if clx.reviewer is not None:
1349                         cl.reviewer = clx.reviewer
1350                         dirty[cl] = True
1351                 if clx.cc is not None:
1352                         cl.cc = clx.cc
1353                         dirty[cl] = True
1354                 if clx.files is not None:
1355                         cl.files = clx.files
1356                         dirty[cl] = True
1357                 if clx.private != cl.private:
1358                         cl.private = clx.private
1359                         dirty[cl] = True
1360
1361         if not opts["stdin"] and not opts["stdout"]:
1362                 if name == "new":
1363                         cl.files = files
1364                 err = EditCL(ui, repo, cl)
1365                 if err != "":
1366                         return err
1367                 dirty[cl] = True
1368
1369         for d, _ in dirty.items():
1370                 name = d.name
1371                 d.Flush(ui, repo)
1372                 if name == "new":
1373                         d.Upload(ui, repo, quiet=True)
1374
1375         if opts["stdout"]:
1376                 ui.write(cl.EditorText())
1377         elif opts["pending"]:
1378                 ui.write(cl.PendingText())
1379         elif name == "new":
1380                 if ui.quiet:
1381                         ui.write(cl.name)
1382                 else:
1383                         ui.write("CL created: " + cl.url + "\n")
1384         return
1385
1386 #######################################################################
1387 # hg code-login (broken?)
1388
1389 @hgcommand
1390 def code_login(ui, repo, **opts):
1391         """log in to code review server
1392
1393         Logs in to the code review server, saving a cookie in
1394         a file in your home directory.
1395         """
1396         if codereview_disabled:
1397                 return codereview_disabled
1398
1399         MySend(None)
1400
1401 #######################################################################
1402 # hg clpatch / undo / release-apply / download
1403 # All concerned with applying or unapplying patches to the repository.
1404
1405 @hgcommand
1406 def clpatch(ui, repo, clname, **opts):
1407         """import a patch from the code review server
1408
1409         Imports a patch from the code review server into the local client.
1410         If the local client has already modified any of the files that the
1411         patch modifies, this command will refuse to apply the patch.
1412
1413         Submitting an imported patch will keep the original author's
1414         name as the Author: line but add your own name to a Committer: line.
1415         """
1416         if repo[None].branch() != "default":
1417                 return "cannot run hg clpatch outside default branch"
1418         return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch")
1419
1420 @hgcommand
1421 def undo(ui, repo, clname, **opts):
1422         """undo the effect of a CL
1423
1424         Creates a new CL that undoes an earlier CL.
1425         After creating the CL, opens the CL text for editing so that
1426         you can add the reason for the undo to the description.
1427         """
1428         if repo[None].branch() != "default":
1429                 return "cannot run hg undo outside default branch"
1430         return clpatch_or_undo(ui, repo, clname, opts, mode="undo")
1431
1432 @hgcommand
1433 def release_apply(ui, repo, clname, **opts):
1434         """apply a CL to the release branch
1435
1436         Creates a new CL copying a previously committed change
1437         from the main branch to the release branch.
1438         The current client must either be clean or already be in
1439         the release branch.
1440
1441         The release branch must be created by starting with a
1442         clean client, disabling the code review plugin, and running:
1443
1444                 hg update weekly.YYYY-MM-DD
1445                 hg branch release-branch.rNN
1446                 hg commit -m 'create release-branch.rNN'
1447                 hg push --new-branch
1448
1449         Then re-enable the code review plugin.
1450
1451         People can test the release branch by running
1452
1453                 hg update release-branch.rNN
1454
1455         in a clean client.  To return to the normal tree,
1456
1457                 hg update default
1458
1459         Move changes since the weekly into the release branch
1460         using hg release-apply followed by the usual code review
1461         process and hg submit.
1462
1463         When it comes time to tag the release, record the
1464         final long-form tag of the release-branch.rNN
1465         in the *default* branch's .hgtags file.  That is, run
1466
1467                 hg update default
1468
1469         and then edit .hgtags as you would for a weekly.
1470
1471         """
1472         c = repo[None]
1473         if not releaseBranch:
1474                 return "no active release branches"
1475         if c.branch() != releaseBranch:
1476                 if c.modified() or c.added() or c.removed():
1477                         raise hg_util.Abort("uncommitted local changes - cannot switch branches")
1478                 err = hg_clean(repo, releaseBranch)
1479                 if err:
1480                         return err
1481         try:
1482                 err = clpatch_or_undo(ui, repo, clname, opts, mode="backport")
1483                 if err:
1484                         raise hg_util.Abort(err)
1485         except Exception, e:
1486                 hg_clean(repo, "default")
1487                 raise e
1488         return None
1489
1490 def rev2clname(rev):
1491         # Extract CL name from revision description.
1492         # The last line in the description that is a codereview URL is the real one.
1493         # Earlier lines might be part of the user-written description.
1494         all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description())
1495         if len(all) > 0:
1496                 return all[-1]
1497         return ""
1498
1499 undoHeader = """undo CL %s / %s
1500
1501 <enter reason for undo>
1502
1503 ««« original CL description
1504 """
1505
1506 undoFooter = """
1507 »»»
1508 """
1509
1510 backportHeader = """[%s] %s
1511
1512 ««« CL %s / %s
1513 """
1514
1515 backportFooter = """
1516 »»»
1517 """
1518
1519 # Implementation of clpatch/undo.
1520 def clpatch_or_undo(ui, repo, clname, opts, mode):
1521         if codereview_disabled:
1522                 return codereview_disabled
1523
1524         if mode == "undo" or mode == "backport":
1525                 # Find revision in Mercurial repository.
1526                 # Assume CL number is 7+ decimal digits.
1527                 # Otherwise is either change log sequence number (fewer decimal digits),
1528                 # hexadecimal hash, or tag name.
1529                 # Mercurial will fall over long before the change log
1530                 # sequence numbers get to be 7 digits long.
1531                 if re.match('^[0-9]{7,}$', clname):
1532                         found = False
1533                         for r in hg_log(ui, repo, keyword="codereview.appspot.com/"+clname, limit=100, template="{node}\n").split():
1534                                 rev = repo[r]
1535                                 # Last line with a code review URL is the actual review URL.
1536                                 # Earlier ones might be part of the CL description.
1537                                 n = rev2clname(rev)
1538                                 if n == clname:
1539                                         found = True
1540                                         break
1541                         if not found:
1542                                 return "cannot find CL %s in local repository" % clname
1543                 else:
1544                         rev = repo[clname]
1545                         if not rev:
1546                                 return "unknown revision %s" % clname
1547                         clname = rev2clname(rev)
1548                         if clname == "":
1549                                 return "cannot find CL name in revision description"
1550
1551                 # Create fresh CL and start with patch that would reverse the change.
1552                 vers = hg_node.short(rev.node())
1553                 cl = CL("new")
1554                 desc = str(rev.description())
1555                 if mode == "undo":
1556                         cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter
1557                 else:
1558                         cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter
1559                 v1 = vers
1560                 v0 = hg_node.short(rev.parents()[0].node())
1561                 if mode == "undo":
1562                         arg = v1 + ":" + v0
1563                 else:
1564                         vers = v0
1565                         arg = v0 + ":" + v1
1566                 patch = RunShell(["hg", "diff", "--git", "-r", arg])
1567
1568         else:  # clpatch
1569                 cl, vers, patch, err = DownloadCL(ui, repo, clname)
1570                 if err != "":
1571                         return err
1572                 if patch == emptydiff:
1573                         return "codereview issue %s has no diff" % clname
1574
1575         # find current hg version (hg identify)
1576         ctx = repo[None]
1577         parents = ctx.parents()
1578         id = '+'.join([hg_node.short(p.node()) for p in parents])
1579
1580         # if version does not match the patch version,
1581         # try to update the patch line numbers.
1582         if vers != "" and id != vers:
1583                 # "vers in repo" gives the wrong answer
1584                 # on some versions of Mercurial.  Instead, do the actual
1585                 # lookup and catch the exception.
1586                 try:
1587                         repo[vers].description()
1588                 except:
1589                         return "local repository is out of date; sync to get %s" % (vers)
1590                 patch1, err = portPatch(repo, patch, vers, id)
1591                 if err != "":
1592                         if not opts["ignore_hgpatch_failure"]:
1593                                 return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id)
1594                 else:
1595                         patch = patch1
1596         argv = ["hgpatch"]
1597         if opts["no_incoming"] or mode == "backport":
1598                 argv += ["--checksync=false"]
1599         try:
1600                 cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32")
1601         except:
1602                 return "hgpatch: " + ExceptionDetail() + "\nInstall hgpatch with:\n$ go get code.google.com/p/go.codereview/cmd/hgpatch\n"
1603
1604         out, err = cmd.communicate(patch)
1605         if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]:
1606                 return "hgpatch failed"
1607         cl.local = True
1608         cl.files = out.strip().split()
1609         if not cl.files and not opts["ignore_hgpatch_failure"]:
1610                 return "codereview issue %s has no changed files" % clname
1611         files = ChangedFiles(ui, repo, [])
1612         extra = Sub(cl.files, files)
1613         if extra:
1614                 ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
1615         cl.Flush(ui, repo)
1616         if mode == "undo":
1617                 err = EditCL(ui, repo, cl)
1618                 if err != "":
1619                         return "CL created, but error editing: " + err
1620                 cl.Flush(ui, repo)
1621         else:
1622                 ui.write(cl.PendingText() + "\n")
1623
1624 # portPatch rewrites patch from being a patch against
1625 # oldver to being a patch against newver.
1626 def portPatch(repo, patch, oldver, newver):
1627         lines = patch.splitlines(True) # True = keep \n
1628         delta = None
1629         for i in range(len(lines)):
1630                 line = lines[i]
1631                 if line.startswith('--- a/'):
1632                         file = line[6:-1]
1633                         delta = fileDeltas(repo, file, oldver, newver)
1634                 if not delta or not line.startswith('@@ '):
1635                         continue
1636                 # @@ -x,y +z,w @@ means the patch chunk replaces
1637                 # the original file's line numbers x up to x+y with the
1638                 # line numbers z up to z+w in the new file.
1639                 # Find the delta from x in the original to the same
1640                 # line in the current version and add that delta to both
1641                 # x and z.
1642                 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1643                 if not m:
1644                         return None, "error parsing patch line numbers"
1645                 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1646                 d, err = lineDelta(delta, n1, len1)
1647                 if err != "":
1648                         return "", err
1649                 n1 += d
1650                 n2 += d
1651                 lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2)
1652
1653         newpatch = ''.join(lines)
1654         return newpatch, ""
1655
1656 # fileDelta returns the line number deltas for the given file's
1657 # changes from oldver to newver.
1658 # The deltas are a list of (n, len, newdelta) triples that say
1659 # lines [n, n+len) were modified, and after that range the
1660 # line numbers are +newdelta from what they were before.
1661 def fileDeltas(repo, file, oldver, newver):
1662         cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file]
1663         data = RunShell(cmd, silent_ok=True)
1664         deltas = []
1665         for line in data.splitlines():
1666                 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1667                 if not m:
1668                         continue
1669                 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1670                 deltas.append((n1, len1, n2+len2-(n1+len1)))
1671         return deltas
1672
1673 # lineDelta finds the appropriate line number delta to apply to the lines [n, n+len).
1674 # It returns an error if those lines were rewritten by the patch.
1675 def lineDelta(deltas, n, len):
1676         d = 0
1677         for (old, oldlen, newdelta) in deltas:
1678                 if old >= n+len:
1679                         break
1680                 if old+len > n:
1681                         return 0, "patch and recent changes conflict"
1682                 d = newdelta
1683         return d, ""
1684
1685 @hgcommand
1686 def download(ui, repo, clname, **opts):
1687         """download a change from the code review server
1688
1689         Download prints a description of the given change list
1690         followed by its diff, downloaded from the code review server.
1691         """
1692         if codereview_disabled:
1693                 return codereview_disabled
1694
1695         cl, vers, patch, err = DownloadCL(ui, repo, clname)
1696         if err != "":
1697                 return err
1698         ui.write(cl.EditorText() + "\n")
1699         ui.write(patch + "\n")
1700         return
1701
1702 #######################################################################
1703 # hg file
1704
1705 @hgcommand
1706 def file(ui, repo, clname, pat, *pats, **opts):
1707         """assign files to or remove files from a change list
1708
1709         Assign files to or (with -d) remove files from a change list.
1710
1711         The -d option only removes files from the change list.
1712         It does not edit them or remove them from the repository.
1713         """
1714         if codereview_disabled:
1715                 return codereview_disabled
1716
1717         pats = tuple([pat] + list(pats))
1718         if not GoodCLName(clname):
1719                 return "invalid CL name " + clname
1720
1721         dirty = {}
1722         cl, err = LoadCL(ui, repo, clname, web=False)
1723         if err != '':
1724                 return err
1725         if not cl.local:
1726                 return "cannot change non-local CL " + clname
1727
1728         files = ChangedFiles(ui, repo, pats)
1729
1730         if opts["delete"]:
1731                 oldfiles = Intersect(files, cl.files)
1732                 if oldfiles:
1733                         if not ui.quiet:
1734                                 ui.status("# Removing files from CL.  To undo:\n")
1735                                 ui.status("#    cd %s\n" % (repo.root))
1736                                 for f in oldfiles:
1737                                         ui.status("#    hg file %s %s\n" % (cl.name, f))
1738                         cl.files = Sub(cl.files, oldfiles)
1739                         cl.Flush(ui, repo)
1740                 else:
1741                         ui.status("no such files in CL")
1742                 return
1743
1744         if not files:
1745                 return "no such modified files"
1746
1747         files = Sub(files, cl.files)
1748         taken = Taken(ui, repo)
1749         warned = False
1750         for f in files:
1751                 if f in taken:
1752                         if not warned and not ui.quiet:
1753                                 ui.status("# Taking files from other CLs.  To undo:\n")
1754                                 ui.status("#    cd %s\n" % (repo.root))
1755                                 warned = True
1756                         ocl = taken[f]
1757                         if not ui.quiet:
1758                                 ui.status("#    hg file %s %s\n" % (ocl.name, f))
1759                         if ocl not in dirty:
1760                                 ocl.files = Sub(ocl.files, files)
1761                                 dirty[ocl] = True
1762         cl.files = Add(cl.files, files)
1763         dirty[cl] = True
1764         for d, _ in dirty.items():
1765                 d.Flush(ui, repo)
1766         return
1767
1768 #######################################################################
1769 # hg gofmt
1770
1771 @hgcommand
1772 def gofmt(ui, repo, *pats, **opts):
1773         """apply gofmt to modified files
1774
1775         Applies gofmt to the modified files in the repository that match
1776         the given patterns.
1777         """
1778         if codereview_disabled:
1779                 return codereview_disabled
1780
1781         files = ChangedExistingFiles(ui, repo, pats, opts)
1782         files = gofmt_required(files)
1783         if not files:
1784                 return "no modified go files"
1785         cwd = os.getcwd()
1786         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1787         try:
1788                 cmd = ["gofmt", "-l"]
1789                 if not opts["list"]:
1790                         cmd += ["-w"]
1791                 if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
1792                         raise hg_util.Abort("gofmt did not exit cleanly")
1793         except hg_error.Abort, e:
1794                 raise
1795         except:
1796                 raise hg_util.Abort("gofmt: " + ExceptionDetail())
1797         return
1798
1799 def gofmt_required(files):
1800         return [f for f in files if (not f.startswith('test/') or f.startswith('test/bench/')) and f.endswith('.go')]
1801
1802 #######################################################################
1803 # hg mail
1804
1805 @hgcommand
1806 def mail(ui, repo, *pats, **opts):
1807         """mail a change for review
1808
1809         Uploads a patch to the code review server and then sends mail
1810         to the reviewer and CC list asking for a review.
1811         """
1812         if codereview_disabled:
1813                 return codereview_disabled
1814
1815         cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1816         if err != "":
1817                 return err
1818         cl.Upload(ui, repo, gofmt_just_warn=True)
1819         if not cl.reviewer:
1820                 # If no reviewer is listed, assign the review to defaultcc.
1821                 # This makes sure that it appears in the
1822                 # codereview.appspot.com/user/defaultcc
1823                 # page, so that it doesn't get dropped on the floor.
1824                 if not defaultcc:
1825                         return "no reviewers listed in CL"
1826                 cl.cc = Sub(cl.cc, defaultcc)
1827                 cl.reviewer = defaultcc
1828                 cl.Flush(ui, repo)
1829
1830         if cl.files == []:
1831                 return "no changed files, not sending mail"
1832
1833         cl.Mail(ui, repo)
1834
1835 #######################################################################
1836 # hg p / hg pq / hg ps / hg pending
1837
1838 @hgcommand
1839 def ps(ui, repo, *pats, **opts):
1840         """alias for hg p --short
1841         """
1842         opts['short'] = True
1843         return pending(ui, repo, *pats, **opts)
1844
1845 @hgcommand
1846 def pq(ui, repo, *pats, **opts):
1847         """alias for hg p --quick
1848         """
1849         opts['quick'] = True
1850         return pending(ui, repo, *pats, **opts)
1851
1852 @hgcommand
1853 def pending(ui, repo, *pats, **opts):
1854         """show pending changes
1855
1856         Lists pending changes followed by a list of unassigned but modified files.
1857         """
1858         if codereview_disabled:
1859                 return codereview_disabled
1860
1861         quick = opts.get('quick', False)
1862         short = opts.get('short', False)
1863         m = LoadAllCL(ui, repo, web=not quick and not short)
1864         names = m.keys()
1865         names.sort()
1866         for name in names:
1867                 cl = m[name]
1868                 if short:
1869                         ui.write(name + "\t" + line1(cl.desc) + "\n")
1870                 else:
1871                         ui.write(cl.PendingText(quick=quick) + "\n")
1872
1873         if short:
1874                 return
1875         files = DefaultFiles(ui, repo, [])
1876         if len(files) > 0:
1877                 s = "Changed files not in any CL:\n"
1878                 for f in files:
1879                         s += "\t" + f + "\n"
1880                 ui.write(s)
1881
1882 #######################################################################
1883 # hg submit
1884
1885 def need_sync():
1886         raise hg_util.Abort("local repository out of date; must sync before submit")
1887
1888 @hgcommand
1889 def submit(ui, repo, *pats, **opts):
1890         """submit change to remote repository
1891
1892         Submits change to remote repository.
1893         Bails out if the local repository is not in sync with the remote one.
1894         """
1895         if codereview_disabled:
1896                 return codereview_disabled
1897
1898         # We already called this on startup but sometimes Mercurial forgets.
1899         set_mercurial_encoding_to_utf8()
1900
1901         if not opts["no_incoming"] and hg_incoming(ui, repo):
1902                 need_sync()
1903
1904         cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1905         if err != "":
1906                 return err
1907
1908         user = None
1909         if cl.copied_from:
1910                 user = cl.copied_from
1911         userline = CheckContributor(ui, repo, user)
1912         typecheck(userline, str)
1913
1914         about = ""
1915         if cl.reviewer:
1916                 about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1917         if opts.get('tbr'):
1918                 tbr = SplitCommaSpace(opts.get('tbr'))
1919                 cl.reviewer = Add(cl.reviewer, tbr)
1920                 about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1921         if cl.cc:
1922                 about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1923
1924         if not cl.reviewer:
1925                 return "no reviewers listed in CL"
1926
1927         if not cl.local:
1928                 return "cannot submit non-local CL"
1929
1930         # upload, to sync current patch and also get change number if CL is new.
1931         if not cl.copied_from:
1932                 cl.Upload(ui, repo, gofmt_just_warn=True)
1933
1934         # check gofmt for real; allowed upload to warn in order to save CL.
1935         cl.Flush(ui, repo)
1936         CheckFormat(ui, repo, cl.files)
1937
1938         about += "%s%s\n" % (server_url_base, cl.name)
1939
1940         if cl.copied_from:
1941                 about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1942         typecheck(about, str)
1943
1944         if not cl.mailed and not cl.copied_from:                # in case this is TBR
1945                 cl.Mail(ui, repo)
1946
1947         # submit changes locally
1948         message = cl.desc.rstrip() + "\n\n" + about
1949         typecheck(message, str)
1950
1951         set_status("pushing " + cl.name + " to remote server")
1952
1953         if hg_outgoing(ui, repo):
1954                 raise hg_util.Abort("local repository corrupt or out-of-phase with remote: found outgoing changes")
1955
1956         old_heads = len(hg_heads(ui, repo).split())
1957
1958         global commit_okay
1959         commit_okay = True
1960         ret = hg_commit(ui, repo, *['path:'+f for f in cl.files], message=message, user=userline)
1961         commit_okay = False
1962         if ret:
1963                 return "nothing changed"
1964         node = repo["-1"].node()
1965         # push to remote; if it fails for any reason, roll back
1966         try:
1967                 new_heads = len(hg_heads(ui, repo).split())
1968                 if old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
1969                         # Created new head, so we weren't up to date.
1970                         need_sync()
1971
1972                 # Push changes to remote.  If it works, we're committed.  If not, roll back.
1973                 try:
1974                         hg_push(ui, repo)
1975                 except hg_error.Abort, e:
1976                         if e.message.find("push creates new heads") >= 0:
1977                                 # Remote repository had changes we missed.
1978                                 need_sync()
1979                         raise
1980         except:
1981                 real_rollback()
1982                 raise
1983
1984         # We're committed. Upload final patch, close review, add commit message.
1985         changeURL = hg_node.short(node)
1986         url = ui.expandpath("default")
1987         m = re.match("(^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?)" + "|" +
1988                 "(^https?://([^@/]+@)?code\.google\.com/p/([^/.]+)(\.[^./]+)?/?)", url)
1989         if m:
1990                 if m.group(1): # prj.googlecode.com/hg/ case
1991                         changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(3), changeURL)
1992                 elif m.group(4) and m.group(7): # code.google.com/p/prj.subrepo/ case
1993                         changeURL = "http://code.google.com/p/%s/source/detail?r=%s&repo=%s" % (m.group(6), changeURL, m.group(7)[1:])
1994                 elif m.group(4): # code.google.com/p/prj/ case
1995                         changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(6), changeURL)
1996                 else:
1997                         print >>sys.stderr, "URL: ", url
1998         else:
1999                 print >>sys.stderr, "URL: ", url
2000         pmsg = "*** Submitted as " + changeURL + " ***\n\n" + message
2001
2002         # When posting, move reviewers to CC line,
2003         # so that the issue stops showing up in their "My Issues" page.
2004         PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
2005
2006         if not cl.copied_from:
2007                 EditDesc(cl.name, closed=True, private=cl.private)
2008         cl.Delete(ui, repo)
2009
2010         c = repo[None]
2011         if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed():
2012                 ui.write("switching from %s to default branch.\n" % releaseBranch)
2013                 err = hg_clean(repo, "default")
2014                 if err:
2015                         return err
2016         return None
2017
2018 #######################################################################
2019 # hg sync
2020
2021 @hgcommand
2022 def sync(ui, repo, **opts):
2023         """synchronize with remote repository
2024
2025         Incorporates recent changes from the remote repository
2026         into the local repository.
2027         """
2028         if codereview_disabled:
2029                 return codereview_disabled
2030
2031         if not opts["local"]:
2032                 err = hg_pull(ui, repo, update=True)
2033                 if err:
2034                         return err
2035         sync_changes(ui, repo)
2036
2037 def sync_changes(ui, repo):
2038         # Look through recent change log descriptions to find
2039         # potential references to http://.*/our-CL-number.
2040         # Double-check them by looking at the Rietveld log.
2041         for rev in hg_log(ui, repo, limit=100, template="{node}\n").split():
2042                 desc = repo[rev].description().strip()
2043                 for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
2044                         if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
2045                                 ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
2046                                 cl, err = LoadCL(ui, repo, clname, web=False)
2047                                 if err != "":
2048                                         ui.warn("loading CL %s: %s\n" % (clname, err))
2049                                         continue
2050                                 if not cl.copied_from:
2051                                         EditDesc(cl.name, closed=True, private=cl.private)
2052                                 cl.Delete(ui, repo)
2053
2054         # Remove files that are not modified from the CLs in which they appear.
2055         all = LoadAllCL(ui, repo, web=False)
2056         changed = ChangedFiles(ui, repo, [])
2057         for cl in all.values():
2058                 extra = Sub(cl.files, changed)
2059                 if extra:
2060                         ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
2061                         for f in extra:
2062                                 ui.warn("\t%s\n" % (f,))
2063                         cl.files = Sub(cl.files, extra)
2064                         cl.Flush(ui, repo)
2065                 if not cl.files:
2066                         if not cl.copied_from:
2067                                 ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name))
2068                         else:
2069                                 ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name))
2070         return
2071
2072 #######################################################################
2073 # hg upload
2074
2075 @hgcommand
2076 def upload(ui, repo, name, **opts):
2077         """upload diffs to the code review server
2078
2079         Uploads the current modifications for a given change to the server.
2080         """
2081         if codereview_disabled:
2082                 return codereview_disabled
2083
2084         repo.ui.quiet = True
2085         cl, err = LoadCL(ui, repo, name, web=True)
2086         if err != "":
2087                 return err
2088         if not cl.local:
2089                 return "cannot upload non-local change"
2090         cl.Upload(ui, repo)
2091         print "%s%s\n" % (server_url_base, cl.name)
2092         return
2093
2094 #######################################################################
2095 # Table of commands, supplied to Mercurial for installation.
2096
2097 review_opts = [
2098         ('r', 'reviewer', '', 'add reviewer'),
2099         ('', 'cc', '', 'add cc'),
2100         ('', 'tbr', '', 'add future reviewer'),
2101         ('m', 'message', '', 'change description (for new change)'),
2102 ]
2103
2104 cmdtable = {
2105         # The ^ means to show this command in the help text that
2106         # is printed when running hg with no arguments.
2107         "^change": (
2108                 change,
2109                 [
2110                         ('d', 'delete', None, 'delete existing change list'),
2111                         ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
2112                         ('i', 'stdin', None, 'read change list from standard input'),
2113                         ('o', 'stdout', None, 'print change list to standard output'),
2114                         ('p', 'pending', None, 'print pending summary to standard output'),
2115                 ],
2116                 "[-d | -D] [-i] [-o] change# or FILE ..."
2117         ),
2118         "^clpatch": (
2119                 clpatch,
2120                 [
2121                         ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2122                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2123                 ],
2124                 "change#"
2125         ),
2126         # Would prefer to call this codereview-login, but then
2127         # hg help codereview prints the help for this command
2128         # instead of the help for the extension.
2129         "code-login": (
2130                 code_login,
2131                 [],
2132                 "",
2133         ),
2134         "^download": (
2135                 download,
2136                 [],
2137                 "change#"
2138         ),
2139         "^file": (
2140                 file,
2141                 [
2142                         ('d', 'delete', None, 'delete files from change list (but not repository)'),
2143                 ],
2144                 "[-d] change# FILE ..."
2145         ),
2146         "^gofmt": (
2147                 gofmt,
2148                 [
2149                         ('l', 'list', None, 'list files that would change, but do not edit them'),
2150                 ],
2151                 "FILE ..."
2152         ),
2153         "^pending|p": (
2154                 pending,
2155                 [
2156                         ('s', 'short', False, 'show short result form'),
2157                         ('', 'quick', False, 'do not consult codereview server'),
2158                 ],
2159                 "[FILE ...]"
2160         ),
2161         "^ps": (
2162                 ps,
2163                 [],
2164                 "[FILE ...]"
2165         ),
2166         "^pq": (
2167                 pq,
2168                 [],
2169                 "[FILE ...]"
2170         ),
2171         "^mail": (
2172                 mail,
2173                 review_opts + [
2174                 ] + hg_commands.walkopts,
2175                 "[-r reviewer] [--cc cc] [change# | file ...]"
2176         ),
2177         "^release-apply": (
2178                 release_apply,
2179                 [
2180                         ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2181                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2182                 ],
2183                 "change#"
2184         ),
2185         # TODO: release-start, release-tag, weekly-tag
2186         "^submit": (
2187                 submit,
2188                 review_opts + [
2189                         ('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
2190                 ] + hg_commands.walkopts + hg_commands.commitopts + hg_commands.commitopts2,
2191                 "[-r reviewer] [--cc cc] [change# | file ...]"
2192         ),
2193         "^sync": (
2194                 sync,
2195                 [
2196                         ('', 'local', None, 'do not pull changes from remote repository')
2197                 ],
2198                 "[--local]",
2199         ),
2200         "^undo": (
2201                 undo,
2202                 [
2203                         ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2204                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2205                 ],
2206                 "change#"
2207         ),
2208         "^upload": (
2209                 upload,
2210                 [],
2211                 "change#"
2212         ),
2213 }
2214
2215 #######################################################################
2216 # Mercurial extension initialization
2217
2218 def norollback(*pats, **opts):
2219         """(disabled when using this extension)"""
2220         raise hg_util.Abort("codereview extension enabled; use undo instead of rollback")
2221
2222 codereview_init = False
2223
2224 def reposetup(ui, repo):
2225         global codereview_disabled
2226         global defaultcc
2227
2228         # reposetup gets called both for the local repository
2229         # and also for any repository we are pulling or pushing to.
2230         # Only initialize the first time.
2231         global codereview_init
2232         if codereview_init:
2233                 return
2234         codereview_init = True
2235
2236         # Read repository-specific options from lib/codereview/codereview.cfg or codereview.cfg.
2237         root = ''
2238         try:
2239                 root = repo.root
2240         except:
2241                 # Yes, repo might not have root; see issue 959.
2242                 codereview_disabled = 'codereview disabled: repository has no root'
2243                 return
2244
2245         repo_config_path = ''
2246         p1 = root + '/lib/codereview/codereview.cfg'
2247         p2 = root + '/codereview.cfg'
2248         if os.access(p1, os.F_OK):
2249                 repo_config_path = p1
2250         else:
2251                 repo_config_path = p2
2252         try:
2253                 f = open(repo_config_path)
2254                 for line in f:
2255                         if line.startswith('defaultcc:'):
2256                                 defaultcc = SplitCommaSpace(line[len('defaultcc:'):])
2257                         if line.startswith('contributors:'):
2258                                 global contributorsURL
2259                                 contributorsURL = line[len('contributors:'):].strip()
2260         except:
2261                 codereview_disabled = 'codereview disabled: cannot open ' + repo_config_path
2262                 return
2263
2264         remote = ui.config("paths", "default", "")
2265         if remote.find("://") < 0:
2266                 raise hg_util.Abort("codereview: default path '%s' is not a URL" % (remote,))
2267
2268         InstallMatch(ui, repo)
2269         RietveldSetup(ui, repo)
2270
2271         # Disable the Mercurial commands that might change the repository.
2272         # Only commands in this extension are supposed to do that.
2273         ui.setconfig("hooks", "precommit.codereview", precommithook)
2274
2275         # Rollback removes an existing commit.  Don't do that either.
2276         global real_rollback
2277         real_rollback = repo.rollback
2278         repo.rollback = norollback
2279
2280
2281 #######################################################################
2282 # Wrappers around upload.py for interacting with Rietveld
2283
2284 from HTMLParser import HTMLParser
2285
2286 # HTML form parser
2287 class FormParser(HTMLParser):
2288         def __init__(self):
2289                 self.map = {}
2290                 self.curtag = None
2291                 self.curdata = None
2292                 HTMLParser.__init__(self)
2293         def handle_starttag(self, tag, attrs):
2294                 if tag == "input":
2295                         key = None
2296                         value = ''
2297                         for a in attrs:
2298                                 if a[0] == 'name':
2299                                         key = a[1]
2300                                 if a[0] == 'value':
2301                                         value = a[1]
2302                         if key is not None:
2303                                 self.map[key] = value
2304                 if tag == "textarea":
2305                         key = None
2306                         for a in attrs:
2307                                 if a[0] == 'name':
2308                                         key = a[1]
2309                         if key is not None:
2310                                 self.curtag = key
2311                                 self.curdata = ''
2312         def handle_endtag(self, tag):
2313                 if tag == "textarea" and self.curtag is not None:
2314                         self.map[self.curtag] = self.curdata
2315                         self.curtag = None
2316                         self.curdata = None
2317         def handle_charref(self, name):
2318                 self.handle_data(unichr(int(name)))
2319         def handle_entityref(self, name):
2320                 import htmlentitydefs
2321                 if name in htmlentitydefs.entitydefs:
2322                         self.handle_data(htmlentitydefs.entitydefs[name])
2323                 else:
2324                         self.handle_data("&" + name + ";")
2325         def handle_data(self, data):
2326                 if self.curdata is not None:
2327                         self.curdata += data
2328
2329 def JSONGet(ui, path):
2330         try:
2331                 data = MySend(path, force_auth=False)
2332                 typecheck(data, str)
2333                 d = fix_json(json.loads(data))
2334         except:
2335                 ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail()))
2336                 return None
2337         return d
2338
2339 # Clean up json parser output to match our expectations:
2340 #   * all strings are UTF-8-encoded str, not unicode.
2341 #   * missing fields are missing, not None,
2342 #     so that d.get("foo", defaultvalue) works.
2343 def fix_json(x):
2344         if type(x) in [str, int, float, bool, type(None)]:
2345                 pass
2346         elif type(x) is unicode:
2347                 x = x.encode("utf-8")
2348         elif type(x) is list:
2349                 for i in range(len(x)):
2350                         x[i] = fix_json(x[i])
2351         elif type(x) is dict:
2352                 todel = []
2353                 for k in x:
2354                         if x[k] is None:
2355                                 todel.append(k)
2356                         else:
2357                                 x[k] = fix_json(x[k])
2358                 for k in todel:
2359                         del x[k]
2360         else:
2361                 raise hg_util.Abort("unknown type " + str(type(x)) + " in fix_json")
2362         if type(x) is str:
2363                 x = x.replace('\r\n', '\n')
2364         return x
2365
2366 def IsRietveldSubmitted(ui, clname, hex):
2367         dict = JSONGet(ui, "/api/" + clname + "?messages=true")
2368         if dict is None:
2369                 return False
2370         for msg in dict.get("messages", []):
2371                 text = msg.get("text", "")
2372                 m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
2373                 if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
2374                         return True
2375         return False
2376
2377 def IsRietveldMailed(cl):
2378         for msg in cl.dict.get("messages", []):
2379                 if msg.get("text", "").find("I'd like you to review this change") >= 0:
2380                         return True
2381         return False
2382
2383 def DownloadCL(ui, repo, clname):
2384         set_status("downloading CL " + clname)
2385         cl, err = LoadCL(ui, repo, clname, web=True)
2386         if err != "":
2387                 return None, None, None, "error loading CL %s: %s" % (clname, err)
2388
2389         # Find most recent diff
2390         diffs = cl.dict.get("patchsets", [])
2391         if not diffs:
2392                 return None, None, None, "CL has no patch sets"
2393         patchid = diffs[-1]
2394
2395         patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid))
2396         if patchset is None:
2397                 return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid)
2398         if patchset.get("patchset", 0) != patchid:
2399                 return None, None, None, "malformed patchset information"
2400
2401         vers = ""
2402         msg = patchset.get("message", "").split()
2403         if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r":
2404                 vers = msg[2]
2405         diff = "/download/issue" + clname + "_" + str(patchid) + ".diff"
2406
2407         diffdata = MySend(diff, force_auth=False)
2408
2409         # Print warning if email is not in CONTRIBUTORS file.
2410         email = cl.dict.get("owner_email", "")
2411         if not email:
2412                 return None, None, None, "cannot find owner for %s" % (clname)
2413         him = FindContributor(ui, repo, email)
2414         me = FindContributor(ui, repo, None)
2415         if him == me:
2416                 cl.mailed = IsRietveldMailed(cl)
2417         else:
2418                 cl.copied_from = email
2419
2420         return cl, vers, diffdata, ""
2421
2422 def MySend(request_path, payload=None,
2423                 content_type="application/octet-stream",
2424                 timeout=None, force_auth=True,
2425                 **kwargs):
2426         """Run MySend1 maybe twice, because Rietveld is unreliable."""
2427         try:
2428                 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2429         except Exception, e:
2430                 if type(e) != urllib2.HTTPError or e.code != 500:       # only retry on HTTP 500 error
2431                         raise
2432                 print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
2433                 time.sleep(2)
2434                 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2435
2436 # Like upload.py Send but only authenticates when the
2437 # redirect is to www.google.com/accounts.  This keeps
2438 # unnecessary redirects from happening during testing.
2439 def MySend1(request_path, payload=None,
2440                                 content_type="application/octet-stream",
2441                                 timeout=None, force_auth=True,
2442                                 **kwargs):
2443         """Sends an RPC and returns the response.
2444
2445         Args:
2446                 request_path: The path to send the request to, eg /api/appversion/create.
2447                 payload: The body of the request, or None to send an empty request.
2448                 content_type: The Content-Type header to use.
2449                 timeout: timeout in seconds; default None i.e. no timeout.
2450                         (Note: for large requests on OS X, the timeout doesn't work right.)
2451                 kwargs: Any keyword arguments are converted into query string parameters.
2452
2453         Returns:
2454                 The response body, as a string.
2455         """
2456         # TODO: Don't require authentication.  Let the server say
2457         # whether it is necessary.
2458         global rpc
2459         if rpc == None:
2460                 rpc = GetRpcServer(upload_options)
2461         self = rpc
2462         if not self.authenticated and force_auth:
2463                 self._Authenticate()
2464         if request_path is None:
2465                 return
2466
2467         old_timeout = socket.getdefaulttimeout()
2468         socket.setdefaulttimeout(timeout)
2469         try:
2470                 tries = 0
2471                 while True:
2472                         tries += 1
2473                         args = dict(kwargs)
2474                         url = "http://%s%s" % (self.host, request_path)
2475                         if args:
2476                                 url += "?" + urllib.urlencode(args)
2477                         req = self._CreateRequest(url=url, data=payload)
2478                         req.add_header("Content-Type", content_type)
2479                         try:
2480                                 f = self.opener.open(req)
2481                                 response = f.read()
2482                                 f.close()
2483                                 # Translate \r\n into \n, because Rietveld doesn't.
2484                                 response = response.replace('\r\n', '\n')
2485                                 # who knows what urllib will give us
2486                                 if type(response) == unicode:
2487                                         response = response.encode("utf-8")
2488                                 typecheck(response, str)
2489                                 return response
2490                         except urllib2.HTTPError, e:
2491                                 if tries > 3:
2492                                         raise
2493                                 elif e.code == 401:
2494                                         self._Authenticate()
2495                                 elif e.code == 302:
2496                                         loc = e.info()["location"]
2497                                         if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
2498                                                 return ''
2499                                         self._Authenticate()
2500                                 else:
2501                                         raise
2502         finally:
2503                 socket.setdefaulttimeout(old_timeout)
2504
2505 def GetForm(url):
2506         f = FormParser()
2507         f.feed(ustr(MySend(url)))       # f.feed wants unicode
2508         f.close()
2509         # convert back to utf-8 to restore sanity
2510         m = {}
2511         for k,v in f.map.items():
2512                 m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8")
2513         return m
2514
2515 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False):
2516         set_status("uploading change to description")
2517         form_fields = GetForm("/" + issue + "/edit")
2518         if subject is not None:
2519                 form_fields['subject'] = subject
2520         if desc is not None:
2521                 form_fields['description'] = desc
2522         if reviewers is not None:
2523                 form_fields['reviewers'] = reviewers
2524         if cc is not None:
2525                 form_fields['cc'] = cc
2526         if closed:
2527                 form_fields['closed'] = "checked"
2528         if private:
2529                 form_fields['private'] = "checked"
2530         ctype, body = EncodeMultipartFormData(form_fields.items(), [])
2531         response = MySend("/" + issue + "/edit", body, content_type=ctype)
2532         if response != "":
2533                 print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
2534                 sys.exit(2)
2535
2536 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
2537         set_status("uploading message")
2538         form_fields = GetForm("/" + issue + "/publish")
2539         if reviewers is not None:
2540                 form_fields['reviewers'] = reviewers
2541         if cc is not None:
2542                 form_fields['cc'] = cc
2543         if send_mail:
2544                 form_fields['send_mail'] = "checked"
2545         else:
2546                 del form_fields['send_mail']
2547         if subject is not None:
2548                 form_fields['subject'] = subject
2549         form_fields['message'] = message
2550
2551         form_fields['message_only'] = '1'       # Don't include draft comments
2552         if reviewers is not None or cc is not None:
2553                 form_fields['message_only'] = ''        # Must set '' in order to override cc/reviewer
2554         ctype = "applications/x-www-form-urlencoded"
2555         body = urllib.urlencode(form_fields)
2556         response = MySend("/" + issue + "/publish", body, content_type=ctype)
2557         if response != "":
2558                 print response
2559                 sys.exit(2)
2560
2561 class opt(object):
2562         pass
2563
2564 def RietveldSetup(ui, repo):
2565         global force_google_account
2566         global rpc
2567         global server
2568         global server_url_base
2569         global upload_options
2570         global verbosity
2571
2572         if not ui.verbose:
2573                 verbosity = 0
2574
2575         # Config options.
2576         x = ui.config("codereview", "server")
2577         if x is not None:
2578                 server = x
2579
2580         # TODO(rsc): Take from ui.username?
2581         email = None
2582         x = ui.config("codereview", "email")
2583         if x is not None:
2584                 email = x
2585
2586         server_url_base = "http://" + server + "/"
2587
2588         testing = ui.config("codereview", "testing")
2589         force_google_account = ui.configbool("codereview", "force_google_account", False)
2590
2591         upload_options = opt()
2592         upload_options.email = email
2593         upload_options.host = None
2594         upload_options.verbose = 0
2595         upload_options.description = None
2596         upload_options.description_file = None
2597         upload_options.reviewers = None
2598         upload_options.cc = None
2599         upload_options.message = None
2600         upload_options.issue = None
2601         upload_options.download_base = False
2602         upload_options.revision = None
2603         upload_options.send_mail = False
2604         upload_options.vcs = None
2605         upload_options.server = server
2606         upload_options.save_cookies = True
2607
2608         if testing:
2609                 upload_options.save_cookies = False
2610                 upload_options.email = "test@example.com"
2611
2612         rpc = None
2613
2614         global releaseBranch
2615         tags = repo.branchmap().keys()
2616         if 'release-branch.go10' in tags:
2617                 # NOTE(rsc): This tags.sort is going to get the wrong
2618                 # answer when comparing release-branch.go9 with
2619                 # release-branch.go10.  It will be a while before we care.
2620                 raise hg_util.Abort('tags.sort needs to be fixed for release-branch.go10')
2621         tags.sort()
2622         for t in tags:
2623                 if t.startswith('release-branch.go'):
2624                         releaseBranch = t
2625
2626 #######################################################################
2627 # http://codereview.appspot.com/static/upload.py, heavily edited.
2628
2629 #!/usr/bin/env python
2630 #
2631 # Copyright 2007 Google Inc.
2632 #
2633 # Licensed under the Apache License, Version 2.0 (the "License");
2634 # you may not use this file except in compliance with the License.
2635 # You may obtain a copy of the License at
2636 #
2637 #       http://www.apache.org/licenses/LICENSE-2.0
2638 #
2639 # Unless required by applicable law or agreed to in writing, software
2640 # distributed under the License is distributed on an "AS IS" BASIS,
2641 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2642 # See the License for the specific language governing permissions and
2643 # limitations under the License.
2644
2645 """Tool for uploading diffs from a version control system to the codereview app.
2646
2647 Usage summary: upload.py [options] [-- diff_options]
2648
2649 Diff options are passed to the diff command of the underlying system.
2650
2651 Supported version control systems:
2652         Git
2653         Mercurial
2654         Subversion
2655
2656 It is important for Git/Mercurial users to specify a tree/node/branch to diff
2657 against by using the '--rev' option.
2658 """
2659 # This code is derived from appcfg.py in the App Engine SDK (open source),
2660 # and from ASPN recipe #146306.
2661
2662 import cookielib
2663 import getpass
2664 import logging
2665 import mimetypes
2666 import optparse
2667 import os
2668 import re
2669 import socket
2670 import subprocess
2671 import sys
2672 import urllib
2673 import urllib2
2674 import urlparse
2675
2676 # The md5 module was deprecated in Python 2.5.
2677 try:
2678         from hashlib import md5
2679 except ImportError:
2680         from md5 import md5
2681
2682 try:
2683         import readline
2684 except ImportError:
2685         pass
2686
2687 # The logging verbosity:
2688 #  0: Errors only.
2689 #  1: Status messages.
2690 #  2: Info logs.
2691 #  3: Debug logs.
2692 verbosity = 1
2693
2694 # Max size of patch or base file.
2695 MAX_UPLOAD_SIZE = 900 * 1024
2696
2697 # whitelist for non-binary filetypes which do not start with "text/"
2698 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
2699 TEXT_MIMETYPES = [
2700         'application/javascript',
2701         'application/x-javascript',
2702         'application/x-freemind'
2703 ]
2704
2705 def GetEmail(prompt):
2706         """Prompts the user for their email address and returns it.
2707
2708         The last used email address is saved to a file and offered up as a suggestion
2709         to the user. If the user presses enter without typing in anything the last
2710         used email address is used. If the user enters a new address, it is saved
2711         for next time we prompt.
2712
2713         """
2714         last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
2715         last_email = ""
2716         if os.path.exists(last_email_file_name):
2717                 try:
2718                         last_email_file = open(last_email_file_name, "r")
2719                         last_email = last_email_file.readline().strip("\n")
2720                         last_email_file.close()
2721                         prompt += " [%s]" % last_email
2722                 except IOError, e:
2723                         pass
2724         email = raw_input(prompt + ": ").strip()
2725         if email:
2726                 try:
2727                         last_email_file = open(last_email_file_name, "w")
2728                         last_email_file.write(email)
2729                         last_email_file.close()
2730                 except IOError, e:
2731                         pass
2732         else:
2733                 email = last_email
2734         return email
2735
2736
2737 def StatusUpdate(msg):
2738         """Print a status message to stdout.
2739
2740         If 'verbosity' is greater than 0, print the message.
2741
2742         Args:
2743                 msg: The string to print.
2744         """
2745         if verbosity > 0:
2746                 print msg
2747
2748
2749 def ErrorExit(msg):
2750         """Print an error message to stderr and exit."""
2751         print >>sys.stderr, msg
2752         sys.exit(1)
2753
2754
2755 class ClientLoginError(urllib2.HTTPError):
2756         """Raised to indicate there was an error authenticating with ClientLogin."""
2757
2758         def __init__(self, url, code, msg, headers, args):
2759                 urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
2760                 self.args = args
2761                 self.reason = args["Error"]
2762
2763
2764 class AbstractRpcServer(object):
2765         """Provides a common interface for a simple RPC server."""
2766
2767         def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False):
2768                 """Creates a new HttpRpcServer.
2769
2770                 Args:
2771                         host: The host to send requests to.
2772                         auth_function: A function that takes no arguments and returns an
2773                                 (email, password) tuple when called. Will be called if authentication
2774                                 is required.
2775                         host_override: The host header to send to the server (defaults to host).
2776                         extra_headers: A dict of extra headers to append to every request.
2777                         save_cookies: If True, save the authentication cookies to local disk.
2778                                 If False, use an in-memory cookiejar instead.  Subclasses must
2779                                 implement this functionality.  Defaults to False.
2780                 """
2781                 self.host = host
2782                 self.host_override = host_override
2783                 self.auth_function = auth_function
2784                 self.authenticated = False
2785                 self.extra_headers = extra_headers
2786                 self.save_cookies = save_cookies
2787                 self.opener = self._GetOpener()
2788                 if self.host_override:
2789                         logging.info("Server: %s; Host: %s", self.host, self.host_override)
2790                 else:
2791                         logging.info("Server: %s", self.host)
2792
2793         def _GetOpener(self):
2794                 """Returns an OpenerDirector for making HTTP requests.
2795
2796                 Returns:
2797                         A urllib2.OpenerDirector object.
2798                 """
2799                 raise NotImplementedError()
2800
2801         def _CreateRequest(self, url, data=None):
2802                 """Creates a new urllib request."""
2803                 logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
2804                 req = urllib2.Request(url, data=data)
2805                 if self.host_override:
2806                         req.add_header("Host", self.host_override)
2807                 for key, value in self.extra_headers.iteritems():
2808                         req.add_header(key, value)
2809                 return req
2810
2811         def _GetAuthToken(self, email, password):
2812                 """Uses ClientLogin to authenticate the user, returning an auth token.
2813
2814                 Args:
2815                         email:    The user's email address
2816                         password: The user's password
2817
2818                 Raises:
2819                         ClientLoginError: If there was an error authenticating with ClientLogin.
2820                         HTTPError: If there was some other form of HTTP error.
2821
2822                 Returns:
2823                         The authentication token returned by ClientLogin.
2824                 """
2825                 account_type = "GOOGLE"
2826                 if self.host.endswith(".google.com") and not force_google_account:
2827                         # Needed for use inside Google.
2828                         account_type = "HOSTED"
2829                 req = self._CreateRequest(
2830                                 url="https://www.google.com/accounts/ClientLogin",
2831                                 data=urllib.urlencode({
2832                                                 "Email": email,
2833                                                 "Passwd": password,
2834                                                 "service": "ah",
2835                                                 "source": "rietveld-codereview-upload",
2836                                                 "accountType": account_type,
2837                                 }),
2838                 )
2839                 try:
2840                         response = self.opener.open(req)
2841                         response_body = response.read()
2842                         response_dict = dict(x.split("=") for x in response_body.split("\n") if x)
2843                         return response_dict["Auth"]
2844                 except urllib2.HTTPError, e:
2845                         if e.code == 403:
2846                                 body = e.read()
2847                                 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
2848                                 raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict)
2849                         else:
2850                                 raise
2851
2852         def _GetAuthCookie(self, auth_token):
2853                 """Fetches authentication cookies for an authentication token.
2854
2855                 Args:
2856                         auth_token: The authentication token returned by ClientLogin.
2857
2858                 Raises:
2859                         HTTPError: If there was an error fetching the authentication cookies.
2860                 """
2861                 # This is a dummy value to allow us to identify when we're successful.
2862                 continue_location = "http://localhost/"
2863                 args = {"continue": continue_location, "auth": auth_token}
2864                 req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args)))
2865                 try:
2866                         response = self.opener.open(req)
2867                 except urllib2.HTTPError, e:
2868                         response = e
2869                 if (response.code != 302 or
2870                                 response.info()["location"] != continue_location):
2871                         raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp)
2872                 self.authenticated = True
2873
2874         def _Authenticate(self):
2875                 """Authenticates the user.
2876
2877                 The authentication process works as follows:
2878                 1) We get a username and password from the user
2879                 2) We use ClientLogin to obtain an AUTH token for the user
2880                                 (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
2881                 3) We pass the auth token to /_ah/login on the server to obtain an
2882                                 authentication cookie. If login was successful, it tries to redirect
2883                                 us to the URL we provided.
2884
2885                 If we attempt to access the upload API without first obtaining an
2886                 authentication cookie, it returns a 401 response (or a 302) and
2887                 directs us to authenticate ourselves with ClientLogin.
2888                 """
2889                 for i in range(3):
2890                         credentials = self.auth_function()
2891                         try:
2892                                 auth_token = self._GetAuthToken(credentials[0], credentials[1])
2893                         except ClientLoginError, e:
2894                                 if e.reason == "BadAuthentication":
2895                                         print >>sys.stderr, "Invalid username or password."
2896                                         continue
2897                                 if e.reason == "CaptchaRequired":
2898                                         print >>sys.stderr, (
2899                                                 "Please go to\n"
2900                                                 "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2901                                                 "and verify you are a human.  Then try again.")
2902                                         break
2903                                 if e.reason == "NotVerified":
2904                                         print >>sys.stderr, "Account not verified."
2905                                         break
2906                                 if e.reason == "TermsNotAgreed":
2907                                         print >>sys.stderr, "User has not agreed to TOS."
2908                                         break
2909                                 if e.reason == "AccountDeleted":
2910                                         print >>sys.stderr, "The user account has been deleted."
2911                                         break
2912                                 if e.reason == "AccountDisabled":
2913                                         print >>sys.stderr, "The user account has been disabled."
2914                                         break
2915                                 if e.reason == "ServiceDisabled":
2916                                         print >>sys.stderr, "The user's access to the service has been disabled."
2917                                         break
2918                                 if e.reason == "ServiceUnavailable":
2919                                         print >>sys.stderr, "The service is not available; try again later."
2920                                         break
2921                                 raise
2922                         self._GetAuthCookie(auth_token)
2923                         return
2924
2925         def Send(self, request_path, payload=None,
2926                                         content_type="application/octet-stream",
2927                                         timeout=None,
2928                                         **kwargs):
2929                 """Sends an RPC and returns the response.
2930
2931                 Args:
2932                         request_path: The path to send the request to, eg /api/appversion/create.
2933                         payload: The body of the request, or None to send an empty request.
2934                         content_type: The Content-Type header to use.
2935                         timeout: timeout in seconds; default None i.e. no timeout.
2936                                 (Note: for large requests on OS X, the timeout doesn't work right.)
2937                         kwargs: Any keyword arguments are converted into query string parameters.
2938
2939                 Returns:
2940                         The response body, as a string.
2941                 """
2942                 # TODO: Don't require authentication.  Let the server say
2943                 # whether it is necessary.
2944                 if not self.authenticated:
2945                         self._Authenticate()
2946
2947                 old_timeout = socket.getdefaulttimeout()
2948                 socket.setdefaulttimeout(timeout)
2949                 try:
2950                         tries = 0
2951                         while True:
2952                                 tries += 1
2953                                 args = dict(kwargs)
2954                                 url = "http://%s%s" % (self.host, request_path)
2955                                 if args:
2956                                         url += "?" + urllib.urlencode(args)
2957                                 req = self._CreateRequest(url=url, data=payload)
2958                                 req.add_header("Content-Type", content_type)
2959                                 try:
2960                                         f = self.opener.open(req)
2961                                         response = f.read()
2962                                         f.close()
2963                                         return response
2964                                 except urllib2.HTTPError, e:
2965                                         if tries > 3:
2966                                                 raise
2967                                         elif e.code == 401 or e.code == 302:
2968                                                 self._Authenticate()
2969                                         else:
2970                                                 raise
2971                 finally:
2972                         socket.setdefaulttimeout(old_timeout)
2973
2974
2975 class HttpRpcServer(AbstractRpcServer):
2976         """Provides a simplified RPC-style interface for HTTP requests."""
2977
2978         def _Authenticate(self):
2979                 """Save the cookie jar after authentication."""
2980                 super(HttpRpcServer, self)._Authenticate()
2981                 if self.save_cookies:
2982                         StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2983                         self.cookie_jar.save()
2984
2985         def _GetOpener(self):
2986                 """Returns an OpenerDirector that supports cookies and ignores redirects.
2987
2988                 Returns:
2989                         A urllib2.OpenerDirector object.
2990                 """
2991                 opener = urllib2.OpenerDirector()
2992                 opener.add_handler(urllib2.ProxyHandler())
2993                 opener.add_handler(urllib2.UnknownHandler())
2994                 opener.add_handler(urllib2.HTTPHandler())
2995                 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2996                 opener.add_handler(urllib2.HTTPSHandler())
2997                 opener.add_handler(urllib2.HTTPErrorProcessor())
2998                 if self.save_cookies:
2999                         self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
3000                         self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
3001                         if os.path.exists(self.cookie_file):
3002                                 try:
3003                                         self.cookie_jar.load()
3004                                         self.authenticated = True
3005                                         StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file)
3006                                 except (cookielib.LoadError, IOError):
3007                                         # Failed to load cookies - just ignore them.
3008                                         pass
3009                         else:
3010                                 # Create an empty cookie file with mode 600
3011                                 fd = os.open(self.cookie_file, os.O_CREAT, 0600)
3012                                 os.close(fd)
3013                         # Always chmod the cookie file
3014                         os.chmod(self.cookie_file, 0600)
3015                 else:
3016                         # Don't save cookies across runs of update.py.
3017                         self.cookie_jar = cookielib.CookieJar()
3018                 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
3019                 return opener
3020
3021
3022 def GetRpcServer(options):
3023         """Returns an instance of an AbstractRpcServer.
3024
3025         Returns:
3026                 A new AbstractRpcServer, on which RPC calls can be made.
3027         """
3028
3029         rpc_server_class = HttpRpcServer
3030
3031         def GetUserCredentials():
3032                 """Prompts the user for a username and password."""
3033                 # Disable status prints so they don't obscure the password prompt.
3034                 global global_status
3035                 st = global_status
3036                 global_status = None
3037
3038                 email = options.email
3039                 if email is None:
3040                         email = GetEmail("Email (login for uploading to %s)" % options.server)
3041                 password = getpass.getpass("Password for %s: " % email)
3042
3043                 # Put status back.
3044                 global_status = st
3045                 return (email, password)
3046
3047         # If this is the dev_appserver, use fake authentication.
3048         host = (options.host or options.server).lower()
3049         if host == "localhost" or host.startswith("localhost:"):
3050                 email = options.email
3051                 if email is None:
3052                         email = "test@example.com"
3053                         logging.info("Using debug user %s.  Override with --email" % email)
3054                 server = rpc_server_class(
3055                                 options.server,
3056                                 lambda: (email, "password"),
3057                                 host_override=options.host,
3058                                 extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
3059                                 save_cookies=options.save_cookies)
3060                 # Don't try to talk to ClientLogin.
3061                 server.authenticated = True
3062                 return server
3063
3064         return rpc_server_class(options.server, GetUserCredentials,
3065                 host_override=options.host, save_cookies=options.save_cookies)
3066
3067
3068 def EncodeMultipartFormData(fields, files):
3069         """Encode form fields for multipart/form-data.
3070
3071         Args:
3072                 fields: A sequence of (name, value) elements for regular form fields.
3073                 files: A sequence of (name, filename, value) elements for data to be
3074                                         uploaded as files.
3075         Returns:
3076                 (content_type, body) ready for httplib.HTTP instance.
3077
3078         Source:
3079                 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
3080         """
3081         BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
3082         CRLF = '\r\n'
3083         lines = []
3084         for (key, value) in fields:
3085                 typecheck(key, str)
3086                 typecheck(value, str)
3087                 lines.append('--' + BOUNDARY)
3088                 lines.append('Content-Disposition: form-data; name="%s"' % key)
3089                 lines.append('')
3090                 lines.append(value)
3091         for (key, filename, value) in files:
3092                 typecheck(key, str)
3093                 typecheck(filename, str)
3094                 typecheck(value, str)
3095                 lines.append('--' + BOUNDARY)
3096                 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
3097                 lines.append('Content-Type: %s' % GetContentType(filename))
3098                 lines.append('')
3099                 lines.append(value)
3100         lines.append('--' + BOUNDARY + '--')
3101         lines.append('')
3102         body = CRLF.join(lines)
3103         content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
3104         return content_type, body
3105
3106
3107 def GetContentType(filename):
3108         """Helper to guess the content-type from the filename."""
3109         return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
3110
3111
3112 # Use a shell for subcommands on Windows to get a PATH search.
3113 use_shell = sys.platform.startswith("win")
3114
3115 def RunShellWithReturnCode(command, print_output=False,
3116                 universal_newlines=True, env=os.environ):
3117         """Executes a command and returns the output from stdout and the return code.
3118
3119         Args:
3120                 command: Command to execute.
3121                 print_output: If True, the output is printed to stdout.
3122                         If False, both stdout and stderr are ignored.
3123                 universal_newlines: Use universal_newlines flag (default: True).
3124
3125         Returns:
3126                 Tuple (output, return code)
3127         """
3128         logging.info("Running %s", command)
3129         p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
3130                 shell=use_shell, universal_newlines=universal_newlines, env=env)
3131         if print_output:
3132                 output_array = []
3133                 while True:
3134                         line = p.stdout.readline()
3135                         if not line:
3136                                 break
3137                         print line.strip("\n")
3138                         output_array.append(line)
3139                 output = "".join(output_array)
3140         else:
3141                 output = p.stdout.read()
3142         p.wait()
3143         errout = p.stderr.read()
3144         if print_output and errout:
3145                 print >>sys.stderr, errout
3146         p.stdout.close()
3147         p.stderr.close()
3148         return output, p.returncode
3149
3150
3151 def RunShell(command, silent_ok=False, universal_newlines=True,
3152                 print_output=False, env=os.environ):
3153         data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env)
3154         if retcode:
3155                 ErrorExit("Got error status from %s:\n%s" % (command, data))
3156         if not silent_ok and not data:
3157                 ErrorExit("No output from %s" % command)
3158         return data
3159
3160
3161 class VersionControlSystem(object):
3162         """Abstract base class providing an interface to the VCS."""
3163
3164         def __init__(self, options):
3165                 """Constructor.
3166
3167                 Args:
3168                         options: Command line options.
3169                 """
3170                 self.options = options
3171
3172         def GenerateDiff(self, args):
3173                 """Return the current diff as a string.
3174
3175                 Args:
3176                         args: Extra arguments to pass to the diff command.
3177                 """
3178                 raise NotImplementedError(
3179                                 "abstract method -- subclass %s must override" % self.__class__)
3180
3181         def GetUnknownFiles(self):
3182                 """Return a list of files unknown to the VCS."""
3183                 raise NotImplementedError(
3184                                 "abstract method -- subclass %s must override" % self.__class__)
3185
3186         def CheckForUnknownFiles(self):
3187                 """Show an "are you sure?" prompt if there are unknown files."""
3188                 unknown_files = self.GetUnknownFiles()
3189                 if unknown_files:
3190                         print "The following files are not added to version control:"
3191                         for line in unknown_files:
3192                                 print line
3193                         prompt = "Are you sure to continue?(y/N) "
3194                         answer = raw_input(prompt).strip()
3195                         if answer != "y":
3196                                 ErrorExit("User aborted")
3197
3198         def GetBaseFile(self, filename):
3199                 """Get the content of the upstream version of a file.
3200
3201                 Returns:
3202                         A tuple (base_content, new_content, is_binary, status)
3203                                 base_content: The contents of the base file.
3204                                 new_content: For text files, this is empty.  For binary files, this is
3205                                         the contents of the new file, since the diff output won't contain
3206                                         information to reconstruct the current file.
3207                                 is_binary: True iff the file is binary.
3208                                 status: The status of the file.
3209                 """
3210
3211                 raise NotImplementedError(
3212                                 "abstract method -- subclass %s must override" % self.__class__)
3213
3214
3215         def GetBaseFiles(self, diff):
3216                 """Helper that calls GetBase file for each file in the patch.
3217
3218                 Returns:
3219                         A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
3220                         are retrieved based on lines that start with "Index:" or
3221                         "Property changes on:".
3222                 """
3223                 files = {}
3224                 for line in diff.splitlines(True):
3225                         if line.startswith('Index:') or line.startswith('Property changes on:'):
3226                                 unused, filename = line.split(':', 1)
3227                                 # On Windows if a file has property changes its filename uses '\'
3228                                 # instead of '/'.
3229                                 filename = to_slash(filename.strip())
3230                                 files[filename] = self.GetBaseFile(filename)
3231                 return files
3232
3233
3234         def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
3235                                                                                         files):
3236                 """Uploads the base files (and if necessary, the current ones as well)."""
3237
3238                 def UploadFile(filename, file_id, content, is_binary, status, is_base):
3239                         """Uploads a file to the server."""
3240                         set_status("uploading " + filename)
3241                         file_too_large = False
3242                         if is_base:
3243                                 type = "base"
3244                         else:
3245                                 type = "current"
3246                         if len(content) > MAX_UPLOAD_SIZE:
3247                                 print ("Not uploading the %s file for %s because it's too large." %
3248                                                         (type, filename))
3249                                 file_too_large = True
3250                                 content = ""
3251                         checksum = md5(content).hexdigest()
3252                         if options.verbose > 0 and not file_too_large:
3253                                 print "Uploading %s file for %s" % (type, filename)
3254                         url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
3255                         form_fields = [
3256                                 ("filename", filename),
3257                                 ("status", status),
3258                                 ("checksum", checksum),
3259                                 ("is_binary", str(is_binary)),
3260                                 ("is_current", str(not is_base)),
3261                         ]
3262                         if file_too_large:
3263                                 form_fields.append(("file_too_large", "1"))
3264                         if options.email:
3265                                 form_fields.append(("user", options.email))
3266                         ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)])
3267                         response_body = rpc_server.Send(url, body, content_type=ctype)
3268                         if not response_body.startswith("OK"):
3269                                 StatusUpdate("  --> %s" % response_body)
3270                                 sys.exit(1)
3271
3272                 # Don't want to spawn too many threads, nor do we want to
3273                 # hit Rietveld too hard, or it will start serving 500 errors.
3274                 # When 8 works, it's no better than 4, and sometimes 8 is
3275                 # too many for Rietveld to handle.
3276                 MAX_PARALLEL_UPLOADS = 4
3277
3278                 sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS)
3279                 upload_threads = []
3280                 finished_upload_threads = []
3281
3282                 class UploadFileThread(threading.Thread):
3283                         def __init__(self, args):
3284                                 threading.Thread.__init__(self)
3285                                 self.args = args
3286                         def run(self):
3287                                 UploadFile(*self.args)
3288                                 finished_upload_threads.append(self)
3289                                 sema.release()
3290
3291                 def StartUploadFile(*args):
3292                         sema.acquire()
3293                         while len(finished_upload_threads) > 0:
3294                                 t = finished_upload_threads.pop()
3295                                 upload_threads.remove(t)
3296                                 t.join()
3297                         t = UploadFileThread(args)
3298                         upload_threads.append(t)
3299                         t.start()
3300
3301                 def WaitForUploads():
3302                         for t in upload_threads:
3303                                 t.join()
3304
3305                 patches = dict()
3306                 [patches.setdefault(v, k) for k, v in patch_list]
3307                 for filename in patches.keys():
3308                         base_content, new_content, is_binary, status = files[filename]
3309                         file_id_str = patches.get(filename)
3310                         if file_id_str.find("nobase") != -1:
3311                                 base_content = None
3312                                 file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
3313                         file_id = int(file_id_str)
3314                         if base_content != None:
3315                                 StartUploadFile(filename, file_id, base_content, is_binary, status, True)
3316                         if new_content != None:
3317                                 StartUploadFile(filename, file_id, new_content, is_binary, status, False)
3318                 WaitForUploads()
3319
3320         def IsImage(self, filename):
3321                 """Returns true if the filename has an image extension."""
3322                 mimetype =  mimetypes.guess_type(filename)[0]
3323                 if not mimetype:
3324                         return False
3325                 return mimetype.startswith("image/")
3326
3327         def IsBinary(self, filename):
3328                 """Returns true if the guessed mimetyped isnt't in text group."""
3329                 mimetype = mimetypes.guess_type(filename)[0]
3330                 if not mimetype:
3331                         return False  # e.g. README, "real" binaries usually have an extension
3332                 # special case for text files which don't start with text/
3333                 if mimetype in TEXT_MIMETYPES:
3334                         return False
3335                 return not mimetype.startswith("text/")
3336
3337
3338 class FakeMercurialUI(object):
3339         def __init__(self):
3340                 self.quiet = True
3341                 self.output = ''
3342
3343         def write(self, *args, **opts):
3344                 self.output += ' '.join(args)
3345         def copy(self):
3346                 return self
3347         def status(self, *args, **opts):
3348                 pass
3349
3350         def formatter(self, topic, opts):
3351                 from mercurial.formatter import plainformatter
3352                 return plainformatter(self, topic, opts)
3353
3354         def readconfig(self, *args, **opts):
3355                 pass
3356         def expandpath(self, *args, **opts):
3357                 return global_ui.expandpath(*args, **opts)
3358         def configitems(self, *args, **opts):
3359                 return global_ui.configitems(*args, **opts)
3360         def config(self, *args, **opts):
3361                 return global_ui.config(*args, **opts)
3362
3363 use_hg_shell = False    # set to True to shell out to hg always; slower
3364
3365 class MercurialVCS(VersionControlSystem):
3366         """Implementation of the VersionControlSystem interface for Mercurial."""
3367
3368         def __init__(self, options, ui, repo):
3369                 super(MercurialVCS, self).__init__(options)
3370                 self.ui = ui
3371                 self.repo = repo
3372                 self.status = None
3373                 # Absolute path to repository (we can be in a subdir)
3374                 self.repo_dir = os.path.normpath(repo.root)
3375                 # Compute the subdir
3376                 cwd = os.path.normpath(os.getcwd())
3377                 assert cwd.startswith(self.repo_dir)
3378                 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
3379                 if self.options.revision:
3380                         self.base_rev = self.options.revision
3381                 else:
3382                         mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
3383                         if not err and mqparent != "":
3384                                 self.base_rev = mqparent
3385                         else:
3386                                 out = RunShell(["hg", "parents", "-q"], silent_ok=True).strip()
3387                                 if not out:
3388                                         # No revisions; use 0 to mean a repository with nothing.
3389                                         out = "0:0"
3390                                 self.base_rev = out.split(':')[1].strip()
3391         def _GetRelPath(self, filename):
3392                 """Get relative path of a file according to the current directory,
3393                 given its logical path in the repo."""
3394                 assert filename.startswith(self.subdir), (filename, self.subdir)
3395                 return filename[len(self.subdir):].lstrip(r"\/")
3396
3397         def GenerateDiff(self, extra_args):
3398                 # If no file specified, restrict to the current subdir
3399                 extra_args = extra_args or ["."]
3400                 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
3401                 data = RunShell(cmd, silent_ok=True)
3402                 svndiff = []
3403                 filecount = 0
3404                 for line in data.splitlines():
3405                         m = re.match("diff --git a/(\S+) b/(\S+)", line)
3406                         if m:
3407                                 # Modify line to make it look like as it comes from svn diff.
3408                                 # With this modification no changes on the server side are required
3409                                 # to make upload.py work with Mercurial repos.
3410                                 # NOTE: for proper handling of moved/copied files, we have to use
3411                                 # the second filename.
3412                                 filename = m.group(2)
3413                                 svndiff.append("Index: %s" % filename)
3414                                 svndiff.append("=" * 67)
3415                                 filecount += 1
3416                                 logging.info(line)
3417                         else:
3418                                 svndiff.append(line)
3419                 if not filecount:
3420                         ErrorExit("No valid patches found in output from hg diff")
3421                 return "\n".join(svndiff) + "\n"
3422
3423         def GetUnknownFiles(self):
3424                 """Return a list of files unknown to the VCS."""
3425                 args = []
3426                 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
3427                                 silent_ok=True)
3428                 unknown_files = []
3429                 for line in status.splitlines():
3430                         st, fn = line.split(" ", 1)
3431                         if st == "?":
3432                                 unknown_files.append(fn)
3433                 return unknown_files
3434
3435         def get_hg_status(self, rev, path):
3436                 # We'd like to use 'hg status -C path', but that is buggy
3437                 # (see http://mercurial.selenic.com/bts/issue3023).
3438                 # Instead, run 'hg status -C' without a path
3439                 # and skim the output for the path we want.
3440                 if self.status is None:
3441                         if use_hg_shell:
3442                                 out = RunShell(["hg", "status", "-C", "--rev", rev])
3443                         else:
3444                                 fui = FakeMercurialUI()
3445                                 ret = hg_commands.status(fui, self.repo, *[], **{'rev': [rev], 'copies': True})
3446                                 if ret:
3447                                         raise hg_util.Abort(ret)
3448                                 out = fui.output
3449                         self.status = out.splitlines()
3450                 for i in range(len(self.status)):
3451                         # line is
3452                         #       A path
3453                         #       M path
3454                         # etc
3455                         line = to_slash(self.status[i])
3456                         if line[2:] == path:
3457                                 if i+1 < len(self.status) and self.status[i+1][:2] == '  ':
3458                                         return self.status[i:i+2]
3459                                 return self.status[i:i+1]
3460                 raise hg_util.Abort("no status for " + path)
3461
3462         def GetBaseFile(self, filename):
3463                 set_status("inspecting " + filename)
3464                 # "hg status" and "hg cat" both take a path relative to the current subdir
3465                 # rather than to the repo root, but "hg diff" has given us the full path
3466                 # to the repo root.
3467                 base_content = ""
3468                 new_content = None
3469                 is_binary = False
3470                 oldrelpath = relpath = self._GetRelPath(filename)
3471                 out = self.get_hg_status(self.base_rev, relpath)
3472                 status, what = out[0].split(' ', 1)
3473                 if len(out) > 1 and status == "A" and what == relpath:
3474                         oldrelpath = out[1].strip()
3475                         status = "M"
3476                 if ":" in self.base_rev:
3477                         base_rev = self.base_rev.split(":", 1)[0]
3478                 else:
3479                         base_rev = self.base_rev
3480                 if status != "A":
3481                         if use_hg_shell:
3482                                 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True)
3483                         else:
3484                                 base_content = str(self.repo[base_rev][oldrelpath].data())
3485                         is_binary = "\0" in base_content  # Mercurial's heuristic
3486                 if status != "R":
3487                         new_content = open(relpath, "rb").read()
3488                         is_binary = is_binary or "\0" in new_content
3489                 if is_binary and base_content and use_hg_shell:
3490                         # Fetch again without converting newlines
3491                         base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
3492                                 silent_ok=True, universal_newlines=False)
3493                 if not is_binary or not self.IsImage(relpath):
3494                         new_content = None
3495                 return base_content, new_content, is_binary, status
3496
3497
3498 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
3499 def SplitPatch(data):
3500         """Splits a patch into separate pieces for each file.
3501
3502         Args:
3503                 data: A string containing the output of svn diff.
3504
3505         Returns:
3506                 A list of 2-tuple (filename, text) where text is the svn diff output
3507                         pertaining to filename.
3508         """
3509         patches = []
3510         filename = None
3511         diff = []
3512         for line in data.splitlines(True):
3513                 new_filename = None
3514                 if line.startswith('Index:'):
3515                         unused, new_filename = line.split(':', 1)
3516                         new_filename = new_filename.strip()
3517                 elif line.startswith('Property changes on:'):
3518                         unused, temp_filename = line.split(':', 1)
3519                         # When a file is modified, paths use '/' between directories, however
3520                         # when a property is modified '\' is used on Windows.  Make them the same
3521                         # otherwise the file shows up twice.
3522                         temp_filename = to_slash(temp_filename.strip())
3523                         if temp_filename != filename:
3524                                 # File has property changes but no modifications, create a new diff.
3525                                 new_filename = temp_filename
3526                 if new_filename:
3527                         if filename and diff:
3528                                 patches.append((filename, ''.join(diff)))
3529                         filename = new_filename
3530                         diff = [line]
3531                         continue
3532                 if diff is not None:
3533                         diff.append(line)
3534         if filename and diff:
3535                 patches.append((filename, ''.join(diff)))
3536         return patches
3537
3538
3539 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3540         """Uploads a separate patch for each file in the diff output.
3541
3542         Returns a list of [patch_key, filename] for each file.
3543         """
3544         patches = SplitPatch(data)
3545         rv = []
3546         for patch in patches:
3547                 set_status("uploading patch for " + patch[0])
3548                 if len(patch[1]) > MAX_UPLOAD_SIZE:
3549                         print ("Not uploading the patch for " + patch[0] +
3550                                 " because the file is too large.")
3551                         continue
3552                 form_fields = [("filename", patch[0])]
3553                 if not options.download_base:
3554                         form_fields.append(("content_upload", "1"))
3555                 files = [("data", "data.diff", patch[1])]
3556                 ctype, body = EncodeMultipartFormData(form_fields, files)
3557                 url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3558                 print "Uploading patch for " + patch[0]
3559                 response_body = rpc_server.Send(url, body, content_type=ctype)
3560                 lines = response_body.splitlines()
3561                 if not lines or lines[0] != "OK":
3562                         StatusUpdate("  --> %s" % response_body)
3563                         sys.exit(1)
3564                 rv.append([lines[1], patch[0]])
3565         return rv