[SyncFS] Build indexes from FileTracker entries on disk.
[chromium-blink-merge.git] / third_party / re2 / lib / codereview / codereview.py
blobd26df2a5f18028629e225dcb7648936d1ef4273c
1 # coding=utf-8
2 # (The line above is necessary so that I can use 世界 in the
3 # *comment* below without Python getting all bent out of shape.)
5 # Copyright 2007-2009 Google Inc.
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 '''Mercurial interface to codereview.appspot.com.
21 To configure, set the following options in
22 your repository's .hg/hgrc file.
24 [extensions]
25 codereview = /path/to/codereview.py
27 [codereview]
28 server = codereview.appspot.com
30 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
32 In addition to the new commands, this extension introduces
33 the file pattern syntax @nnnnnn, where nnnnnn is a change list
34 number, to mean the files included in that change list, which
35 must be associated with the current client.
37 For example, if change 123456 contains the files x.go and y.go,
38 "hg diff @123456" is equivalent to"hg diff x.go y.go".
39 '''
41 import sys
43 if __name__ == "__main__":
44 print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
45 sys.exit(2)
47 # We require Python 2.6 for the json package.
48 if sys.version < '2.6':
49 print >>sys.stderr, "The codereview extension requires Python 2.6 or newer."
50 print >>sys.stderr, "You are running Python " + sys.version
51 sys.exit(2)
53 import json
54 import os
55 import re
56 import stat
57 import subprocess
58 import threading
59 import time
61 from mercurial import commands as hg_commands
62 from mercurial import util as hg_util
64 defaultcc = None
65 codereview_disabled = None
66 real_rollback = None
67 releaseBranch = None
68 server = "codereview.appspot.com"
69 server_url_base = None
71 #######################################################################
72 # Normally I would split this into multiple files, but it simplifies
73 # import path headaches to keep it all in one file. Sorry.
74 # The different parts of the file are separated by banners like this one.
76 #######################################################################
77 # Helpers
79 def RelativePath(path, cwd):
80 n = len(cwd)
81 if path.startswith(cwd) and path[n] == '/':
82 return path[n+1:]
83 return path
85 def Sub(l1, l2):
86 return [l for l in l1 if l not in l2]
88 def Add(l1, l2):
89 l = l1 + Sub(l2, l1)
90 l.sort()
91 return l
93 def Intersect(l1, l2):
94 return [l for l in l1 if l in l2]
96 #######################################################################
97 # RE: UNICODE STRING HANDLING
99 # Python distinguishes between the str (string of bytes)
100 # and unicode (string of code points) types. Most operations
101 # work on either one just fine, but some (like regexp matching)
102 # require unicode, and others (like write) require str.
104 # As befits the language, Python hides the distinction between
105 # unicode and str by converting between them silently, but
106 # *only* if all the bytes/code points involved are 7-bit ASCII.
107 # This means that if you're not careful, your program works
108 # fine on "hello, world" and fails on "hello, 世界". And of course,
109 # the obvious way to be careful - use static types - is unavailable.
110 # So the only way is trial and error to find where to put explicit
111 # conversions.
113 # Because more functions do implicit conversion to str (string of bytes)
114 # than do implicit conversion to unicode (string of code points),
115 # the convention in this module is to represent all text as str,
116 # converting to unicode only when calling a unicode-only function
117 # and then converting back to str as soon as possible.
119 def typecheck(s, t):
120 if type(s) != t:
121 raise hg_util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t))
123 # If we have to pass unicode instead of str, ustr does that conversion clearly.
124 def ustr(s):
125 typecheck(s, str)
126 return s.decode("utf-8")
128 # Even with those, Mercurial still sometimes turns unicode into str
129 # and then tries to use it as ascii. Change Mercurial's default.
130 def set_mercurial_encoding_to_utf8():
131 from mercurial import encoding
132 encoding.encoding = 'utf-8'
134 set_mercurial_encoding_to_utf8()
136 # Even with those we still run into problems.
137 # I tried to do things by the book but could not convince
138 # Mercurial to let me check in a change with UTF-8 in the
139 # CL description or author field, no matter how many conversions
140 # between str and unicode I inserted and despite changing the
141 # default encoding. I'm tired of this game, so set the default
142 # encoding for all of Python to 'utf-8', not 'ascii'.
143 def default_to_utf8():
144 import sys
145 stdout, __stdout__ = sys.stdout, sys.__stdout__
146 reload(sys) # site.py deleted setdefaultencoding; get it back
147 sys.stdout, sys.__stdout__ = stdout, __stdout__
148 sys.setdefaultencoding('utf-8')
150 default_to_utf8()
152 #######################################################################
153 # Status printer for long-running commands
155 global_status = None
157 def set_status(s):
158 # print >>sys.stderr, "\t", time.asctime(), s
159 global global_status
160 global_status = s
162 class StatusThread(threading.Thread):
163 def __init__(self):
164 threading.Thread.__init__(self)
165 def run(self):
166 # pause a reasonable amount of time before
167 # starting to display status messages, so that
168 # most hg commands won't ever see them.
169 time.sleep(30)
171 # now show status every 15 seconds
172 while True:
173 time.sleep(15 - time.time() % 15)
174 s = global_status
175 if s is None:
176 continue
177 if s == "":
178 s = "(unknown status)"
179 print >>sys.stderr, time.asctime(), s
181 def start_status_thread():
182 t = StatusThread()
183 t.setDaemon(True) # allowed to exit if t is still running
184 t.start()
186 #######################################################################
187 # Change list parsing.
189 # Change lists are stored in .hg/codereview/cl.nnnnnn
190 # where nnnnnn is the number assigned by the code review server.
191 # Most data about a change list is stored on the code review server
192 # too: the description, reviewer, and cc list are all stored there.
193 # The only thing in the cl.nnnnnn file is the list of relevant files.
194 # Also, the existence of the cl.nnnnnn file marks this repository
195 # as the one where the change list lives.
197 emptydiff = """Index: ~rietveld~placeholder~
198 ===================================================================
199 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
200 new file mode 100644
203 class CL(object):
204 def __init__(self, name):
205 typecheck(name, str)
206 self.name = name
207 self.desc = ''
208 self.files = []
209 self.reviewer = []
210 self.cc = []
211 self.url = ''
212 self.local = False
213 self.web = False
214 self.copied_from = None # None means current user
215 self.mailed = False
216 self.private = False
217 self.lgtm = []
219 def DiskText(self):
220 cl = self
221 s = ""
222 if cl.copied_from:
223 s += "Author: " + cl.copied_from + "\n\n"
224 if cl.private:
225 s += "Private: " + str(self.private) + "\n"
226 s += "Mailed: " + str(self.mailed) + "\n"
227 s += "Description:\n"
228 s += Indent(cl.desc, "\t")
229 s += "Files:\n"
230 for f in cl.files:
231 s += "\t" + f + "\n"
232 typecheck(s, str)
233 return s
235 def EditorText(self):
236 cl = self
237 s = _change_prolog
238 s += "\n"
239 if cl.copied_from:
240 s += "Author: " + cl.copied_from + "\n"
241 if cl.url != '':
242 s += 'URL: ' + cl.url + ' # cannot edit\n\n'
243 if cl.private:
244 s += "Private: True\n"
245 s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
246 s += "CC: " + JoinComma(cl.cc) + "\n"
247 s += "\n"
248 s += "Description:\n"
249 if cl.desc == '':
250 s += "\t<enter description here>\n"
251 else:
252 s += Indent(cl.desc, "\t")
253 s += "\n"
254 if cl.local or cl.name == "new":
255 s += "Files:\n"
256 for f in cl.files:
257 s += "\t" + f + "\n"
258 s += "\n"
259 typecheck(s, str)
260 return s
262 def PendingText(self, quick=False):
263 cl = self
264 s = cl.name + ":" + "\n"
265 s += Indent(cl.desc, "\t")
266 s += "\n"
267 if cl.copied_from:
268 s += "\tAuthor: " + cl.copied_from + "\n"
269 if not quick:
270 s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
271 for (who, line) in cl.lgtm:
272 s += "\t\t" + who + ": " + line + "\n"
273 s += "\tCC: " + JoinComma(cl.cc) + "\n"
274 s += "\tFiles:\n"
275 for f in cl.files:
276 s += "\t\t" + f + "\n"
277 typecheck(s, str)
278 return s
280 def Flush(self, ui, repo):
281 if self.name == "new":
282 self.Upload(ui, repo, gofmt_just_warn=True, creating=True)
283 dir = CodeReviewDir(ui, repo)
284 path = dir + '/cl.' + self.name
285 f = open(path+'!', "w")
286 f.write(self.DiskText())
287 f.close()
288 if sys.platform == "win32" and os.path.isfile(path):
289 os.remove(path)
290 os.rename(path+'!', path)
291 if self.web and not self.copied_from:
292 EditDesc(self.name, desc=self.desc,
293 reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc),
294 private=self.private)
296 def Delete(self, ui, repo):
297 dir = CodeReviewDir(ui, repo)
298 os.unlink(dir + "/cl." + self.name)
300 def Subject(self):
301 s = line1(self.desc)
302 if len(s) > 60:
303 s = s[0:55] + "..."
304 if self.name != "new":
305 s = "code review %s: %s" % (self.name, s)
306 typecheck(s, str)
307 return s
309 def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False):
310 if not self.files and not creating:
311 ui.warn("no files in change list\n")
312 if ui.configbool("codereview", "force_gofmt", True) and gofmt:
313 CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn)
314 set_status("uploading CL metadata + diffs")
315 os.chdir(repo.root)
316 form_fields = [
317 ("content_upload", "1"),
318 ("reviewers", JoinComma(self.reviewer)),
319 ("cc", JoinComma(self.cc)),
320 ("description", self.desc),
321 ("base_hashes", ""),
324 if self.name != "new":
325 form_fields.append(("issue", self.name))
326 vcs = None
327 # We do not include files when creating the issue,
328 # because we want the patch sets to record the repository
329 # and base revision they are diffs against. We use the patch
330 # set message for that purpose, but there is no message with
331 # the first patch set. Instead the message gets used as the
332 # new CL's overall subject. So omit the diffs when creating
333 # and then we'll run an immediate upload.
334 # This has the effect that every CL begins with an empty "Patch set 1".
335 if self.files and not creating:
336 vcs = MercurialVCS(upload_options, ui, repo)
337 data = vcs.GenerateDiff(self.files)
338 files = vcs.GetBaseFiles(data)
339 if len(data) > MAX_UPLOAD_SIZE:
340 uploaded_diff_file = []
341 form_fields.append(("separate_patches", "1"))
342 else:
343 uploaded_diff_file = [("data", "data.diff", data)]
344 else:
345 uploaded_diff_file = [("data", "data.diff", emptydiff)]
347 if vcs and self.name != "new":
348 form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + ui.expandpath("default")))
349 else:
350 # First upload sets the subject for the CL itself.
351 form_fields.append(("subject", self.Subject()))
352 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
353 response_body = MySend("/upload", body, content_type=ctype)
354 patchset = None
355 msg = response_body
356 lines = msg.splitlines()
357 if len(lines) >= 2:
358 msg = lines[0]
359 patchset = lines[1].strip()
360 patches = [x.split(" ", 1) for x in lines[2:]]
361 if response_body.startswith("Issue updated.") and quiet:
362 pass
363 else:
364 ui.status(msg + "\n")
365 set_status("uploaded CL metadata + diffs")
366 if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
367 raise hg_util.Abort("failed to update issue: " + response_body)
368 issue = msg[msg.rfind("/")+1:]
369 self.name = issue
370 if not self.url:
371 self.url = server_url_base + self.name
372 if not uploaded_diff_file:
373 set_status("uploading patches")
374 patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
375 if vcs:
376 set_status("uploading base files")
377 vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
378 if send_mail:
379 set_status("sending mail")
380 MySend("/" + issue + "/mail", payload="")
381 self.web = True
382 set_status("flushing changes to disk")
383 self.Flush(ui, repo)
384 return
386 def Mail(self, ui, repo):
387 pmsg = "Hello " + JoinComma(self.reviewer)
388 if self.cc:
389 pmsg += " (cc: %s)" % (', '.join(self.cc),)
390 pmsg += ",\n"
391 pmsg += "\n"
392 repourl = ui.expandpath("default")
393 if not self.mailed:
394 pmsg += "I'd like you to review this change to\n" + repourl + "\n"
395 else:
396 pmsg += "Please take another look.\n"
397 typecheck(pmsg, str)
398 PostMessage(ui, self.name, pmsg, subject=self.Subject())
399 self.mailed = True
400 self.Flush(ui, repo)
402 def GoodCLName(name):
403 typecheck(name, str)
404 return re.match("^[0-9]+$", name)
406 def ParseCL(text, name):
407 typecheck(text, str)
408 typecheck(name, str)
409 sname = None
410 lineno = 0
411 sections = {
412 'Author': '',
413 'Description': '',
414 'Files': '',
415 'URL': '',
416 'Reviewer': '',
417 'CC': '',
418 'Mailed': '',
419 'Private': '',
421 for line in text.split('\n'):
422 lineno += 1
423 line = line.rstrip()
424 if line != '' and line[0] == '#':
425 continue
426 if line == '' or line[0] == ' ' or line[0] == '\t':
427 if sname == None and line != '':
428 return None, lineno, 'text outside section'
429 if sname != None:
430 sections[sname] += line + '\n'
431 continue
432 p = line.find(':')
433 if p >= 0:
434 s, val = line[:p].strip(), line[p+1:].strip()
435 if s in sections:
436 sname = s
437 if val != '':
438 sections[sname] += val + '\n'
439 continue
440 return None, lineno, 'malformed section header'
442 for k in sections:
443 sections[k] = StripCommon(sections[k]).rstrip()
445 cl = CL(name)
446 if sections['Author']:
447 cl.copied_from = sections['Author']
448 cl.desc = sections['Description']
449 for line in sections['Files'].split('\n'):
450 i = line.find('#')
451 if i >= 0:
452 line = line[0:i].rstrip()
453 line = line.strip()
454 if line == '':
455 continue
456 cl.files.append(line)
457 cl.reviewer = SplitCommaSpace(sections['Reviewer'])
458 cl.cc = SplitCommaSpace(sections['CC'])
459 cl.url = sections['URL']
460 if sections['Mailed'] != 'False':
461 # Odd default, but avoids spurious mailings when
462 # reading old CLs that do not have a Mailed: line.
463 # CLs created with this update will always have
464 # Mailed: False on disk.
465 cl.mailed = True
466 if sections['Private'] in ('True', 'true', 'Yes', 'yes'):
467 cl.private = True
468 if cl.desc == '<enter description here>':
469 cl.desc = ''
470 return cl, 0, ''
472 def SplitCommaSpace(s):
473 typecheck(s, str)
474 s = s.strip()
475 if s == "":
476 return []
477 return re.split(", *", s)
479 def CutDomain(s):
480 typecheck(s, str)
481 i = s.find('@')
482 if i >= 0:
483 s = s[0:i]
484 return s
486 def JoinComma(l):
487 for s in l:
488 typecheck(s, str)
489 return ", ".join(l)
491 def ExceptionDetail():
492 s = str(sys.exc_info()[0])
493 if s.startswith("<type '") and s.endswith("'>"):
494 s = s[7:-2]
495 elif s.startswith("<class '") and s.endswith("'>"):
496 s = s[8:-2]
497 arg = str(sys.exc_info()[1])
498 if len(arg) > 0:
499 s += ": " + arg
500 return s
502 def IsLocalCL(ui, repo, name):
503 return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
505 # Load CL from disk and/or the web.
506 def LoadCL(ui, repo, name, web=True):
507 typecheck(name, str)
508 set_status("loading CL " + name)
509 if not GoodCLName(name):
510 return None, "invalid CL name"
511 dir = CodeReviewDir(ui, repo)
512 path = dir + "cl." + name
513 if os.access(path, 0):
514 ff = open(path)
515 text = ff.read()
516 ff.close()
517 cl, lineno, err = ParseCL(text, name)
518 if err != "":
519 return None, "malformed CL data: "+err
520 cl.local = True
521 else:
522 cl = CL(name)
523 if web:
524 set_status("getting issue metadata from web")
525 d = JSONGet(ui, "/api/" + name + "?messages=true")
526 set_status(None)
527 if d is None:
528 return None, "cannot load CL %s from server" % (name,)
529 if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name:
530 return None, "malformed response loading CL data from code review server"
531 cl.dict = d
532 cl.reviewer = d.get('reviewers', [])
533 cl.cc = d.get('cc', [])
534 if cl.local and cl.copied_from and cl.desc:
535 # local copy of CL written by someone else
536 # and we saved a description. use that one,
537 # so that committers can edit the description
538 # before doing hg submit.
539 pass
540 else:
541 cl.desc = d.get('description', "")
542 cl.url = server_url_base + name
543 cl.web = True
544 cl.private = d.get('private', False) != False
545 cl.lgtm = []
546 for m in d.get('messages', []):
547 if m.get('approval', False) == True:
548 who = re.sub('@.*', '', m.get('sender', ''))
549 text = re.sub("\n(.|\n)*", '', m.get('text', ''))
550 cl.lgtm.append((who, text))
552 set_status("loaded CL " + name)
553 return cl, ''
555 class LoadCLThread(threading.Thread):
556 def __init__(self, ui, repo, dir, f, web):
557 threading.Thread.__init__(self)
558 self.ui = ui
559 self.repo = repo
560 self.dir = dir
561 self.f = f
562 self.web = web
563 self.cl = None
564 def run(self):
565 cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
566 if err != '':
567 self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
568 return
569 self.cl = cl
571 # Load all the CLs from this repository.
572 def LoadAllCL(ui, repo, web=True):
573 dir = CodeReviewDir(ui, repo)
574 m = {}
575 files = [f for f in os.listdir(dir) if f.startswith('cl.')]
576 if not files:
577 return m
578 active = []
579 first = True
580 for f in files:
581 t = LoadCLThread(ui, repo, dir, f, web)
582 t.start()
583 if web and first:
584 # first request: wait in case it needs to authenticate
585 # otherwise we get lots of user/password prompts
586 # running in parallel.
587 t.join()
588 if t.cl:
589 m[t.cl.name] = t.cl
590 first = False
591 else:
592 active.append(t)
593 for t in active:
594 t.join()
595 if t.cl:
596 m[t.cl.name] = t.cl
597 return m
599 # Find repository root. On error, ui.warn and return None
600 def RepoDir(ui, repo):
601 url = repo.url();
602 if not url.startswith('file:'):
603 ui.warn("repository %s is not in local file system\n" % (url,))
604 return None
605 url = url[5:]
606 if url.endswith('/'):
607 url = url[:-1]
608 typecheck(url, str)
609 return url
611 # Find (or make) code review directory. On error, ui.warn and return None
612 def CodeReviewDir(ui, repo):
613 dir = RepoDir(ui, repo)
614 if dir == None:
615 return None
616 dir += '/.hg/codereview/'
617 if not os.path.isdir(dir):
618 try:
619 os.mkdir(dir, 0700)
620 except:
621 ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
622 return None
623 typecheck(dir, str)
624 return dir
626 # Turn leading tabs into spaces, so that the common white space
627 # prefix doesn't get confused when people's editors write out
628 # some lines with spaces, some with tabs. Only a heuristic
629 # (some editors don't use 8 spaces either) but a useful one.
630 def TabsToSpaces(line):
631 i = 0
632 while i < len(line) and line[i] == '\t':
633 i += 1
634 return ' '*(8*i) + line[i:]
636 # Strip maximal common leading white space prefix from text
637 def StripCommon(text):
638 typecheck(text, str)
639 ws = None
640 for line in text.split('\n'):
641 line = line.rstrip()
642 if line == '':
643 continue
644 line = TabsToSpaces(line)
645 white = line[:len(line)-len(line.lstrip())]
646 if ws == None:
647 ws = white
648 else:
649 common = ''
650 for i in range(min(len(white), len(ws))+1):
651 if white[0:i] == ws[0:i]:
652 common = white[0:i]
653 ws = common
654 if ws == '':
655 break
656 if ws == None:
657 return text
658 t = ''
659 for line in text.split('\n'):
660 line = line.rstrip()
661 line = TabsToSpaces(line)
662 if line.startswith(ws):
663 line = line[len(ws):]
664 if line == '' and t == '':
665 continue
666 t += line + '\n'
667 while len(t) >= 2 and t[-2:] == '\n\n':
668 t = t[:-1]
669 typecheck(t, str)
670 return t
672 # Indent text with indent.
673 def Indent(text, indent):
674 typecheck(text, str)
675 typecheck(indent, str)
676 t = ''
677 for line in text.split('\n'):
678 t += indent + line + '\n'
679 typecheck(t, str)
680 return t
682 # Return the first line of l
683 def line1(text):
684 typecheck(text, str)
685 return text.split('\n')[0]
687 _change_prolog = """# Change list.
688 # Lines beginning with # are ignored.
689 # Multi-line values should be indented.
692 desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)'
694 desc_msg = '''Your CL description appears not to use the standard form.
696 The first line of your change description is conventionally a
697 one-line summary of the change, prefixed by the primary affected package,
698 and is used as the subject for code review mail; the rest of the description
699 elaborates.
701 Examples:
703 encoding/rot13: new package
705 math: add IsInf, IsNaN
707 net: fix cname in LookupHost
709 unicode: update to Unicode 5.0.2
713 def promptyesno(ui, msg):
714 return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
716 def promptremove(ui, repo, f):
717 if promptyesno(ui, "hg remove %s (y/n)?" % (f,)):
718 if hg_commands.remove(ui, repo, 'path:'+f) != 0:
719 ui.warn("error removing %s" % (f,))
721 def promptadd(ui, repo, f):
722 if promptyesno(ui, "hg add %s (y/n)?" % (f,)):
723 if hg_commands.add(ui, repo, 'path:'+f) != 0:
724 ui.warn("error adding %s" % (f,))
726 def EditCL(ui, repo, cl):
727 set_status(None) # do not show status
728 s = cl.EditorText()
729 while True:
730 s = ui.edit(s, ui.username())
732 # We can't trust Mercurial + Python not to die before making the change,
733 # so, by popular demand, just scribble the most recent CL edit into
734 # $(hg root)/last-change so that if Mercurial does die, people
735 # can look there for their work.
736 try:
737 f = open(repo.root+"/last-change", "w")
738 f.write(s)
739 f.close()
740 except:
741 pass
743 clx, line, err = ParseCL(s, cl.name)
744 if err != '':
745 if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
746 return "change list not modified"
747 continue
749 # Check description.
750 if clx.desc == '':
751 if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"):
752 continue
753 elif re.search('<enter reason for undo>', clx.desc):
754 if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"):
755 continue
756 elif not re.match(desc_re, clx.desc.split('\n')[0]):
757 if promptyesno(ui, desc_msg + "re-edit (y/n)?"):
758 continue
760 # Check file list for files that need to be hg added or hg removed
761 # or simply aren't understood.
762 pats = ['path:'+f for f in clx.files]
763 changed = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
764 deleted = hg_matchPattern(ui, repo, *pats, deleted=True)
765 unknown = hg_matchPattern(ui, repo, *pats, unknown=True)
766 ignored = hg_matchPattern(ui, repo, *pats, ignored=True)
767 clean = hg_matchPattern(ui, repo, *pats, clean=True)
768 files = []
769 for f in clx.files:
770 if f in changed:
771 files.append(f)
772 continue
773 if f in deleted:
774 promptremove(ui, repo, f)
775 files.append(f)
776 continue
777 if f in unknown:
778 promptadd(ui, repo, f)
779 files.append(f)
780 continue
781 if f in ignored:
782 ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,))
783 continue
784 if f in clean:
785 ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,))
786 files.append(f)
787 continue
788 p = repo.root + '/' + f
789 if os.path.isfile(p):
790 ui.warn("warning: %s is a file but not known to hg\n" % (f,))
791 files.append(f)
792 continue
793 if os.path.isdir(p):
794 ui.warn("error: %s is a directory, not a file; omitting\n" % (f,))
795 continue
796 ui.warn("error: %s does not exist; omitting\n" % (f,))
797 clx.files = files
799 cl.desc = clx.desc
800 cl.reviewer = clx.reviewer
801 cl.cc = clx.cc
802 cl.files = clx.files
803 cl.private = clx.private
804 break
805 return ""
807 # For use by submit, etc. (NOT by change)
808 # Get change list number or list of files from command line.
809 # If files are given, make a new change list.
810 def CommandLineCL(ui, repo, pats, opts, defaultcc=None):
811 if len(pats) > 0 and GoodCLName(pats[0]):
812 if len(pats) != 1:
813 return None, "cannot specify change number and file names"
814 if opts.get('message'):
815 return None, "cannot use -m with existing CL"
816 cl, err = LoadCL(ui, repo, pats[0], web=True)
817 if err != "":
818 return None, err
819 else:
820 cl = CL("new")
821 cl.local = True
822 cl.files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
823 if not cl.files:
824 return None, "no files changed"
825 if opts.get('reviewer'):
826 cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
827 if opts.get('cc'):
828 cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
829 if defaultcc:
830 cl.cc = Add(cl.cc, defaultcc)
831 if cl.name == "new":
832 if opts.get('message'):
833 cl.desc = opts.get('message')
834 else:
835 err = EditCL(ui, repo, cl)
836 if err != '':
837 return None, err
838 return cl, ""
840 #######################################################################
841 # Change list file management
843 # Return list of changed files in repository that match pats.
844 # The patterns came from the command line, so we warn
845 # if they have no effect or cannot be understood.
846 def ChangedFiles(ui, repo, pats, taken=None):
847 taken = taken or {}
848 # Run each pattern separately so that we can warn about
849 # patterns that didn't do anything useful.
850 for p in pats:
851 for f in hg_matchPattern(ui, repo, p, unknown=True):
852 promptadd(ui, repo, f)
853 for f in hg_matchPattern(ui, repo, p, removed=True):
854 promptremove(ui, repo, f)
855 files = hg_matchPattern(ui, repo, p, modified=True, added=True, removed=True)
856 for f in files:
857 if f in taken:
858 ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name))
859 if not files:
860 ui.warn("warning: %s did not match any modified files\n" % (p,))
862 # Again, all at once (eliminates duplicates)
863 l = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
864 l.sort()
865 if taken:
866 l = Sub(l, taken.keys())
867 return l
869 # Return list of changed files in repository that match pats and still exist.
870 def ChangedExistingFiles(ui, repo, pats, opts):
871 l = hg_matchPattern(ui, repo, *pats, modified=True, added=True)
872 l.sort()
873 return l
875 # Return list of files claimed by existing CLs
876 def Taken(ui, repo):
877 all = LoadAllCL(ui, repo, web=False)
878 taken = {}
879 for _, cl in all.items():
880 for f in cl.files:
881 taken[f] = cl
882 return taken
884 # Return list of changed files that are not claimed by other CLs
885 def DefaultFiles(ui, repo, pats):
886 return ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
888 #######################################################################
889 # File format checking.
891 def CheckFormat(ui, repo, files, just_warn=False):
892 set_status("running gofmt")
893 CheckGofmt(ui, repo, files, just_warn)
894 CheckTabfmt(ui, repo, files, just_warn)
896 # Check that gofmt run on the list of files does not change them
897 def CheckGofmt(ui, repo, files, just_warn):
898 files = gofmt_required(files)
899 if not files:
900 return
901 cwd = os.getcwd()
902 files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
903 files = [f for f in files if os.access(f, 0)]
904 if not files:
905 return
906 try:
907 cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32")
908 cmd.stdin.close()
909 except:
910 raise hg_util.Abort("gofmt: " + ExceptionDetail())
911 data = cmd.stdout.read()
912 errors = cmd.stderr.read()
913 cmd.wait()
914 set_status("done with gofmt")
915 if len(errors) > 0:
916 ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
917 return
918 if len(data) > 0:
919 msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
920 if just_warn:
921 ui.warn("warning: " + msg + "\n")
922 else:
923 raise hg_util.Abort(msg)
924 return
926 # Check that *.[chys] files indent using tabs.
927 def CheckTabfmt(ui, repo, files, just_warn):
928 files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f) and not re.search(r"\.tab\.[ch]$", f)]
929 if not files:
930 return
931 cwd = os.getcwd()
932 files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
933 files = [f for f in files if os.access(f, 0)]
934 badfiles = []
935 for f in files:
936 try:
937 for line in open(f, 'r'):
938 # Four leading spaces is enough to complain about,
939 # except that some Plan 9 code uses four spaces as the label indent,
940 # so allow that.
941 if line.startswith(' ') and not re.match(' [A-Za-z0-9_]+:', line):
942 badfiles.append(f)
943 break
944 except:
945 # ignore cannot open file, etc.
946 pass
947 if len(badfiles) > 0:
948 msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles)
949 if just_warn:
950 ui.warn("warning: " + msg + "\n")
951 else:
952 raise hg_util.Abort(msg)
953 return
955 #######################################################################
956 # CONTRIBUTORS file parsing
958 contributorsCache = None
959 contributorsURL = None
961 def ReadContributors(ui, repo):
962 global contributorsCache
963 if contributorsCache is not None:
964 return contributorsCache
966 try:
967 if contributorsURL is not None:
968 opening = contributorsURL
969 f = urllib2.urlopen(contributorsURL)
970 else:
971 opening = repo.root + '/CONTRIBUTORS'
972 f = open(repo.root + '/CONTRIBUTORS', 'r')
973 except:
974 ui.write("warning: cannot open %s: %s\n" % (opening, ExceptionDetail()))
975 return
977 contributors = {}
978 for line in f:
979 # CONTRIBUTORS is a list of lines like:
980 # Person <email>
981 # Person <email> <alt-email>
982 # The first email address is the one used in commit logs.
983 if line.startswith('#'):
984 continue
985 m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
986 if m:
987 name = m.group(1)
988 email = m.group(2)[1:-1]
989 contributors[email.lower()] = (name, email)
990 for extra in m.group(3).split():
991 contributors[extra[1:-1].lower()] = (name, email)
993 contributorsCache = contributors
994 return contributors
996 def CheckContributor(ui, repo, user=None):
997 set_status("checking CONTRIBUTORS file")
998 user, userline = FindContributor(ui, repo, user, warn=False)
999 if not userline:
1000 raise hg_util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1001 return userline
1003 def FindContributor(ui, repo, user=None, warn=True):
1004 if not user:
1005 user = ui.config("ui", "username")
1006 if not user:
1007 raise hg_util.Abort("[ui] username is not configured in .hgrc")
1008 user = user.lower()
1009 m = re.match(r".*<(.*)>", user)
1010 if m:
1011 user = m.group(1)
1013 contributors = ReadContributors(ui, repo)
1014 if user not in contributors:
1015 if warn:
1016 ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1017 return user, None
1019 user, email = contributors[user]
1020 return email, "%s <%s>" % (user, email)
1022 #######################################################################
1023 # Mercurial helper functions.
1024 # Read http://mercurial.selenic.com/wiki/MercurialApi before writing any of these.
1025 # We use the ui.pushbuffer/ui.popbuffer + hg_commands.xxx tricks for all interaction
1026 # with Mercurial. It has proved the most stable as they make changes.
1028 hgversion = hg_util.version()
1030 # We require Mercurial 1.9 and suggest Mercurial 2.0.
1031 # The details of the scmutil package changed then,
1032 # so allowing earlier versions would require extra band-aids below.
1033 # Ubuntu 11.10 ships with Mercurial 1.9.1 as the default version.
1034 hg_required = "1.9"
1035 hg_suggested = "2.0"
1037 old_message = """
1039 The code review extension requires Mercurial """+hg_required+""" or newer.
1040 You are using Mercurial """+hgversion+""".
1042 To install a new Mercurial, use
1044 sudo easy_install mercurial=="""+hg_suggested+"""
1046 or visit http://mercurial.selenic.com/downloads/.
1049 linux_message = """
1050 You may need to clear your current Mercurial installation by running:
1052 sudo apt-get remove mercurial mercurial-common
1053 sudo rm -rf /etc/mercurial
1056 if hgversion < hg_required:
1057 msg = old_message
1058 if os.access("/etc/mercurial", 0):
1059 msg += linux_message
1060 raise hg_util.Abort(msg)
1062 from mercurial.hg import clean as hg_clean
1063 from mercurial import cmdutil as hg_cmdutil
1064 from mercurial import error as hg_error
1065 from mercurial import match as hg_match
1066 from mercurial import node as hg_node
1068 class uiwrap(object):
1069 def __init__(self, ui):
1070 self.ui = ui
1071 ui.pushbuffer()
1072 self.oldQuiet = ui.quiet
1073 ui.quiet = True
1074 self.oldVerbose = ui.verbose
1075 ui.verbose = False
1076 def output(self):
1077 ui = self.ui
1078 ui.quiet = self.oldQuiet
1079 ui.verbose = self.oldVerbose
1080 return ui.popbuffer()
1082 def to_slash(path):
1083 if sys.platform == "win32":
1084 return path.replace('\\', '/')
1085 return path
1087 def hg_matchPattern(ui, repo, *pats, **opts):
1088 w = uiwrap(ui)
1089 hg_commands.status(ui, repo, *pats, **opts)
1090 text = w.output()
1091 ret = []
1092 prefix = to_slash(os.path.realpath(repo.root))+'/'
1093 for line in text.split('\n'):
1094 f = line.split()
1095 if len(f) > 1:
1096 if len(pats) > 0:
1097 # Given patterns, Mercurial shows relative to cwd
1098 p = to_slash(os.path.realpath(f[1]))
1099 if not p.startswith(prefix):
1100 print >>sys.stderr, "File %s not in repo root %s.\n" % (p, prefix)
1101 else:
1102 ret.append(p[len(prefix):])
1103 else:
1104 # Without patterns, Mercurial shows relative to root (what we want)
1105 ret.append(to_slash(f[1]))
1106 return ret
1108 def hg_heads(ui, repo):
1109 w = uiwrap(ui)
1110 hg_commands.heads(ui, repo)
1111 return w.output()
1113 noise = [
1115 "resolving manifests",
1116 "searching for changes",
1117 "couldn't find merge tool hgmerge",
1118 "adding changesets",
1119 "adding manifests",
1120 "adding file changes",
1121 "all local heads known remotely",
1124 def isNoise(line):
1125 line = str(line)
1126 for x in noise:
1127 if line == x:
1128 return True
1129 return False
1131 def hg_incoming(ui, repo):
1132 w = uiwrap(ui)
1133 ret = hg_commands.incoming(ui, repo, force=False, bundle="")
1134 if ret and ret != 1:
1135 raise hg_util.Abort(ret)
1136 return w.output()
1138 def hg_log(ui, repo, **opts):
1139 for k in ['date', 'keyword', 'rev', 'user']:
1140 if not opts.has_key(k):
1141 opts[k] = ""
1142 w = uiwrap(ui)
1143 ret = hg_commands.log(ui, repo, **opts)
1144 if ret:
1145 raise hg_util.Abort(ret)
1146 return w.output()
1148 def hg_outgoing(ui, repo, **opts):
1149 w = uiwrap(ui)
1150 ret = hg_commands.outgoing(ui, repo, **opts)
1151 if ret and ret != 1:
1152 raise hg_util.Abort(ret)
1153 return w.output()
1155 def hg_pull(ui, repo, **opts):
1156 w = uiwrap(ui)
1157 ui.quiet = False
1158 ui.verbose = True # for file list
1159 err = hg_commands.pull(ui, repo, **opts)
1160 for line in w.output().split('\n'):
1161 if isNoise(line):
1162 continue
1163 if line.startswith('moving '):
1164 line = 'mv ' + line[len('moving '):]
1165 if line.startswith('getting ') and line.find(' to ') >= 0:
1166 line = 'mv ' + line[len('getting '):]
1167 if line.startswith('getting '):
1168 line = '+ ' + line[len('getting '):]
1169 if line.startswith('removing '):
1170 line = '- ' + line[len('removing '):]
1171 ui.write(line + '\n')
1172 return err
1174 def hg_push(ui, repo, **opts):
1175 w = uiwrap(ui)
1176 ui.quiet = False
1177 ui.verbose = True
1178 err = hg_commands.push(ui, repo, **opts)
1179 for line in w.output().split('\n'):
1180 if not isNoise(line):
1181 ui.write(line + '\n')
1182 return err
1184 def hg_commit(ui, repo, *pats, **opts):
1185 return hg_commands.commit(ui, repo, *pats, **opts)
1187 #######################################################################
1188 # Mercurial precommit hook to disable commit except through this interface.
1190 commit_okay = False
1192 def precommithook(ui, repo, **opts):
1193 if commit_okay:
1194 return False # False means okay.
1195 ui.write("\ncodereview extension enabled; use mail, upload, or submit instead of commit\n\n")
1196 return True
1198 #######################################################################
1199 # @clnumber file pattern support
1201 # We replace scmutil.match with the MatchAt wrapper to add the @clnumber pattern.
1203 match_repo = None
1204 match_ui = None
1205 match_orig = None
1207 def InstallMatch(ui, repo):
1208 global match_repo
1209 global match_ui
1210 global match_orig
1212 match_ui = ui
1213 match_repo = repo
1215 from mercurial import scmutil
1216 match_orig = scmutil.match
1217 scmutil.match = MatchAt
1219 def MatchAt(ctx, pats=None, opts=None, globbed=False, default='relpath'):
1220 taken = []
1221 files = []
1222 pats = pats or []
1223 opts = opts or {}
1225 for p in pats:
1226 if p.startswith('@'):
1227 taken.append(p)
1228 clname = p[1:]
1229 if clname == "default":
1230 files = DefaultFiles(match_ui, match_repo, [])
1231 else:
1232 if not GoodCLName(clname):
1233 raise hg_util.Abort("invalid CL name " + clname)
1234 cl, err = LoadCL(match_repo.ui, match_repo, clname, web=False)
1235 if err != '':
1236 raise hg_util.Abort("loading CL " + clname + ": " + err)
1237 if not cl.files:
1238 raise hg_util.Abort("no files in CL " + clname)
1239 files = Add(files, cl.files)
1240 pats = Sub(pats, taken) + ['path:'+f for f in files]
1242 # work-around for http://selenic.com/hg/rev/785bbc8634f8
1243 if not hasattr(ctx, 'match'):
1244 ctx = ctx[None]
1245 return match_orig(ctx, pats=pats, opts=opts, globbed=globbed, default=default)
1247 #######################################################################
1248 # Commands added by code review extension.
1250 # As of Mercurial 2.1 the commands are all required to return integer
1251 # exit codes, whereas earlier versions allowed returning arbitrary strings
1252 # to be printed as errors. We wrap the old functions to make sure we
1253 # always return integer exit codes now. Otherwise Mercurial dies
1254 # with a TypeError traceback (unsupported operand type(s) for &: 'str' and 'int').
1255 # Introduce a Python decorator to convert old functions to the new
1256 # stricter convention.
1258 def hgcommand(f):
1259 def wrapped(ui, repo, *pats, **opts):
1260 err = f(ui, repo, *pats, **opts)
1261 if type(err) is int:
1262 return err
1263 if not err:
1264 return 0
1265 raise hg_util.Abort(err)
1266 wrapped.__doc__ = f.__doc__
1267 return wrapped
1269 #######################################################################
1270 # hg change
1272 @hgcommand
1273 def change(ui, repo, *pats, **opts):
1274 """create, edit or delete a change list
1276 Create, edit or delete a change list.
1277 A change list is a group of files to be reviewed and submitted together,
1278 plus a textual description of the change.
1279 Change lists are referred to by simple alphanumeric names.
1281 Changes must be reviewed before they can be submitted.
1283 In the absence of options, the change command opens the
1284 change list for editing in the default editor.
1286 Deleting a change with the -d or -D flag does not affect
1287 the contents of the files listed in that change. To revert
1288 the files listed in a change, use
1290 hg revert @123456
1292 before running hg change -d 123456.
1295 if codereview_disabled:
1296 return codereview_disabled
1298 dirty = {}
1299 if len(pats) > 0 and GoodCLName(pats[0]):
1300 name = pats[0]
1301 if len(pats) != 1:
1302 return "cannot specify CL name and file patterns"
1303 pats = pats[1:]
1304 cl, err = LoadCL(ui, repo, name, web=True)
1305 if err != '':
1306 return err
1307 if not cl.local and (opts["stdin"] or not opts["stdout"]):
1308 return "cannot change non-local CL " + name
1309 else:
1310 name = "new"
1311 cl = CL("new")
1312 if repo[None].branch() != "default":
1313 return "cannot create CL outside default branch; switch with 'hg update default'"
1314 dirty[cl] = True
1315 files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
1317 if opts["delete"] or opts["deletelocal"]:
1318 if opts["delete"] and opts["deletelocal"]:
1319 return "cannot use -d and -D together"
1320 flag = "-d"
1321 if opts["deletelocal"]:
1322 flag = "-D"
1323 if name == "new":
1324 return "cannot use "+flag+" with file patterns"
1325 if opts["stdin"] or opts["stdout"]:
1326 return "cannot use "+flag+" with -i or -o"
1327 if not cl.local:
1328 return "cannot change non-local CL " + name
1329 if opts["delete"]:
1330 if cl.copied_from:
1331 return "original author must delete CL; hg change -D will remove locally"
1332 PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed)
1333 EditDesc(cl.name, closed=True, private=cl.private)
1334 cl.Delete(ui, repo)
1335 return
1337 if opts["stdin"]:
1338 s = sys.stdin.read()
1339 clx, line, err = ParseCL(s, name)
1340 if err != '':
1341 return "error parsing change list: line %d: %s" % (line, err)
1342 if clx.desc is not None:
1343 cl.desc = clx.desc;
1344 dirty[cl] = True
1345 if clx.reviewer is not None:
1346 cl.reviewer = clx.reviewer
1347 dirty[cl] = True
1348 if clx.cc is not None:
1349 cl.cc = clx.cc
1350 dirty[cl] = True
1351 if clx.files is not None:
1352 cl.files = clx.files
1353 dirty[cl] = True
1354 if clx.private != cl.private:
1355 cl.private = clx.private
1356 dirty[cl] = True
1358 if not opts["stdin"] and not opts["stdout"]:
1359 if name == "new":
1360 cl.files = files
1361 err = EditCL(ui, repo, cl)
1362 if err != "":
1363 return err
1364 dirty[cl] = True
1366 for d, _ in dirty.items():
1367 name = d.name
1368 d.Flush(ui, repo)
1369 if name == "new":
1370 d.Upload(ui, repo, quiet=True)
1372 if opts["stdout"]:
1373 ui.write(cl.EditorText())
1374 elif opts["pending"]:
1375 ui.write(cl.PendingText())
1376 elif name == "new":
1377 if ui.quiet:
1378 ui.write(cl.name)
1379 else:
1380 ui.write("CL created: " + cl.url + "\n")
1381 return
1383 #######################################################################
1384 # hg code-login (broken?)
1386 @hgcommand
1387 def code_login(ui, repo, **opts):
1388 """log in to code review server
1390 Logs in to the code review server, saving a cookie in
1391 a file in your home directory.
1393 if codereview_disabled:
1394 return codereview_disabled
1396 MySend(None)
1398 #######################################################################
1399 # hg clpatch / undo / release-apply / download
1400 # All concerned with applying or unapplying patches to the repository.
1402 @hgcommand
1403 def clpatch(ui, repo, clname, **opts):
1404 """import a patch from the code review server
1406 Imports a patch from the code review server into the local client.
1407 If the local client has already modified any of the files that the
1408 patch modifies, this command will refuse to apply the patch.
1410 Submitting an imported patch will keep the original author's
1411 name as the Author: line but add your own name to a Committer: line.
1413 if repo[None].branch() != "default":
1414 return "cannot run hg clpatch outside default branch"
1415 return clpatch_or_undo(ui, repo, clname, opts, mode="clpatch")
1417 @hgcommand
1418 def undo(ui, repo, clname, **opts):
1419 """undo the effect of a CL
1421 Creates a new CL that undoes an earlier CL.
1422 After creating the CL, opens the CL text for editing so that
1423 you can add the reason for the undo to the description.
1425 if repo[None].branch() != "default":
1426 return "cannot run hg undo outside default branch"
1427 return clpatch_or_undo(ui, repo, clname, opts, mode="undo")
1429 @hgcommand
1430 def release_apply(ui, repo, clname, **opts):
1431 """apply a CL to the release branch
1433 Creates a new CL copying a previously committed change
1434 from the main branch to the release branch.
1435 The current client must either be clean or already be in
1436 the release branch.
1438 The release branch must be created by starting with a
1439 clean client, disabling the code review plugin, and running:
1441 hg update weekly.YYYY-MM-DD
1442 hg branch release-branch.rNN
1443 hg commit -m 'create release-branch.rNN'
1444 hg push --new-branch
1446 Then re-enable the code review plugin.
1448 People can test the release branch by running
1450 hg update release-branch.rNN
1452 in a clean client. To return to the normal tree,
1454 hg update default
1456 Move changes since the weekly into the release branch
1457 using hg release-apply followed by the usual code review
1458 process and hg submit.
1460 When it comes time to tag the release, record the
1461 final long-form tag of the release-branch.rNN
1462 in the *default* branch's .hgtags file. That is, run
1464 hg update default
1466 and then edit .hgtags as you would for a weekly.
1469 c = repo[None]
1470 if not releaseBranch:
1471 return "no active release branches"
1472 if c.branch() != releaseBranch:
1473 if c.modified() or c.added() or c.removed():
1474 raise hg_util.Abort("uncommitted local changes - cannot switch branches")
1475 err = hg_clean(repo, releaseBranch)
1476 if err:
1477 return err
1478 try:
1479 err = clpatch_or_undo(ui, repo, clname, opts, mode="backport")
1480 if err:
1481 raise hg_util.Abort(err)
1482 except Exception, e:
1483 hg_clean(repo, "default")
1484 raise e
1485 return None
1487 def rev2clname(rev):
1488 # Extract CL name from revision description.
1489 # The last line in the description that is a codereview URL is the real one.
1490 # Earlier lines might be part of the user-written description.
1491 all = re.findall('(?m)^http://codereview.appspot.com/([0-9]+)$', rev.description())
1492 if len(all) > 0:
1493 return all[-1]
1494 return ""
1496 undoHeader = """undo CL %s / %s
1498 <enter reason for undo>
1500 ««« original CL description
1503 undoFooter = """
1504 »»»
1507 backportHeader = """[%s] %s
1509 ««« CL %s / %s
1512 backportFooter = """
1513 »»»
1516 # Implementation of clpatch/undo.
1517 def clpatch_or_undo(ui, repo, clname, opts, mode):
1518 if codereview_disabled:
1519 return codereview_disabled
1521 if mode == "undo" or mode == "backport":
1522 # Find revision in Mercurial repository.
1523 # Assume CL number is 7+ decimal digits.
1524 # Otherwise is either change log sequence number (fewer decimal digits),
1525 # hexadecimal hash, or tag name.
1526 # Mercurial will fall over long before the change log
1527 # sequence numbers get to be 7 digits long.
1528 if re.match('^[0-9]{7,}$', clname):
1529 found = False
1530 for r in hg_log(ui, repo, keyword="codereview.appspot.com/"+clname, limit=100, template="{node}\n").split():
1531 rev = repo[r]
1532 # Last line with a code review URL is the actual review URL.
1533 # Earlier ones might be part of the CL description.
1534 n = rev2clname(rev)
1535 if n == clname:
1536 found = True
1537 break
1538 if not found:
1539 return "cannot find CL %s in local repository" % clname
1540 else:
1541 rev = repo[clname]
1542 if not rev:
1543 return "unknown revision %s" % clname
1544 clname = rev2clname(rev)
1545 if clname == "":
1546 return "cannot find CL name in revision description"
1548 # Create fresh CL and start with patch that would reverse the change.
1549 vers = hg_node.short(rev.node())
1550 cl = CL("new")
1551 desc = str(rev.description())
1552 if mode == "undo":
1553 cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter
1554 else:
1555 cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter
1556 v1 = vers
1557 v0 = hg_node.short(rev.parents()[0].node())
1558 if mode == "undo":
1559 arg = v1 + ":" + v0
1560 else:
1561 vers = v0
1562 arg = v0 + ":" + v1
1563 patch = RunShell(["hg", "diff", "--git", "-r", arg])
1565 else: # clpatch
1566 cl, vers, patch, err = DownloadCL(ui, repo, clname)
1567 if err != "":
1568 return err
1569 if patch == emptydiff:
1570 return "codereview issue %s has no diff" % clname
1572 # find current hg version (hg identify)
1573 ctx = repo[None]
1574 parents = ctx.parents()
1575 id = '+'.join([hg_node.short(p.node()) for p in parents])
1577 # if version does not match the patch version,
1578 # try to update the patch line numbers.
1579 if vers != "" and id != vers:
1580 # "vers in repo" gives the wrong answer
1581 # on some versions of Mercurial. Instead, do the actual
1582 # lookup and catch the exception.
1583 try:
1584 repo[vers].description()
1585 except:
1586 return "local repository is out of date; sync to get %s" % (vers)
1587 patch1, err = portPatch(repo, patch, vers, id)
1588 if err != "":
1589 if not opts["ignore_hgpatch_failure"]:
1590 return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id)
1591 else:
1592 patch = patch1
1593 argv = ["hgpatch"]
1594 if opts["no_incoming"] or mode == "backport":
1595 argv += ["--checksync=false"]
1596 try:
1597 cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32")
1598 except:
1599 return "hgpatch: " + ExceptionDetail() + "\nInstall hgpatch with:\n$ go get code.google.com/p/go.codereview/cmd/hgpatch\n"
1601 out, err = cmd.communicate(patch)
1602 if cmd.returncode != 0 and not opts["ignore_hgpatch_failure"]:
1603 return "hgpatch failed"
1604 cl.local = True
1605 cl.files = out.strip().split()
1606 if not cl.files and not opts["ignore_hgpatch_failure"]:
1607 return "codereview issue %s has no changed files" % clname
1608 files = ChangedFiles(ui, repo, [])
1609 extra = Sub(cl.files, files)
1610 if extra:
1611 ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
1612 cl.Flush(ui, repo)
1613 if mode == "undo":
1614 err = EditCL(ui, repo, cl)
1615 if err != "":
1616 return "CL created, but error editing: " + err
1617 cl.Flush(ui, repo)
1618 else:
1619 ui.write(cl.PendingText() + "\n")
1621 # portPatch rewrites patch from being a patch against
1622 # oldver to being a patch against newver.
1623 def portPatch(repo, patch, oldver, newver):
1624 lines = patch.splitlines(True) # True = keep \n
1625 delta = None
1626 for i in range(len(lines)):
1627 line = lines[i]
1628 if line.startswith('--- a/'):
1629 file = line[6:-1]
1630 delta = fileDeltas(repo, file, oldver, newver)
1631 if not delta or not line.startswith('@@ '):
1632 continue
1633 # @@ -x,y +z,w @@ means the patch chunk replaces
1634 # the original file's line numbers x up to x+y with the
1635 # line numbers z up to z+w in the new file.
1636 # Find the delta from x in the original to the same
1637 # line in the current version and add that delta to both
1638 # x and z.
1639 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1640 if not m:
1641 return None, "error parsing patch line numbers"
1642 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1643 d, err = lineDelta(delta, n1, len1)
1644 if err != "":
1645 return "", err
1646 n1 += d
1647 n2 += d
1648 lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2)
1650 newpatch = ''.join(lines)
1651 return newpatch, ""
1653 # fileDelta returns the line number deltas for the given file's
1654 # changes from oldver to newver.
1655 # The deltas are a list of (n, len, newdelta) triples that say
1656 # lines [n, n+len) were modified, and after that range the
1657 # line numbers are +newdelta from what they were before.
1658 def fileDeltas(repo, file, oldver, newver):
1659 cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file]
1660 data = RunShell(cmd, silent_ok=True)
1661 deltas = []
1662 for line in data.splitlines():
1663 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1664 if not m:
1665 continue
1666 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1667 deltas.append((n1, len1, n2+len2-(n1+len1)))
1668 return deltas
1670 # lineDelta finds the appropriate line number delta to apply to the lines [n, n+len).
1671 # It returns an error if those lines were rewritten by the patch.
1672 def lineDelta(deltas, n, len):
1673 d = 0
1674 for (old, oldlen, newdelta) in deltas:
1675 if old >= n+len:
1676 break
1677 if old+len > n:
1678 return 0, "patch and recent changes conflict"
1679 d = newdelta
1680 return d, ""
1682 @hgcommand
1683 def download(ui, repo, clname, **opts):
1684 """download a change from the code review server
1686 Download prints a description of the given change list
1687 followed by its diff, downloaded from the code review server.
1689 if codereview_disabled:
1690 return codereview_disabled
1692 cl, vers, patch, err = DownloadCL(ui, repo, clname)
1693 if err != "":
1694 return err
1695 ui.write(cl.EditorText() + "\n")
1696 ui.write(patch + "\n")
1697 return
1699 #######################################################################
1700 # hg file
1702 @hgcommand
1703 def file(ui, repo, clname, pat, *pats, **opts):
1704 """assign files to or remove files from a change list
1706 Assign files to or (with -d) remove files from a change list.
1708 The -d option only removes files from the change list.
1709 It does not edit them or remove them from the repository.
1711 if codereview_disabled:
1712 return codereview_disabled
1714 pats = tuple([pat] + list(pats))
1715 if not GoodCLName(clname):
1716 return "invalid CL name " + clname
1718 dirty = {}
1719 cl, err = LoadCL(ui, repo, clname, web=False)
1720 if err != '':
1721 return err
1722 if not cl.local:
1723 return "cannot change non-local CL " + clname
1725 files = ChangedFiles(ui, repo, pats)
1727 if opts["delete"]:
1728 oldfiles = Intersect(files, cl.files)
1729 if oldfiles:
1730 if not ui.quiet:
1731 ui.status("# Removing files from CL. To undo:\n")
1732 ui.status("# cd %s\n" % (repo.root))
1733 for f in oldfiles:
1734 ui.status("# hg file %s %s\n" % (cl.name, f))
1735 cl.files = Sub(cl.files, oldfiles)
1736 cl.Flush(ui, repo)
1737 else:
1738 ui.status("no such files in CL")
1739 return
1741 if not files:
1742 return "no such modified files"
1744 files = Sub(files, cl.files)
1745 taken = Taken(ui, repo)
1746 warned = False
1747 for f in files:
1748 if f in taken:
1749 if not warned and not ui.quiet:
1750 ui.status("# Taking files from other CLs. To undo:\n")
1751 ui.status("# cd %s\n" % (repo.root))
1752 warned = True
1753 ocl = taken[f]
1754 if not ui.quiet:
1755 ui.status("# hg file %s %s\n" % (ocl.name, f))
1756 if ocl not in dirty:
1757 ocl.files = Sub(ocl.files, files)
1758 dirty[ocl] = True
1759 cl.files = Add(cl.files, files)
1760 dirty[cl] = True
1761 for d, _ in dirty.items():
1762 d.Flush(ui, repo)
1763 return
1765 #######################################################################
1766 # hg gofmt
1768 @hgcommand
1769 def gofmt(ui, repo, *pats, **opts):
1770 """apply gofmt to modified files
1772 Applies gofmt to the modified files in the repository that match
1773 the given patterns.
1775 if codereview_disabled:
1776 return codereview_disabled
1778 files = ChangedExistingFiles(ui, repo, pats, opts)
1779 files = gofmt_required(files)
1780 if not files:
1781 return "no modified go files"
1782 cwd = os.getcwd()
1783 files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1784 try:
1785 cmd = ["gofmt", "-l"]
1786 if not opts["list"]:
1787 cmd += ["-w"]
1788 if os.spawnvp(os.P_WAIT, "gofmt", cmd + files) != 0:
1789 raise hg_util.Abort("gofmt did not exit cleanly")
1790 except hg_error.Abort, e:
1791 raise
1792 except:
1793 raise hg_util.Abort("gofmt: " + ExceptionDetail())
1794 return
1796 def gofmt_required(files):
1797 return [f for f in files if (not f.startswith('test/') or f.startswith('test/bench/')) and f.endswith('.go')]
1799 #######################################################################
1800 # hg mail
1802 @hgcommand
1803 def mail(ui, repo, *pats, **opts):
1804 """mail a change for review
1806 Uploads a patch to the code review server and then sends mail
1807 to the reviewer and CC list asking for a review.
1809 if codereview_disabled:
1810 return codereview_disabled
1812 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1813 if err != "":
1814 return err
1815 cl.Upload(ui, repo, gofmt_just_warn=True)
1816 if not cl.reviewer:
1817 # If no reviewer is listed, assign the review to defaultcc.
1818 # This makes sure that it appears in the
1819 # codereview.appspot.com/user/defaultcc
1820 # page, so that it doesn't get dropped on the floor.
1821 if not defaultcc:
1822 return "no reviewers listed in CL"
1823 cl.cc = Sub(cl.cc, defaultcc)
1824 cl.reviewer = defaultcc
1825 cl.Flush(ui, repo)
1827 if cl.files == []:
1828 return "no changed files, not sending mail"
1830 cl.Mail(ui, repo)
1832 #######################################################################
1833 # hg p / hg pq / hg ps / hg pending
1835 @hgcommand
1836 def ps(ui, repo, *pats, **opts):
1837 """alias for hg p --short
1839 opts['short'] = True
1840 return pending(ui, repo, *pats, **opts)
1842 @hgcommand
1843 def pq(ui, repo, *pats, **opts):
1844 """alias for hg p --quick
1846 opts['quick'] = True
1847 return pending(ui, repo, *pats, **opts)
1849 @hgcommand
1850 def pending(ui, repo, *pats, **opts):
1851 """show pending changes
1853 Lists pending changes followed by a list of unassigned but modified files.
1855 if codereview_disabled:
1856 return codereview_disabled
1858 quick = opts.get('quick', False)
1859 short = opts.get('short', False)
1860 m = LoadAllCL(ui, repo, web=not quick and not short)
1861 names = m.keys()
1862 names.sort()
1863 for name in names:
1864 cl = m[name]
1865 if short:
1866 ui.write(name + "\t" + line1(cl.desc) + "\n")
1867 else:
1868 ui.write(cl.PendingText(quick=quick) + "\n")
1870 if short:
1871 return
1872 files = DefaultFiles(ui, repo, [])
1873 if len(files) > 0:
1874 s = "Changed files not in any CL:\n"
1875 for f in files:
1876 s += "\t" + f + "\n"
1877 ui.write(s)
1879 #######################################################################
1880 # hg submit
1882 def need_sync():
1883 raise hg_util.Abort("local repository out of date; must sync before submit")
1885 @hgcommand
1886 def submit(ui, repo, *pats, **opts):
1887 """submit change to remote repository
1889 Submits change to remote repository.
1890 Bails out if the local repository is not in sync with the remote one.
1892 if codereview_disabled:
1893 return codereview_disabled
1895 # We already called this on startup but sometimes Mercurial forgets.
1896 set_mercurial_encoding_to_utf8()
1898 if not opts["no_incoming"] and hg_incoming(ui, repo):
1899 need_sync()
1901 cl, err = CommandLineCL(ui, repo, pats, opts, defaultcc=defaultcc)
1902 if err != "":
1903 return err
1905 user = None
1906 if cl.copied_from:
1907 user = cl.copied_from
1908 userline = CheckContributor(ui, repo, user)
1909 typecheck(userline, str)
1911 about = ""
1912 if cl.reviewer:
1913 about += "R=" + JoinComma([CutDomain(s) for s in cl.reviewer]) + "\n"
1914 if opts.get('tbr'):
1915 tbr = SplitCommaSpace(opts.get('tbr'))
1916 cl.reviewer = Add(cl.reviewer, tbr)
1917 about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1918 if cl.cc:
1919 about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1921 if not cl.reviewer:
1922 return "no reviewers listed in CL"
1924 if not cl.local:
1925 return "cannot submit non-local CL"
1927 # upload, to sync current patch and also get change number if CL is new.
1928 if not cl.copied_from:
1929 cl.Upload(ui, repo, gofmt_just_warn=True)
1931 # check gofmt for real; allowed upload to warn in order to save CL.
1932 cl.Flush(ui, repo)
1933 CheckFormat(ui, repo, cl.files)
1935 about += "%s%s\n" % (server_url_base, cl.name)
1937 if cl.copied_from:
1938 about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1939 typecheck(about, str)
1941 if not cl.mailed and not cl.copied_from: # in case this is TBR
1942 cl.Mail(ui, repo)
1944 # submit changes locally
1945 message = cl.desc.rstrip() + "\n\n" + about
1946 typecheck(message, str)
1948 set_status("pushing " + cl.name + " to remote server")
1950 if hg_outgoing(ui, repo):
1951 raise hg_util.Abort("local repository corrupt or out-of-phase with remote: found outgoing changes")
1953 old_heads = len(hg_heads(ui, repo).split())
1955 global commit_okay
1956 commit_okay = True
1957 ret = hg_commit(ui, repo, *['path:'+f for f in cl.files], message=message, user=userline)
1958 commit_okay = False
1959 if ret:
1960 return "nothing changed"
1961 node = repo["-1"].node()
1962 # push to remote; if it fails for any reason, roll back
1963 try:
1964 new_heads = len(hg_heads(ui, repo).split())
1965 if old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
1966 # Created new head, so we weren't up to date.
1967 need_sync()
1969 # Push changes to remote. If it works, we're committed. If not, roll back.
1970 try:
1971 hg_push(ui, repo)
1972 except hg_error.Abort, e:
1973 if e.message.find("push creates new heads") >= 0:
1974 # Remote repository had changes we missed.
1975 need_sync()
1976 raise
1977 except:
1978 real_rollback()
1979 raise
1981 # We're committed. Upload final patch, close review, add commit message.
1982 changeURL = hg_node.short(node)
1983 url = ui.expandpath("default")
1984 m = re.match("(^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?)" + "|" +
1985 "(^https?://([^@/]+@)?code\.google\.com/p/([^/.]+)(\.[^./]+)?/?)", url)
1986 if m:
1987 if m.group(1): # prj.googlecode.com/hg/ case
1988 changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(3), changeURL)
1989 elif m.group(4) and m.group(7): # code.google.com/p/prj.subrepo/ case
1990 changeURL = "http://code.google.com/p/%s/source/detail?r=%s&repo=%s" % (m.group(6), changeURL, m.group(7)[1:])
1991 elif m.group(4): # code.google.com/p/prj/ case
1992 changeURL = "http://code.google.com/p/%s/source/detail?r=%s" % (m.group(6), changeURL)
1993 else:
1994 print >>sys.stderr, "URL: ", url
1995 else:
1996 print >>sys.stderr, "URL: ", url
1997 pmsg = "*** Submitted as " + changeURL + " ***\n\n" + message
1999 # When posting, move reviewers to CC line,
2000 # so that the issue stops showing up in their "My Issues" page.
2001 PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
2003 if not cl.copied_from:
2004 EditDesc(cl.name, closed=True, private=cl.private)
2005 cl.Delete(ui, repo)
2007 c = repo[None]
2008 if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed():
2009 ui.write("switching from %s to default branch.\n" % releaseBranch)
2010 err = hg_clean(repo, "default")
2011 if err:
2012 return err
2013 return None
2015 #######################################################################
2016 # hg sync
2018 @hgcommand
2019 def sync(ui, repo, **opts):
2020 """synchronize with remote repository
2022 Incorporates recent changes from the remote repository
2023 into the local repository.
2025 if codereview_disabled:
2026 return codereview_disabled
2028 if not opts["local"]:
2029 err = hg_pull(ui, repo, update=True)
2030 if err:
2031 return err
2032 sync_changes(ui, repo)
2034 def sync_changes(ui, repo):
2035 # Look through recent change log descriptions to find
2036 # potential references to http://.*/our-CL-number.
2037 # Double-check them by looking at the Rietveld log.
2038 for rev in hg_log(ui, repo, limit=100, template="{node}\n").split():
2039 desc = repo[rev].description().strip()
2040 for clname in re.findall('(?m)^http://(?:[^\n]+)/([0-9]+)$', desc):
2041 if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
2042 ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
2043 cl, err = LoadCL(ui, repo, clname, web=False)
2044 if err != "":
2045 ui.warn("loading CL %s: %s\n" % (clname, err))
2046 continue
2047 if not cl.copied_from:
2048 EditDesc(cl.name, closed=True, private=cl.private)
2049 cl.Delete(ui, repo)
2051 # Remove files that are not modified from the CLs in which they appear.
2052 all = LoadAllCL(ui, repo, web=False)
2053 changed = ChangedFiles(ui, repo, [])
2054 for cl in all.values():
2055 extra = Sub(cl.files, changed)
2056 if extra:
2057 ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
2058 for f in extra:
2059 ui.warn("\t%s\n" % (f,))
2060 cl.files = Sub(cl.files, extra)
2061 cl.Flush(ui, repo)
2062 if not cl.files:
2063 if not cl.copied_from:
2064 ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name))
2065 else:
2066 ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name))
2067 return
2069 #######################################################################
2070 # hg upload
2072 @hgcommand
2073 def upload(ui, repo, name, **opts):
2074 """upload diffs to the code review server
2076 Uploads the current modifications for a given change to the server.
2078 if codereview_disabled:
2079 return codereview_disabled
2081 repo.ui.quiet = True
2082 cl, err = LoadCL(ui, repo, name, web=True)
2083 if err != "":
2084 return err
2085 if not cl.local:
2086 return "cannot upload non-local change"
2087 cl.Upload(ui, repo)
2088 print "%s%s\n" % (server_url_base, cl.name)
2089 return
2091 #######################################################################
2092 # Table of commands, supplied to Mercurial for installation.
2094 review_opts = [
2095 ('r', 'reviewer', '', 'add reviewer'),
2096 ('', 'cc', '', 'add cc'),
2097 ('', 'tbr', '', 'add future reviewer'),
2098 ('m', 'message', '', 'change description (for new change)'),
2101 cmdtable = {
2102 # The ^ means to show this command in the help text that
2103 # is printed when running hg with no arguments.
2104 "^change": (
2105 change,
2107 ('d', 'delete', None, 'delete existing change list'),
2108 ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
2109 ('i', 'stdin', None, 'read change list from standard input'),
2110 ('o', 'stdout', None, 'print change list to standard output'),
2111 ('p', 'pending', None, 'print pending summary to standard output'),
2113 "[-d | -D] [-i] [-o] change# or FILE ..."
2115 "^clpatch": (
2116 clpatch,
2118 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2119 ('', 'no_incoming', None, 'disable check for incoming changes'),
2121 "change#"
2123 # Would prefer to call this codereview-login, but then
2124 # hg help codereview prints the help for this command
2125 # instead of the help for the extension.
2126 "code-login": (
2127 code_login,
2131 "^download": (
2132 download,
2134 "change#"
2136 "^file": (
2137 file,
2139 ('d', 'delete', None, 'delete files from change list (but not repository)'),
2141 "[-d] change# FILE ..."
2143 "^gofmt": (
2144 gofmt,
2146 ('l', 'list', None, 'list files that would change, but do not edit them'),
2148 "FILE ..."
2150 "^pending|p": (
2151 pending,
2153 ('s', 'short', False, 'show short result form'),
2154 ('', 'quick', False, 'do not consult codereview server'),
2156 "[FILE ...]"
2158 "^ps": (
2161 "[FILE ...]"
2163 "^pq": (
2166 "[FILE ...]"
2168 "^mail": (
2169 mail,
2170 review_opts + [
2171 ] + hg_commands.walkopts,
2172 "[-r reviewer] [--cc cc] [change# | file ...]"
2174 "^release-apply": (
2175 release_apply,
2177 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2178 ('', 'no_incoming', None, 'disable check for incoming changes'),
2180 "change#"
2182 # TODO: release-start, release-tag, weekly-tag
2183 "^submit": (
2184 submit,
2185 review_opts + [
2186 ('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
2187 ] + hg_commands.walkopts + hg_commands.commitopts + hg_commands.commitopts2,
2188 "[-r reviewer] [--cc cc] [change# | file ...]"
2190 "^sync": (
2191 sync,
2193 ('', 'local', None, 'do not pull changes from remote repository')
2195 "[--local]",
2197 "^undo": (
2198 undo,
2200 ('', 'ignore_hgpatch_failure', None, 'create CL metadata even if hgpatch fails'),
2201 ('', 'no_incoming', None, 'disable check for incoming changes'),
2203 "change#"
2205 "^upload": (
2206 upload,
2208 "change#"
2212 #######################################################################
2213 # Mercurial extension initialization
2215 def norollback(*pats, **opts):
2216 """(disabled when using this extension)"""
2217 raise hg_util.Abort("codereview extension enabled; use undo instead of rollback")
2219 codereview_init = False
2221 def reposetup(ui, repo):
2222 global codereview_disabled
2223 global defaultcc
2225 # reposetup gets called both for the local repository
2226 # and also for any repository we are pulling or pushing to.
2227 # Only initialize the first time.
2228 global codereview_init
2229 if codereview_init:
2230 return
2231 codereview_init = True
2233 # Read repository-specific options from lib/codereview/codereview.cfg or codereview.cfg.
2234 root = ''
2235 try:
2236 root = repo.root
2237 except:
2238 # Yes, repo might not have root; see issue 959.
2239 codereview_disabled = 'codereview disabled: repository has no root'
2240 return
2242 repo_config_path = ''
2243 p1 = root + '/lib/codereview/codereview.cfg'
2244 p2 = root + '/codereview.cfg'
2245 if os.access(p1, os.F_OK):
2246 repo_config_path = p1
2247 else:
2248 repo_config_path = p2
2249 try:
2250 f = open(repo_config_path)
2251 for line in f:
2252 if line.startswith('defaultcc:'):
2253 defaultcc = SplitCommaSpace(line[len('defaultcc:'):])
2254 if line.startswith('contributors:'):
2255 global contributorsURL
2256 contributorsURL = line[len('contributors:'):].strip()
2257 except:
2258 codereview_disabled = 'codereview disabled: cannot open ' + repo_config_path
2259 return
2261 remote = ui.config("paths", "default", "")
2262 if remote.find("://") < 0:
2263 raise hg_util.Abort("codereview: default path '%s' is not a URL" % (remote,))
2265 InstallMatch(ui, repo)
2266 RietveldSetup(ui, repo)
2268 # Disable the Mercurial commands that might change the repository.
2269 # Only commands in this extension are supposed to do that.
2270 ui.setconfig("hooks", "precommit.codereview", precommithook)
2272 # Rollback removes an existing commit. Don't do that either.
2273 global real_rollback
2274 real_rollback = repo.rollback
2275 repo.rollback = norollback
2278 #######################################################################
2279 # Wrappers around upload.py for interacting with Rietveld
2281 from HTMLParser import HTMLParser
2283 # HTML form parser
2284 class FormParser(HTMLParser):
2285 def __init__(self):
2286 self.map = {}
2287 self.curtag = None
2288 self.curdata = None
2289 HTMLParser.__init__(self)
2290 def handle_starttag(self, tag, attrs):
2291 if tag == "input":
2292 key = None
2293 value = ''
2294 for a in attrs:
2295 if a[0] == 'name':
2296 key = a[1]
2297 if a[0] == 'value':
2298 value = a[1]
2299 if key is not None:
2300 self.map[key] = value
2301 if tag == "textarea":
2302 key = None
2303 for a in attrs:
2304 if a[0] == 'name':
2305 key = a[1]
2306 if key is not None:
2307 self.curtag = key
2308 self.curdata = ''
2309 def handle_endtag(self, tag):
2310 if tag == "textarea" and self.curtag is not None:
2311 self.map[self.curtag] = self.curdata
2312 self.curtag = None
2313 self.curdata = None
2314 def handle_charref(self, name):
2315 self.handle_data(unichr(int(name)))
2316 def handle_entityref(self, name):
2317 import htmlentitydefs
2318 if name in htmlentitydefs.entitydefs:
2319 self.handle_data(htmlentitydefs.entitydefs[name])
2320 else:
2321 self.handle_data("&" + name + ";")
2322 def handle_data(self, data):
2323 if self.curdata is not None:
2324 self.curdata += data
2326 def JSONGet(ui, path):
2327 try:
2328 data = MySend(path, force_auth=False)
2329 typecheck(data, str)
2330 d = fix_json(json.loads(data))
2331 except:
2332 ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail()))
2333 return None
2334 return d
2336 # Clean up json parser output to match our expectations:
2337 # * all strings are UTF-8-encoded str, not unicode.
2338 # * missing fields are missing, not None,
2339 # so that d.get("foo", defaultvalue) works.
2340 def fix_json(x):
2341 if type(x) in [str, int, float, bool, type(None)]:
2342 pass
2343 elif type(x) is unicode:
2344 x = x.encode("utf-8")
2345 elif type(x) is list:
2346 for i in range(len(x)):
2347 x[i] = fix_json(x[i])
2348 elif type(x) is dict:
2349 todel = []
2350 for k in x:
2351 if x[k] is None:
2352 todel.append(k)
2353 else:
2354 x[k] = fix_json(x[k])
2355 for k in todel:
2356 del x[k]
2357 else:
2358 raise hg_util.Abort("unknown type " + str(type(x)) + " in fix_json")
2359 if type(x) is str:
2360 x = x.replace('\r\n', '\n')
2361 return x
2363 def IsRietveldSubmitted(ui, clname, hex):
2364 dict = JSONGet(ui, "/api/" + clname + "?messages=true")
2365 if dict is None:
2366 return False
2367 for msg in dict.get("messages", []):
2368 text = msg.get("text", "")
2369 m = re.match('\*\*\* Submitted as [^*]*?([0-9a-f]+) \*\*\*', text)
2370 if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
2371 return True
2372 return False
2374 def IsRietveldMailed(cl):
2375 for msg in cl.dict.get("messages", []):
2376 if msg.get("text", "").find("I'd like you to review this change") >= 0:
2377 return True
2378 return False
2380 def DownloadCL(ui, repo, clname):
2381 set_status("downloading CL " + clname)
2382 cl, err = LoadCL(ui, repo, clname, web=True)
2383 if err != "":
2384 return None, None, None, "error loading CL %s: %s" % (clname, err)
2386 # Find most recent diff
2387 diffs = cl.dict.get("patchsets", [])
2388 if not diffs:
2389 return None, None, None, "CL has no patch sets"
2390 patchid = diffs[-1]
2392 patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid))
2393 if patchset is None:
2394 return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid)
2395 if patchset.get("patchset", 0) != patchid:
2396 return None, None, None, "malformed patchset information"
2398 vers = ""
2399 msg = patchset.get("message", "").split()
2400 if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r":
2401 vers = msg[2]
2402 diff = "/download/issue" + clname + "_" + str(patchid) + ".diff"
2404 diffdata = MySend(diff, force_auth=False)
2406 # Print warning if email is not in CONTRIBUTORS file.
2407 email = cl.dict.get("owner_email", "")
2408 if not email:
2409 return None, None, None, "cannot find owner for %s" % (clname)
2410 him = FindContributor(ui, repo, email)
2411 me = FindContributor(ui, repo, None)
2412 if him == me:
2413 cl.mailed = IsRietveldMailed(cl)
2414 else:
2415 cl.copied_from = email
2417 return cl, vers, diffdata, ""
2419 def MySend(request_path, payload=None,
2420 content_type="application/octet-stream",
2421 timeout=None, force_auth=True,
2422 **kwargs):
2423 """Run MySend1 maybe twice, because Rietveld is unreliable."""
2424 try:
2425 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2426 except Exception, e:
2427 if type(e) != urllib2.HTTPError or e.code != 500: # only retry on HTTP 500 error
2428 raise
2429 print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
2430 time.sleep(2)
2431 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2433 # Like upload.py Send but only authenticates when the
2434 # redirect is to www.google.com/accounts. This keeps
2435 # unnecessary redirects from happening during testing.
2436 def MySend1(request_path, payload=None,
2437 content_type="application/octet-stream",
2438 timeout=None, force_auth=True,
2439 **kwargs):
2440 """Sends an RPC and returns the response.
2442 Args:
2443 request_path: The path to send the request to, eg /api/appversion/create.
2444 payload: The body of the request, or None to send an empty request.
2445 content_type: The Content-Type header to use.
2446 timeout: timeout in seconds; default None i.e. no timeout.
2447 (Note: for large requests on OS X, the timeout doesn't work right.)
2448 kwargs: Any keyword arguments are converted into query string parameters.
2450 Returns:
2451 The response body, as a string.
2453 # TODO: Don't require authentication. Let the server say
2454 # whether it is necessary.
2455 global rpc
2456 if rpc == None:
2457 rpc = GetRpcServer(upload_options)
2458 self = rpc
2459 if not self.authenticated and force_auth:
2460 self._Authenticate()
2461 if request_path is None:
2462 return
2464 old_timeout = socket.getdefaulttimeout()
2465 socket.setdefaulttimeout(timeout)
2466 try:
2467 tries = 0
2468 while True:
2469 tries += 1
2470 args = dict(kwargs)
2471 url = "http://%s%s" % (self.host, request_path)
2472 if args:
2473 url += "?" + urllib.urlencode(args)
2474 req = self._CreateRequest(url=url, data=payload)
2475 req.add_header("Content-Type", content_type)
2476 try:
2477 f = self.opener.open(req)
2478 response = f.read()
2479 f.close()
2480 # Translate \r\n into \n, because Rietveld doesn't.
2481 response = response.replace('\r\n', '\n')
2482 # who knows what urllib will give us
2483 if type(response) == unicode:
2484 response = response.encode("utf-8")
2485 typecheck(response, str)
2486 return response
2487 except urllib2.HTTPError, e:
2488 if tries > 3:
2489 raise
2490 elif e.code == 401:
2491 self._Authenticate()
2492 elif e.code == 302:
2493 loc = e.info()["location"]
2494 if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
2495 return ''
2496 self._Authenticate()
2497 else:
2498 raise
2499 finally:
2500 socket.setdefaulttimeout(old_timeout)
2502 def GetForm(url):
2503 f = FormParser()
2504 f.feed(ustr(MySend(url))) # f.feed wants unicode
2505 f.close()
2506 # convert back to utf-8 to restore sanity
2507 m = {}
2508 for k,v in f.map.items():
2509 m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8")
2510 return m
2512 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False):
2513 set_status("uploading change to description")
2514 form_fields = GetForm("/" + issue + "/edit")
2515 if subject is not None:
2516 form_fields['subject'] = subject
2517 if desc is not None:
2518 form_fields['description'] = desc
2519 if reviewers is not None:
2520 form_fields['reviewers'] = reviewers
2521 if cc is not None:
2522 form_fields['cc'] = cc
2523 if closed:
2524 form_fields['closed'] = "checked"
2525 if private:
2526 form_fields['private'] = "checked"
2527 ctype, body = EncodeMultipartFormData(form_fields.items(), [])
2528 response = MySend("/" + issue + "/edit", body, content_type=ctype)
2529 if response != "":
2530 print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
2531 sys.exit(2)
2533 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
2534 set_status("uploading message")
2535 form_fields = GetForm("/" + issue + "/publish")
2536 if reviewers is not None:
2537 form_fields['reviewers'] = reviewers
2538 if cc is not None:
2539 form_fields['cc'] = cc
2540 if send_mail:
2541 form_fields['send_mail'] = "checked"
2542 else:
2543 del form_fields['send_mail']
2544 if subject is not None:
2545 form_fields['subject'] = subject
2546 form_fields['message'] = message
2548 form_fields['message_only'] = '1' # Don't include draft comments
2549 if reviewers is not None or cc is not None:
2550 form_fields['message_only'] = '' # Must set '' in order to override cc/reviewer
2551 ctype = "applications/x-www-form-urlencoded"
2552 body = urllib.urlencode(form_fields)
2553 response = MySend("/" + issue + "/publish", body, content_type=ctype)
2554 if response != "":
2555 print response
2556 sys.exit(2)
2558 class opt(object):
2559 pass
2561 def RietveldSetup(ui, repo):
2562 global force_google_account
2563 global rpc
2564 global server
2565 global server_url_base
2566 global upload_options
2567 global verbosity
2569 if not ui.verbose:
2570 verbosity = 0
2572 # Config options.
2573 x = ui.config("codereview", "server")
2574 if x is not None:
2575 server = x
2577 # TODO(rsc): Take from ui.username?
2578 email = None
2579 x = ui.config("codereview", "email")
2580 if x is not None:
2581 email = x
2583 server_url_base = "http://" + server + "/"
2585 testing = ui.config("codereview", "testing")
2586 force_google_account = ui.configbool("codereview", "force_google_account", False)
2588 upload_options = opt()
2589 upload_options.email = email
2590 upload_options.host = None
2591 upload_options.verbose = 0
2592 upload_options.description = None
2593 upload_options.description_file = None
2594 upload_options.reviewers = None
2595 upload_options.cc = None
2596 upload_options.message = None
2597 upload_options.issue = None
2598 upload_options.download_base = False
2599 upload_options.revision = None
2600 upload_options.send_mail = False
2601 upload_options.vcs = None
2602 upload_options.server = server
2603 upload_options.save_cookies = True
2605 if testing:
2606 upload_options.save_cookies = False
2607 upload_options.email = "test@example.com"
2609 rpc = None
2611 global releaseBranch
2612 tags = repo.branchtags().keys()
2613 if 'release-branch.go10' in tags:
2614 # NOTE(rsc): This tags.sort is going to get the wrong
2615 # answer when comparing release-branch.go9 with
2616 # release-branch.go10. It will be a while before we care.
2617 raise hg_util.Abort('tags.sort needs to be fixed for release-branch.go10')
2618 tags.sort()
2619 for t in tags:
2620 if t.startswith('release-branch.go'):
2621 releaseBranch = t
2623 #######################################################################
2624 # http://codereview.appspot.com/static/upload.py, heavily edited.
2626 #!/usr/bin/env python
2628 # Copyright 2007 Google Inc.
2630 # Licensed under the Apache License, Version 2.0 (the "License");
2631 # you may not use this file except in compliance with the License.
2632 # You may obtain a copy of the License at
2634 # http://www.apache.org/licenses/LICENSE-2.0
2636 # Unless required by applicable law or agreed to in writing, software
2637 # distributed under the License is distributed on an "AS IS" BASIS,
2638 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2639 # See the License for the specific language governing permissions and
2640 # limitations under the License.
2642 """Tool for uploading diffs from a version control system to the codereview app.
2644 Usage summary: upload.py [options] [-- diff_options]
2646 Diff options are passed to the diff command of the underlying system.
2648 Supported version control systems:
2650 Mercurial
2651 Subversion
2653 It is important for Git/Mercurial users to specify a tree/node/branch to diff
2654 against by using the '--rev' option.
2656 # This code is derived from appcfg.py in the App Engine SDK (open source),
2657 # and from ASPN recipe #146306.
2659 import cookielib
2660 import getpass
2661 import logging
2662 import mimetypes
2663 import optparse
2664 import os
2665 import re
2666 import socket
2667 import subprocess
2668 import sys
2669 import urllib
2670 import urllib2
2671 import urlparse
2673 # The md5 module was deprecated in Python 2.5.
2674 try:
2675 from hashlib import md5
2676 except ImportError:
2677 from md5 import md5
2679 try:
2680 import readline
2681 except ImportError:
2682 pass
2684 # The logging verbosity:
2685 # 0: Errors only.
2686 # 1: Status messages.
2687 # 2: Info logs.
2688 # 3: Debug logs.
2689 verbosity = 1
2691 # Max size of patch or base file.
2692 MAX_UPLOAD_SIZE = 900 * 1024
2694 # whitelist for non-binary filetypes which do not start with "text/"
2695 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
2696 TEXT_MIMETYPES = [
2697 'application/javascript',
2698 'application/x-javascript',
2699 'application/x-freemind'
2702 def GetEmail(prompt):
2703 """Prompts the user for their email address and returns it.
2705 The last used email address is saved to a file and offered up as a suggestion
2706 to the user. If the user presses enter without typing in anything the last
2707 used email address is used. If the user enters a new address, it is saved
2708 for next time we prompt.
2711 last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
2712 last_email = ""
2713 if os.path.exists(last_email_file_name):
2714 try:
2715 last_email_file = open(last_email_file_name, "r")
2716 last_email = last_email_file.readline().strip("\n")
2717 last_email_file.close()
2718 prompt += " [%s]" % last_email
2719 except IOError, e:
2720 pass
2721 email = raw_input(prompt + ": ").strip()
2722 if email:
2723 try:
2724 last_email_file = open(last_email_file_name, "w")
2725 last_email_file.write(email)
2726 last_email_file.close()
2727 except IOError, e:
2728 pass
2729 else:
2730 email = last_email
2731 return email
2734 def StatusUpdate(msg):
2735 """Print a status message to stdout.
2737 If 'verbosity' is greater than 0, print the message.
2739 Args:
2740 msg: The string to print.
2742 if verbosity > 0:
2743 print msg
2746 def ErrorExit(msg):
2747 """Print an error message to stderr and exit."""
2748 print >>sys.stderr, msg
2749 sys.exit(1)
2752 class ClientLoginError(urllib2.HTTPError):
2753 """Raised to indicate there was an error authenticating with ClientLogin."""
2755 def __init__(self, url, code, msg, headers, args):
2756 urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
2757 self.args = args
2758 self.reason = args["Error"]
2761 class AbstractRpcServer(object):
2762 """Provides a common interface for a simple RPC server."""
2764 def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False):
2765 """Creates a new HttpRpcServer.
2767 Args:
2768 host: The host to send requests to.
2769 auth_function: A function that takes no arguments and returns an
2770 (email, password) tuple when called. Will be called if authentication
2771 is required.
2772 host_override: The host header to send to the server (defaults to host).
2773 extra_headers: A dict of extra headers to append to every request.
2774 save_cookies: If True, save the authentication cookies to local disk.
2775 If False, use an in-memory cookiejar instead. Subclasses must
2776 implement this functionality. Defaults to False.
2778 self.host = host
2779 self.host_override = host_override
2780 self.auth_function = auth_function
2781 self.authenticated = False
2782 self.extra_headers = extra_headers
2783 self.save_cookies = save_cookies
2784 self.opener = self._GetOpener()
2785 if self.host_override:
2786 logging.info("Server: %s; Host: %s", self.host, self.host_override)
2787 else:
2788 logging.info("Server: %s", self.host)
2790 def _GetOpener(self):
2791 """Returns an OpenerDirector for making HTTP requests.
2793 Returns:
2794 A urllib2.OpenerDirector object.
2796 raise NotImplementedError()
2798 def _CreateRequest(self, url, data=None):
2799 """Creates a new urllib request."""
2800 logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
2801 req = urllib2.Request(url, data=data)
2802 if self.host_override:
2803 req.add_header("Host", self.host_override)
2804 for key, value in self.extra_headers.iteritems():
2805 req.add_header(key, value)
2806 return req
2808 def _GetAuthToken(self, email, password):
2809 """Uses ClientLogin to authenticate the user, returning an auth token.
2811 Args:
2812 email: The user's email address
2813 password: The user's password
2815 Raises:
2816 ClientLoginError: If there was an error authenticating with ClientLogin.
2817 HTTPError: If there was some other form of HTTP error.
2819 Returns:
2820 The authentication token returned by ClientLogin.
2822 account_type = "GOOGLE"
2823 if self.host.endswith(".google.com") and not force_google_account:
2824 # Needed for use inside Google.
2825 account_type = "HOSTED"
2826 req = self._CreateRequest(
2827 url="https://www.google.com/accounts/ClientLogin",
2828 data=urllib.urlencode({
2829 "Email": email,
2830 "Passwd": password,
2831 "service": "ah",
2832 "source": "rietveld-codereview-upload",
2833 "accountType": account_type,
2836 try:
2837 response = self.opener.open(req)
2838 response_body = response.read()
2839 response_dict = dict(x.split("=") for x in response_body.split("\n") if x)
2840 return response_dict["Auth"]
2841 except urllib2.HTTPError, e:
2842 if e.code == 403:
2843 body = e.read()
2844 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
2845 raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict)
2846 else:
2847 raise
2849 def _GetAuthCookie(self, auth_token):
2850 """Fetches authentication cookies for an authentication token.
2852 Args:
2853 auth_token: The authentication token returned by ClientLogin.
2855 Raises:
2856 HTTPError: If there was an error fetching the authentication cookies.
2858 # This is a dummy value to allow us to identify when we're successful.
2859 continue_location = "http://localhost/"
2860 args = {"continue": continue_location, "auth": auth_token}
2861 req = self._CreateRequest("http://%s/_ah/login?%s" % (self.host, urllib.urlencode(args)))
2862 try:
2863 response = self.opener.open(req)
2864 except urllib2.HTTPError, e:
2865 response = e
2866 if (response.code != 302 or
2867 response.info()["location"] != continue_location):
2868 raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp)
2869 self.authenticated = True
2871 def _Authenticate(self):
2872 """Authenticates the user.
2874 The authentication process works as follows:
2875 1) We get a username and password from the user
2876 2) We use ClientLogin to obtain an AUTH token for the user
2877 (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
2878 3) We pass the auth token to /_ah/login on the server to obtain an
2879 authentication cookie. If login was successful, it tries to redirect
2880 us to the URL we provided.
2882 If we attempt to access the upload API without first obtaining an
2883 authentication cookie, it returns a 401 response (or a 302) and
2884 directs us to authenticate ourselves with ClientLogin.
2886 for i in range(3):
2887 credentials = self.auth_function()
2888 try:
2889 auth_token = self._GetAuthToken(credentials[0], credentials[1])
2890 except ClientLoginError, e:
2891 if e.reason == "BadAuthentication":
2892 print >>sys.stderr, "Invalid username or password."
2893 continue
2894 if e.reason == "CaptchaRequired":
2895 print >>sys.stderr, (
2896 "Please go to\n"
2897 "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
2898 "and verify you are a human. Then try again.")
2899 break
2900 if e.reason == "NotVerified":
2901 print >>sys.stderr, "Account not verified."
2902 break
2903 if e.reason == "TermsNotAgreed":
2904 print >>sys.stderr, "User has not agreed to TOS."
2905 break
2906 if e.reason == "AccountDeleted":
2907 print >>sys.stderr, "The user account has been deleted."
2908 break
2909 if e.reason == "AccountDisabled":
2910 print >>sys.stderr, "The user account has been disabled."
2911 break
2912 if e.reason == "ServiceDisabled":
2913 print >>sys.stderr, "The user's access to the service has been disabled."
2914 break
2915 if e.reason == "ServiceUnavailable":
2916 print >>sys.stderr, "The service is not available; try again later."
2917 break
2918 raise
2919 self._GetAuthCookie(auth_token)
2920 return
2922 def Send(self, request_path, payload=None,
2923 content_type="application/octet-stream",
2924 timeout=None,
2925 **kwargs):
2926 """Sends an RPC and returns the response.
2928 Args:
2929 request_path: The path to send the request to, eg /api/appversion/create.
2930 payload: The body of the request, or None to send an empty request.
2931 content_type: The Content-Type header to use.
2932 timeout: timeout in seconds; default None i.e. no timeout.
2933 (Note: for large requests on OS X, the timeout doesn't work right.)
2934 kwargs: Any keyword arguments are converted into query string parameters.
2936 Returns:
2937 The response body, as a string.
2939 # TODO: Don't require authentication. Let the server say
2940 # whether it is necessary.
2941 if not self.authenticated:
2942 self._Authenticate()
2944 old_timeout = socket.getdefaulttimeout()
2945 socket.setdefaulttimeout(timeout)
2946 try:
2947 tries = 0
2948 while True:
2949 tries += 1
2950 args = dict(kwargs)
2951 url = "http://%s%s" % (self.host, request_path)
2952 if args:
2953 url += "?" + urllib.urlencode(args)
2954 req = self._CreateRequest(url=url, data=payload)
2955 req.add_header("Content-Type", content_type)
2956 try:
2957 f = self.opener.open(req)
2958 response = f.read()
2959 f.close()
2960 return response
2961 except urllib2.HTTPError, e:
2962 if tries > 3:
2963 raise
2964 elif e.code == 401 or e.code == 302:
2965 self._Authenticate()
2966 else:
2967 raise
2968 finally:
2969 socket.setdefaulttimeout(old_timeout)
2972 class HttpRpcServer(AbstractRpcServer):
2973 """Provides a simplified RPC-style interface for HTTP requests."""
2975 def _Authenticate(self):
2976 """Save the cookie jar after authentication."""
2977 super(HttpRpcServer, self)._Authenticate()
2978 if self.save_cookies:
2979 StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
2980 self.cookie_jar.save()
2982 def _GetOpener(self):
2983 """Returns an OpenerDirector that supports cookies and ignores redirects.
2985 Returns:
2986 A urllib2.OpenerDirector object.
2988 opener = urllib2.OpenerDirector()
2989 opener.add_handler(urllib2.ProxyHandler())
2990 opener.add_handler(urllib2.UnknownHandler())
2991 opener.add_handler(urllib2.HTTPHandler())
2992 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
2993 opener.add_handler(urllib2.HTTPSHandler())
2994 opener.add_handler(urllib2.HTTPErrorProcessor())
2995 if self.save_cookies:
2996 self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
2997 self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
2998 if os.path.exists(self.cookie_file):
2999 try:
3000 self.cookie_jar.load()
3001 self.authenticated = True
3002 StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file)
3003 except (cookielib.LoadError, IOError):
3004 # Failed to load cookies - just ignore them.
3005 pass
3006 else:
3007 # Create an empty cookie file with mode 600
3008 fd = os.open(self.cookie_file, os.O_CREAT, 0600)
3009 os.close(fd)
3010 # Always chmod the cookie file
3011 os.chmod(self.cookie_file, 0600)
3012 else:
3013 # Don't save cookies across runs of update.py.
3014 self.cookie_jar = cookielib.CookieJar()
3015 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
3016 return opener
3019 def GetRpcServer(options):
3020 """Returns an instance of an AbstractRpcServer.
3022 Returns:
3023 A new AbstractRpcServer, on which RPC calls can be made.
3026 rpc_server_class = HttpRpcServer
3028 def GetUserCredentials():
3029 """Prompts the user for a username and password."""
3030 # Disable status prints so they don't obscure the password prompt.
3031 global global_status
3032 st = global_status
3033 global_status = None
3035 email = options.email
3036 if email is None:
3037 email = GetEmail("Email (login for uploading to %s)" % options.server)
3038 password = getpass.getpass("Password for %s: " % email)
3040 # Put status back.
3041 global_status = st
3042 return (email, password)
3044 # If this is the dev_appserver, use fake authentication.
3045 host = (options.host or options.server).lower()
3046 if host == "localhost" or host.startswith("localhost:"):
3047 email = options.email
3048 if email is None:
3049 email = "test@example.com"
3050 logging.info("Using debug user %s. Override with --email" % email)
3051 server = rpc_server_class(
3052 options.server,
3053 lambda: (email, "password"),
3054 host_override=options.host,
3055 extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
3056 save_cookies=options.save_cookies)
3057 # Don't try to talk to ClientLogin.
3058 server.authenticated = True
3059 return server
3061 return rpc_server_class(options.server, GetUserCredentials,
3062 host_override=options.host, save_cookies=options.save_cookies)
3065 def EncodeMultipartFormData(fields, files):
3066 """Encode form fields for multipart/form-data.
3068 Args:
3069 fields: A sequence of (name, value) elements for regular form fields.
3070 files: A sequence of (name, filename, value) elements for data to be
3071 uploaded as files.
3072 Returns:
3073 (content_type, body) ready for httplib.HTTP instance.
3075 Source:
3076 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
3078 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
3079 CRLF = '\r\n'
3080 lines = []
3081 for (key, value) in fields:
3082 typecheck(key, str)
3083 typecheck(value, str)
3084 lines.append('--' + BOUNDARY)
3085 lines.append('Content-Disposition: form-data; name="%s"' % key)
3086 lines.append('')
3087 lines.append(value)
3088 for (key, filename, value) in files:
3089 typecheck(key, str)
3090 typecheck(filename, str)
3091 typecheck(value, str)
3092 lines.append('--' + BOUNDARY)
3093 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
3094 lines.append('Content-Type: %s' % GetContentType(filename))
3095 lines.append('')
3096 lines.append(value)
3097 lines.append('--' + BOUNDARY + '--')
3098 lines.append('')
3099 body = CRLF.join(lines)
3100 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
3101 return content_type, body
3104 def GetContentType(filename):
3105 """Helper to guess the content-type from the filename."""
3106 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
3109 # Use a shell for subcommands on Windows to get a PATH search.
3110 use_shell = sys.platform.startswith("win")
3112 def RunShellWithReturnCode(command, print_output=False,
3113 universal_newlines=True, env=os.environ):
3114 """Executes a command and returns the output from stdout and the return code.
3116 Args:
3117 command: Command to execute.
3118 print_output: If True, the output is printed to stdout.
3119 If False, both stdout and stderr are ignored.
3120 universal_newlines: Use universal_newlines flag (default: True).
3122 Returns:
3123 Tuple (output, return code)
3125 logging.info("Running %s", command)
3126 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
3127 shell=use_shell, universal_newlines=universal_newlines, env=env)
3128 if print_output:
3129 output_array = []
3130 while True:
3131 line = p.stdout.readline()
3132 if not line:
3133 break
3134 print line.strip("\n")
3135 output_array.append(line)
3136 output = "".join(output_array)
3137 else:
3138 output = p.stdout.read()
3139 p.wait()
3140 errout = p.stderr.read()
3141 if print_output and errout:
3142 print >>sys.stderr, errout
3143 p.stdout.close()
3144 p.stderr.close()
3145 return output, p.returncode
3148 def RunShell(command, silent_ok=False, universal_newlines=True,
3149 print_output=False, env=os.environ):
3150 data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env)
3151 if retcode:
3152 ErrorExit("Got error status from %s:\n%s" % (command, data))
3153 if not silent_ok and not data:
3154 ErrorExit("No output from %s" % command)
3155 return data
3158 class VersionControlSystem(object):
3159 """Abstract base class providing an interface to the VCS."""
3161 def __init__(self, options):
3162 """Constructor.
3164 Args:
3165 options: Command line options.
3167 self.options = options
3169 def GenerateDiff(self, args):
3170 """Return the current diff as a string.
3172 Args:
3173 args: Extra arguments to pass to the diff command.
3175 raise NotImplementedError(
3176 "abstract method -- subclass %s must override" % self.__class__)
3178 def GetUnknownFiles(self):
3179 """Return a list of files unknown to the VCS."""
3180 raise NotImplementedError(
3181 "abstract method -- subclass %s must override" % self.__class__)
3183 def CheckForUnknownFiles(self):
3184 """Show an "are you sure?" prompt if there are unknown files."""
3185 unknown_files = self.GetUnknownFiles()
3186 if unknown_files:
3187 print "The following files are not added to version control:"
3188 for line in unknown_files:
3189 print line
3190 prompt = "Are you sure to continue?(y/N) "
3191 answer = raw_input(prompt).strip()
3192 if answer != "y":
3193 ErrorExit("User aborted")
3195 def GetBaseFile(self, filename):
3196 """Get the content of the upstream version of a file.
3198 Returns:
3199 A tuple (base_content, new_content, is_binary, status)
3200 base_content: The contents of the base file.
3201 new_content: For text files, this is empty. For binary files, this is
3202 the contents of the new file, since the diff output won't contain
3203 information to reconstruct the current file.
3204 is_binary: True iff the file is binary.
3205 status: The status of the file.
3208 raise NotImplementedError(
3209 "abstract method -- subclass %s must override" % self.__class__)
3212 def GetBaseFiles(self, diff):
3213 """Helper that calls GetBase file for each file in the patch.
3215 Returns:
3216 A dictionary that maps from filename to GetBaseFile's tuple. Filenames
3217 are retrieved based on lines that start with "Index:" or
3218 "Property changes on:".
3220 files = {}
3221 for line in diff.splitlines(True):
3222 if line.startswith('Index:') or line.startswith('Property changes on:'):
3223 unused, filename = line.split(':', 1)
3224 # On Windows if a file has property changes its filename uses '\'
3225 # instead of '/'.
3226 filename = to_slash(filename.strip())
3227 files[filename] = self.GetBaseFile(filename)
3228 return files
3231 def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
3232 files):
3233 """Uploads the base files (and if necessary, the current ones as well)."""
3235 def UploadFile(filename, file_id, content, is_binary, status, is_base):
3236 """Uploads a file to the server."""
3237 set_status("uploading " + filename)
3238 file_too_large = False
3239 if is_base:
3240 type = "base"
3241 else:
3242 type = "current"
3243 if len(content) > MAX_UPLOAD_SIZE:
3244 print ("Not uploading the %s file for %s because it's too large." %
3245 (type, filename))
3246 file_too_large = True
3247 content = ""
3248 checksum = md5(content).hexdigest()
3249 if options.verbose > 0 and not file_too_large:
3250 print "Uploading %s file for %s" % (type, filename)
3251 url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
3252 form_fields = [
3253 ("filename", filename),
3254 ("status", status),
3255 ("checksum", checksum),
3256 ("is_binary", str(is_binary)),
3257 ("is_current", str(not is_base)),
3259 if file_too_large:
3260 form_fields.append(("file_too_large", "1"))
3261 if options.email:
3262 form_fields.append(("user", options.email))
3263 ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)])
3264 response_body = rpc_server.Send(url, body, content_type=ctype)
3265 if not response_body.startswith("OK"):
3266 StatusUpdate(" --> %s" % response_body)
3267 sys.exit(1)
3269 # Don't want to spawn too many threads, nor do we want to
3270 # hit Rietveld too hard, or it will start serving 500 errors.
3271 # When 8 works, it's no better than 4, and sometimes 8 is
3272 # too many for Rietveld to handle.
3273 MAX_PARALLEL_UPLOADS = 4
3275 sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS)
3276 upload_threads = []
3277 finished_upload_threads = []
3279 class UploadFileThread(threading.Thread):
3280 def __init__(self, args):
3281 threading.Thread.__init__(self)
3282 self.args = args
3283 def run(self):
3284 UploadFile(*self.args)
3285 finished_upload_threads.append(self)
3286 sema.release()
3288 def StartUploadFile(*args):
3289 sema.acquire()
3290 while len(finished_upload_threads) > 0:
3291 t = finished_upload_threads.pop()
3292 upload_threads.remove(t)
3293 t.join()
3294 t = UploadFileThread(args)
3295 upload_threads.append(t)
3296 t.start()
3298 def WaitForUploads():
3299 for t in upload_threads:
3300 t.join()
3302 patches = dict()
3303 [patches.setdefault(v, k) for k, v in patch_list]
3304 for filename in patches.keys():
3305 base_content, new_content, is_binary, status = files[filename]
3306 file_id_str = patches.get(filename)
3307 if file_id_str.find("nobase") != -1:
3308 base_content = None
3309 file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
3310 file_id = int(file_id_str)
3311 if base_content != None:
3312 StartUploadFile(filename, file_id, base_content, is_binary, status, True)
3313 if new_content != None:
3314 StartUploadFile(filename, file_id, new_content, is_binary, status, False)
3315 WaitForUploads()
3317 def IsImage(self, filename):
3318 """Returns true if the filename has an image extension."""
3319 mimetype = mimetypes.guess_type(filename)[0]
3320 if not mimetype:
3321 return False
3322 return mimetype.startswith("image/")
3324 def IsBinary(self, filename):
3325 """Returns true if the guessed mimetyped isnt't in text group."""
3326 mimetype = mimetypes.guess_type(filename)[0]
3327 if not mimetype:
3328 return False # e.g. README, "real" binaries usually have an extension
3329 # special case for text files which don't start with text/
3330 if mimetype in TEXT_MIMETYPES:
3331 return False
3332 return not mimetype.startswith("text/")
3335 class FakeMercurialUI(object):
3336 def __init__(self):
3337 self.quiet = True
3338 self.output = ''
3340 def write(self, *args, **opts):
3341 self.output += ' '.join(args)
3342 def copy(self):
3343 return self
3344 def status(self, *args, **opts):
3345 pass
3347 def formatter(self, topic, opts):
3348 from mercurial.formatter import plainformatter
3349 return plainformatter(self, topic, opts)
3351 def readconfig(self, *args, **opts):
3352 pass
3353 def expandpath(self, *args, **opts):
3354 return global_ui.expandpath(*args, **opts)
3355 def configitems(self, *args, **opts):
3356 return global_ui.configitems(*args, **opts)
3357 def config(self, *args, **opts):
3358 return global_ui.config(*args, **opts)
3360 use_hg_shell = False # set to True to shell out to hg always; slower
3362 class MercurialVCS(VersionControlSystem):
3363 """Implementation of the VersionControlSystem interface for Mercurial."""
3365 def __init__(self, options, ui, repo):
3366 super(MercurialVCS, self).__init__(options)
3367 self.ui = ui
3368 self.repo = repo
3369 self.status = None
3370 # Absolute path to repository (we can be in a subdir)
3371 self.repo_dir = os.path.normpath(repo.root)
3372 # Compute the subdir
3373 cwd = os.path.normpath(os.getcwd())
3374 assert cwd.startswith(self.repo_dir)
3375 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
3376 if self.options.revision:
3377 self.base_rev = self.options.revision
3378 else:
3379 mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
3380 if not err and mqparent != "":
3381 self.base_rev = mqparent
3382 else:
3383 out = RunShell(["hg", "parents", "-q"], silent_ok=True).strip()
3384 if not out:
3385 # No revisions; use 0 to mean a repository with nothing.
3386 out = "0:0"
3387 self.base_rev = out.split(':')[1].strip()
3388 def _GetRelPath(self, filename):
3389 """Get relative path of a file according to the current directory,
3390 given its logical path in the repo."""
3391 assert filename.startswith(self.subdir), (filename, self.subdir)
3392 return filename[len(self.subdir):].lstrip(r"\/")
3394 def GenerateDiff(self, extra_args):
3395 # If no file specified, restrict to the current subdir
3396 extra_args = extra_args or ["."]
3397 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
3398 data = RunShell(cmd, silent_ok=True)
3399 svndiff = []
3400 filecount = 0
3401 for line in data.splitlines():
3402 m = re.match("diff --git a/(\S+) b/(\S+)", line)
3403 if m:
3404 # Modify line to make it look like as it comes from svn diff.
3405 # With this modification no changes on the server side are required
3406 # to make upload.py work with Mercurial repos.
3407 # NOTE: for proper handling of moved/copied files, we have to use
3408 # the second filename.
3409 filename = m.group(2)
3410 svndiff.append("Index: %s" % filename)
3411 svndiff.append("=" * 67)
3412 filecount += 1
3413 logging.info(line)
3414 else:
3415 svndiff.append(line)
3416 if not filecount:
3417 ErrorExit("No valid patches found in output from hg diff")
3418 return "\n".join(svndiff) + "\n"
3420 def GetUnknownFiles(self):
3421 """Return a list of files unknown to the VCS."""
3422 args = []
3423 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
3424 silent_ok=True)
3425 unknown_files = []
3426 for line in status.splitlines():
3427 st, fn = line.split(" ", 1)
3428 if st == "?":
3429 unknown_files.append(fn)
3430 return unknown_files
3432 def get_hg_status(self, rev, path):
3433 # We'd like to use 'hg status -C path', but that is buggy
3434 # (see http://mercurial.selenic.com/bts/issue3023).
3435 # Instead, run 'hg status -C' without a path
3436 # and skim the output for the path we want.
3437 if self.status is None:
3438 if use_hg_shell:
3439 out = RunShell(["hg", "status", "-C", "--rev", rev])
3440 else:
3441 fui = FakeMercurialUI()
3442 ret = hg_commands.status(fui, self.repo, *[], **{'rev': [rev], 'copies': True})
3443 if ret:
3444 raise hg_util.Abort(ret)
3445 out = fui.output
3446 self.status = out.splitlines()
3447 for i in range(len(self.status)):
3448 # line is
3449 # A path
3450 # M path
3451 # etc
3452 line = to_slash(self.status[i])
3453 if line[2:] == path:
3454 if i+1 < len(self.status) and self.status[i+1][:2] == ' ':
3455 return self.status[i:i+2]
3456 return self.status[i:i+1]
3457 raise hg_util.Abort("no status for " + path)
3459 def GetBaseFile(self, filename):
3460 set_status("inspecting " + filename)
3461 # "hg status" and "hg cat" both take a path relative to the current subdir
3462 # rather than to the repo root, but "hg diff" has given us the full path
3463 # to the repo root.
3464 base_content = ""
3465 new_content = None
3466 is_binary = False
3467 oldrelpath = relpath = self._GetRelPath(filename)
3468 out = self.get_hg_status(self.base_rev, relpath)
3469 status, what = out[0].split(' ', 1)
3470 if len(out) > 1 and status == "A" and what == relpath:
3471 oldrelpath = out[1].strip()
3472 status = "M"
3473 if ":" in self.base_rev:
3474 base_rev = self.base_rev.split(":", 1)[0]
3475 else:
3476 base_rev = self.base_rev
3477 if status != "A":
3478 if use_hg_shell:
3479 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True)
3480 else:
3481 base_content = str(self.repo[base_rev][oldrelpath].data())
3482 is_binary = "\0" in base_content # Mercurial's heuristic
3483 if status != "R":
3484 new_content = open(relpath, "rb").read()
3485 is_binary = is_binary or "\0" in new_content
3486 if is_binary and base_content and use_hg_shell:
3487 # Fetch again without converting newlines
3488 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
3489 silent_ok=True, universal_newlines=False)
3490 if not is_binary or not self.IsImage(relpath):
3491 new_content = None
3492 return base_content, new_content, is_binary, status
3495 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
3496 def SplitPatch(data):
3497 """Splits a patch into separate pieces for each file.
3499 Args:
3500 data: A string containing the output of svn diff.
3502 Returns:
3503 A list of 2-tuple (filename, text) where text is the svn diff output
3504 pertaining to filename.
3506 patches = []
3507 filename = None
3508 diff = []
3509 for line in data.splitlines(True):
3510 new_filename = None
3511 if line.startswith('Index:'):
3512 unused, new_filename = line.split(':', 1)
3513 new_filename = new_filename.strip()
3514 elif line.startswith('Property changes on:'):
3515 unused, temp_filename = line.split(':', 1)
3516 # When a file is modified, paths use '/' between directories, however
3517 # when a property is modified '\' is used on Windows. Make them the same
3518 # otherwise the file shows up twice.
3519 temp_filename = to_slash(temp_filename.strip())
3520 if temp_filename != filename:
3521 # File has property changes but no modifications, create a new diff.
3522 new_filename = temp_filename
3523 if new_filename:
3524 if filename and diff:
3525 patches.append((filename, ''.join(diff)))
3526 filename = new_filename
3527 diff = [line]
3528 continue
3529 if diff is not None:
3530 diff.append(line)
3531 if filename and diff:
3532 patches.append((filename, ''.join(diff)))
3533 return patches
3536 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3537 """Uploads a separate patch for each file in the diff output.
3539 Returns a list of [patch_key, filename] for each file.
3541 patches = SplitPatch(data)
3542 rv = []
3543 for patch in patches:
3544 set_status("uploading patch for " + patch[0])
3545 if len(patch[1]) > MAX_UPLOAD_SIZE:
3546 print ("Not uploading the patch for " + patch[0] +
3547 " because the file is too large.")
3548 continue
3549 form_fields = [("filename", patch[0])]
3550 if not options.download_base:
3551 form_fields.append(("content_upload", "1"))
3552 files = [("data", "data.diff", patch[1])]
3553 ctype, body = EncodeMultipartFormData(form_fields, files)
3554 url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3555 print "Uploading patch for " + patch[0]
3556 response_body = rpc_server.Send(url, body, content_type=ctype)
3557 lines = response_body.splitlines()
3558 if not lines or lines[0] != "OK":
3559 StatusUpdate(" --> %s" % response_body)
3560 sys.exit(1)
3561 rv.append([lines[1], patch[0]])
3562 return rv