thirdparty/chromium/gcl.py

   1 #!/usr/bin/python
   2 # Wrapper script around Rietveld's upload.py that groups files into
   3 # changelists.
   4
   5 import getpass
   6 import linecache
   7 import os
   8 import random
   9 import re
  10 import string
  11 import subprocess
  12 import sys
  13 import tempfile
  14 import upload
  15 import urllib2
  16
  17 CODEREVIEW_SETTINGS = {
  18   # Default values.
  19   "CODE_REVIEW_SERVER": "codereviews.googleopensourceprograms.com",
  20   "CC_LIST": "melange-soc-dev@googlegroups.com",
  21   "VIEW_VC": "http://code.google.com/p/soc/source/detail?r=",
  22 }
  23
  24 # Use a shell for subcommands on Windows to get a PATH search, and because svn
  25 # may be a batch file.
  26 use_shell = sys.platform.startswith("win")
  27
  28
  29 # globals that store the root of the current repositary and the directory where
  30 # we store information about changelists.
  31 repository_root = ""
  32 gcl_info_dir = ""
  33
  34
  35 def GetSVNFileInfo(file, field):
  36   """Returns a field from the svn info output for the given file."""
  37   output = RunShell(["svn", "info", file])
  38   for line in output.splitlines():
  39     search = field + ": "
  40     if line.startswith(search):
  41       return line[len(search):]
  42   return ""
  43
  44
  45 def GetRepositoryRoot():
  46   """Returns the top level directory of the current repository."""
  47   global repository_root
  48   if not repository_root:
  49     cur_dir_repo_root = GetSVNFileInfo(os.getcwd(), "Repository Root")
  50     if not cur_dir_repo_root:
  51       ErrorExit("gcl run outside of repository")
  52
  53     repository_root = os.getcwd()
  54     while True:
  55       parent = os.path.dirname(repository_root)
  56       if GetSVNFileInfo(parent, "Repository Root") != cur_dir_repo_root:
  57         break
  58       repository_root = parent
  59   # Now read the code review settings for this repository.
  60   settings_file = os.path.join(repository_root, "codereview.settings")
  61   if os.path.exists(settings_file):
  62     output = ReadFile(settings_file)
  63     for line in output.splitlines():
  64       if not line or line.startswith("#"):
  65         continue
  66       key, value = line.split(": ", 1)
  67       CODEREVIEW_SETTINGS[key] = value
  68   return repository_root
  69
  70
  71 def GetCodeReviewSetting(key):
  72   """Returns a value for the given key for this repository."""
  73   return CODEREVIEW_SETTINGS.get(key, "")
  74
  75
  76 def GetInfoDir():
  77   """Returns the directory where gcl info files are stored."""
  78   global gcl_info_dir
  79   if not gcl_info_dir:
  80     gcl_info_dir = os.path.join(GetRepositoryRoot(), '.svn', 'gcl_info')
  81   return gcl_info_dir
  82
  83
  84 def ErrorExit(msg):
  85   """Print an error message to stderr and exit."""
  86   print >>sys.stderr, msg
  87   sys.exit(1)
  88
  89
  90 def RunShell(command, print_output=False):
  91   """Executes a command and returns the output."""
  92   p = subprocess.Popen(command, stdout = subprocess.PIPE,
  93                        stderr = subprocess.STDOUT, shell = use_shell,
  94                        universal_newlines=True)
  95   if print_output:
  96     output_array = []
  97     while True:
  98       line = p.stdout.readline()
  99       if not line:
 100         break
 101       if print_output:
 102         print line.strip('\n')
 103       output_array.append(line)
 104     output = "".join(output_array)
 105   else:
 106     output = p.stdout.read()
 107   p.wait()
 108   p.stdout.close()
 109   return output
 110
 111
 112 def ReadFile(filename):
 113   """Returns the contents of a file."""
 114   file = open(filename, 'r')
 115   result = file.read()
 116   file.close()
 117   return result
 118
 119
 120 def WriteFile(filename, contents):
 121   """Overwrites the file with the given contents."""
 122   file = open(filename, 'w')
 123   file.write(contents)
 124   file.close()
 125
 126
 127 class ChangeInfo:
 128   """Holds information about a changelist.
 129
 130     issue: the Rietveld issue number, of "" if it hasn't been uploaded yet.
 131     description: the description.
 132     files: a list of 2 tuple containing (status, filename) of changed files,
 133            with paths being relative to the top repository directory.
 134   """
 135   def __init__(self, name="", issue="", description="", files=[]):
 136     self.name = name
 137     self.issue = issue
 138     self.description = description
 139     self.files = files
 140
 141   def FileList(self):
 142     """Returns a list of files."""
 143     return [file[1] for file in self.files]
 144
 145   def Save(self):
 146     """Writes the changelist information to disk."""
 147     data = SEPARATOR.join([self.issue,
 148                           "\n".join([f[0] + f[1] for f in self.files]),
 149                           self.description])
 150     WriteFile(GetChangelistInfoFile(self.name), data)
 151
 152   def Delete(self):
 153     """Removes the changelist information from disk."""
 154     os.remove(GetChangelistInfoFile(self.name))
 155
 156   def CloseIssue(self):
 157     """Closes the Rietveld issue for this changelist."""
 158     data = [("description", self.description),]
 159     ctype, body = upload.EncodeMultipartFormData(data, [])
 160     SendToRietveld("/" + self.issue + "/close", body, ctype)
 161
 162   def UpdateRietveldDescription(self):
 163     """Sets the description for an issue on Rietveld."""
 164     data = [("description", self.description),]
 165     ctype, body = upload.EncodeMultipartFormData(data, [])
 166     SendToRietveld("/" + self.issue + "/description", body, ctype)
 167
 168
 169 SEPARATOR = "\n-----\n"
 170 # The info files have the following format:
 171 # issue_id\n
 172 # SEPARATOR\n
 173 # filepath1\n
 174 # filepath2\n
 175 # .
 176 # .
 177 # filepathn\n
 178 # SEPARATOR\n
 179 # description
 180
 181
 182 def GetChangelistInfoFile(changename):
 183   """Returns the file that stores information about a changelist."""
 184   if not changename or re.search(r'\W', changename):
 185     ErrorExit("Invalid changelist name: " + changename)
 186   return os.path.join(GetInfoDir(), changename)
 187
 188
 189 def LoadChangelistInfo(changename, fail_on_not_found=True,
 190                        update_status=False):
 191   """Gets information about a changelist.
 192
 193   Args:
 194     fail_on_not_found: if True, this function will quit the program if the
 195       changelist doesn't exist.
 196     update_status: if True, the svn status will be updated for all the files
 197       and unchanged files will be removed.
 198
 199   Returns: a ChangeInfo object.
 200   """
 201   info_file = GetChangelistInfoFile(changename)
 202   if not os.path.exists(info_file):
 203     if fail_on_not_found:
 204       ErrorExit("Changelist " + changename + " not found.")
 205     return ChangeInfo(changename)
 206   data = ReadFile(info_file)
 207   split_data = data.split(SEPARATOR, 2)
 208   if len(split_data) != 3:
 209     os.remove(info_file)
 210     ErrorExit("Changelist file %s was corrupt and deleted" % info_file)
 211   issue = split_data[0]
 212   files = []
 213   for line in split_data[1].splitlines():
 214     status = line[:7]
 215     file = line[7:]
 216     files.append((status, file))
 217   description = split_data[2]
 218   save = False
 219   if update_status:
 220     for file in files:
 221       filename = os.path.join(GetRepositoryRoot(), file[1])
 222       status = RunShell(["svn", "status", filename])[:7]
 223       if not status:  # File has been reverted.
 224         save = True
 225         files.remove(file)
 226       elif status != file[0]:
 227         save = True
 228         files[files.index(file)] = (status, file[1])
 229   change_info = ChangeInfo(changename, issue, description, files)
 230   if save:
 231     change_info.Save()
 232   return change_info
 233
 234
 235 def GetCLs():
 236   """Returns a list of all the changelists in this repository."""
 237   return os.listdir(GetInfoDir())
 238
 239
 240 def GenerateChangeName():
 241   """Generate a random changelist name."""
 242   random.seed()
 243   current_cl_names = GetCLs()
 244   while True:
 245     cl_name = (random.choice(string.ascii_lowercase) +
 246                random.choice(string.digits) +
 247                random.choice(string.ascii_lowercase) +
 248                random.choice(string.digits))
 249     if cl_name not in current_cl_names:
 250       return cl_name
 251
 252
 253 def GetModifiedFiles():
 254   """Returns a set that maps from changelist name to (status,filename) tuples.
 255
 256   Files not in a changelist have an empty changelist name.  Filenames are in
 257   relation to the top level directory of the current repositary.  Note that
 258   only the current directory and subdirectories are scanned, in order to
 259   improve performance while still being flexible.
 260   """
 261   files = {}
 262
 263   # Since the files are normalized to the root folder of the repositary, figure
 264   # out what we need to add to the paths.
 265   dir_prefix = os.getcwd()[len(GetRepositoryRoot()):].strip(os.sep)
 266
 267   # Get a list of all files in changelists.
 268   files_in_cl = {}
 269   for cl in GetCLs():
 270     change_info = LoadChangelistInfo(cl)
 271     for status, filename in change_info.files:
 272       files_in_cl[filename] = change_info.name
 273
 274   # Get all the modified files.
 275   status = RunShell(["svn", "status"])
 276   for line in status.splitlines():
 277     if not len(line) or line[0] == "?":
 278       continue
 279     status = line[:7]
 280     filename = line[7:]
 281     if dir_prefix:
 282       filename = os.path.join(dir_prefix, filename)
 283     change_list_name = ""
 284     if filename in files_in_cl:
 285       change_list_name = files_in_cl[filename]
 286     files.setdefault(change_list_name, []).append((status, filename))
 287
 288   return files
 289
 290
 291 def GetFilesNotInCL():
 292   """Returns a list of tuples (status,filename) that aren't in any changelists.
 293
 294   See docstring of GetModifiedFiles for information about path of files and
 295   which directories are scanned.
 296   """
 297   modified_files = GetModifiedFiles()
 298   if "" not in modified_files:
 299     return []
 300   return modified_files[""]
 301
 302
 303 def SendToRietveld(request_path, payload=None,
 304                    content_type="application/octet-stream"):
 305   """Send a POST/GET to Rietveld.  Returns the response body."""
 306   def GetUserCredentials():
 307     """Prompts the user for a username and password."""
 308     email = raw_input("Email: ").strip()
 309     password = getpass.getpass("Password for %s: " % email)
 310     return email, password
 311
 312   server = GetCodeReviewSetting("CODE_REVIEW_SERVER")
 313   rpc_server = upload.HttpRpcServer(server,
 314                                     GetUserCredentials,
 315                                     host_override=server,
 316                                     save_cookies=True)
 317   return rpc_server.Send(request_path, payload, content_type)
 318
 319
 320 def GetIssueDescription(issue):
 321   """Returns the issue description from Rietveld."""
 322   return SendToRietveld("/" + issue + "/description")
 323
 324
 325 def UnknownFiles(extra_args):
 326   """Runs svn status and prints unknown files.
 327
 328   Any args in |extra_args| are passed to the tool to support giving alternate
 329   code locations.
 330   """
 331   args = ["svn", "status"]
 332   args += extra_args
 333   p = subprocess.Popen(args, stdout = subprocess.PIPE,
 334                        stderr = subprocess.STDOUT, shell = use_shell)
 335   while 1:
 336     line = p.stdout.readline()
 337     if not line:
 338       break
 339     if line[0] != '?':
 340       continue  # Not an unknown file to svn.
 341     # The lines look like this:
 342     # "?      foo.txt"
 343     # and we want just "foo.txt"
 344     print line[7:].strip()
 345   p.wait()
 346   p.stdout.close()
 347
 348
 349 def Opened():
 350   """Prints a list of modified files in the current directory down."""
 351   files = GetModifiedFiles()
 352   cl_keys = files.keys()
 353   cl_keys.sort()
 354   for cl_name in cl_keys:
 355     if cl_name:
 356       note = ""
 357       if len(LoadChangelistInfo(cl_name).files) != len(files[cl_name]):
 358         note = " (Note: this changelist contains files outside this directory)"
 359       print "\n--- Changelist " + cl_name + note + ":"
 360     for file in files[cl_name]:
 361       print "".join(file)
 362
 363
 364 def Help():
 365   print ("GCL is a wrapper for Subversion that simplifies working with groups "
 366          "of files.\n")
 367   print "Basic commands:"
 368   print "-----------------------------------------"
 369   print "   gcl change change_name"
 370   print ("      Add/remove files to a changelist.  Only scans the current "
 371          "directory and subdirectories.\n")
 372   print ("   gcl upload change_name [-r reviewer1@gmail.com,"
 373          "reviewer2@gmail.com,...] [--send_mail]")
 374   print "      Uploads the changelist to the server for review.\n"
 375   print "   gcl commit change_name"
 376   print "      Commits the changelist to the repository.\n"
 377   print "Advanced commands:"
 378   print "-----------------------------------------"
 379   print "   gcl delete change_name"
 380   print "      Deletes a changelist.\n"
 381   print "   gcl diff change_name"
 382   print "      Diffs all files in the changelist.\n"
 383   print "   gcl diff"
 384   print ("      Diffs all files in the current directory and subdirectories "
 385          "that aren't in a changelist.\n")
 386   print "   gcl changes"
 387   print "      Lists all the the changelists and the files in them.\n"
 388   print "   gcl nothave [optional directory]"
 389   print "      Lists files unknown to Subversion.\n"
 390   print "   gcl opened"
 391   print ("      Lists modified files in the current directory and "
 392          "subdirectories.\n")
 393   print "   gcl try change_name"
 394   print ("      Sends the change to the tryserver so a trybot can do a test"
 395          " run on your code.\n")
 396
 397
 398 def GetEditor():
 399   editor = os.environ.get("SVN_EDITOR")
 400   if not editor:
 401     editor = os.environ.get("EDITOR")
 402
 403   if not editor:
 404     if sys.platform.startswith("win"):
 405       editor = "notepad"
 406     else:
 407       editor = "vi"
 408
 409   return editor
 410
 411
 412 def GenerateDiff(files):
 413   """Returns a string containing the diff for the given file list."""
 414   diff = []
 415   for file in files:
 416     # Use svn info output instead of os.path.isdir because the latter fails
 417     # when the file is deleted.
 418     if GetSVNFileInfo(file, "Node Kind") == "directory":
 419       continue
 420     # If the user specified a custom diff command in their svn config file,
 421     # then it'll be used when we do svn diff, which we don't want to happen
 422     # since we want the unified diff.  Using --diff-cmd=diff doesn't always
 423     # work, since they can have another diff executable in their path that
 424     # gives different line endings.  So we use a bogus temp directory as the
 425     # config directory, which gets around these problems.
 426     if sys.platform.startswith("win"):
 427       parent_dir = tempfile.gettempdir()
 428     else:
 429       parent_dir = sys.path[0]  # tempdir is not secure.
 430     bogus_dir = os.path.join(parent_dir, "temp_svn_config")
 431     if not os.path.exists(bogus_dir):
 432       os.mkdir(bogus_dir)
 433     diff.append(RunShell(["svn", "diff", "--config-dir", bogus_dir, file]))
 434   return "".join(diff)
 435
 436
 437 def UploadCL(change_info, args):
 438   if not change_info.FileList():
 439     print "Nothing to upload, changelist is empty."
 440     return
 441
 442   upload_arg = ["upload.py", "-y", "-l"]
 443   upload_arg.append("--server=" + GetCodeReviewSetting("CODE_REVIEW_SERVER"))
 444   upload_arg.extend(args)
 445
 446   desc_file = ""
 447   if change_info.issue:  # Uploading a new patchset.
 448     upload_arg.append("--message=''")
 449     upload_arg.append("--issue=" + change_info.issue)
 450   else: # First time we upload.
 451     handle, desc_file = tempfile.mkstemp(text=True)
 452     os.write(handle, change_info.description)
 453     os.close(handle)
 454
 455     upload_arg.append("--cc=" + GetCodeReviewSetting("CC_LIST"))
 456     upload_arg.append("--description_file=" + desc_file + "")
 457     if change_info.description:
 458       subject = change_info.description[:77]
 459       if subject.find("\r\n") != -1:
 460         subject = subject[:subject.find("\r\n")]
 461       if subject.find("\n") != -1:
 462         subject = subject[:subject.find("\n")]
 463       if len(change_info.description) > 77:
 464         subject = subject + "..."
 465       upload_arg.append("--message=" + subject)
 466
 467   # Change the current working directory before calling upload.py so that it
 468   # shows the correct base.
 469   os.chdir(GetRepositoryRoot())
 470
 471   # If we have a lot of files with long paths, then we won't be able to fit
 472   # the command to "svn diff".  Instead, we generate the diff manually for
 473   # each file and concatenate them before passing it to upload.py.
 474   issue = upload.RealMain(upload_arg, GenerateDiff(change_info.FileList()))
 475   if issue and issue != change_info.issue:
 476     change_info.issue = issue
 477     change_info.Save()
 478
 479   if desc_file:
 480     os.remove(desc_file)
 481
 482
 483 def TryChange(change_info, args):
 484   """Create a diff file of change_info and send it to the try server."""
 485   try:
 486     import trychange
 487   except ImportError:
 488     ErrorExit("You need to install trychange.py to use the try server.")
 489
 490   trychange.TryChange(args, change_info.name, change_info.FileList())
 491
 492
 493 def Commit(change_info):
 494   if not change_info.FileList():
 495     print "Nothing to commit, changelist is empty."
 496     return
 497
 498   commit_cmd = ["svn", "commit"]
 499   filename = ''
 500   if change_info.issue:
 501     # Get the latest description from Rietveld.
 502     change_info.description = GetIssueDescription(change_info.issue)
 503
 504   commit_message = change_info.description.replace('\r\n', '\n')
 505   if change_info.issue:
 506     commit_message += ('\nReview URL: http://%s/%s' %
 507                        (GetCodeReviewSetting("CODE_REVIEW_SERVER"),
 508                         change_info.issue))
 509
 510   handle, commit_filename = tempfile.mkstemp(text=True)
 511   os.write(handle, commit_message)
 512   os.close(handle)
 513
 514   handle, targets_filename = tempfile.mkstemp(text=True)
 515   os.write(handle, "\n".join(change_info.FileList()))
 516   os.close(handle)
 517
 518   commit_cmd += ['--file=' + commit_filename]
 519   commit_cmd += ['--targets=' + targets_filename]
 520   # Change the current working directory before calling commit.
 521   os.chdir(GetRepositoryRoot())
 522   output = RunShell(commit_cmd, True)
 523   os.remove(commit_filename)
 524   os.remove(targets_filename)
 525   if output.find("Committed revision") != -1:
 526     change_info.Delete()
 527
 528     if change_info.issue:
 529       revision = re.compile(".*?\nCommitted revision (\d+)",
 530                             re.DOTALL).match(output).group(1)
 531       viewvc_url = GetCodeReviewSetting("VIEW_VC")
 532       change_info.description = (change_info.description +
 533                                  "\n\nCommitted: " + viewvc_url + revision)
 534       change_info.CloseIssue()
 535
 536
 537 def Change(change_info):
 538   """Creates/edits a changelist."""
 539   if change_info.issue:
 540     try:
 541       description = GetIssueDescription(change_info.issue)
 542     except urllib2.HTTPError, err:
 543       if err.code == 404:
 544         # The user deleted the issue in Rietveld, so forget the old issue id.
 545         description = change_info.description
 546         change_info.issue = ""
 547         change_info.Save()
 548       else:
 549         ErrorExit("Error getting the description from Rietveld: " + err)
 550   else:
 551     description = change_info.description
 552
 553   other_files = GetFilesNotInCL()
 554
 555   separator1 = ("\n---All lines above this line become the description.\n"
 556                 "---Repository Root: " + GetRepositoryRoot() + "\n"
 557                 "---Paths in this changelist (" + change_info.name + "):\n")
 558   separator2 = "\n\n---Paths modified but not in any changelist:\n\n"
 559   text = (description + separator1 + '\n' +
 560           '\n'.join([f[0] + f[1] for f in change_info.files]) + separator2 +
 561           '\n'.join([f[0] + f[1] for f in other_files]) + '\n')
 562
 563   handle, filename = tempfile.mkstemp(text=True)
 564   os.write(handle, text)
 565   os.close(handle)
 566
 567   command = GetEditor() + " " + filename
 568   os.system(command)
 569
 570   result = ReadFile(filename)
 571   os.remove(filename)
 572
 573   if not result:
 574     return
 575
 576   split_result = result.split(separator1, 1)
 577   if len(split_result) != 2:
 578     ErrorExit("Don't modify the text starting with ---!\n\n" + result)
 579
 580   new_description = split_result[0]
 581   cl_files_text = split_result[1]
 582   if new_description != description:
 583     change_info.description = new_description
 584     if change_info.issue:
 585       # Update the Rietveld issue with the new description.
 586       change_info.UpdateRietveldDescription()
 587
 588   new_cl_files = []
 589   for line in cl_files_text.splitlines():
 590     if not len(line):
 591       continue
 592     if line.startswith("---"):
 593       break
 594     status = line[:7]
 595     file = line[7:]
 596     new_cl_files.append((status, file))
 597   change_info.files = new_cl_files
 598
 599   change_info.Save()
 600   print change_info.name + " changelist saved."
 601
 602
 603 def Changes():
 604   """Print all the changlists and their files."""
 605   for cl in GetCLs():
 606     change_info = LoadChangelistInfo(cl, True, True)
 607     print "\n--- Changelist " + change_info.name + ":"
 608     for file in change_info.files:
 609       print "".join(file)
 610
 611
 612 def main(argv=None):
 613   if argv is None:
 614     argv = sys.argv
 615
 616   if len(argv) == 1:
 617     Help()
 618     return 0;
 619
 620   # Create the directory where we store information about changelists if it
 621   # doesn't exist.
 622   if not os.path.exists(GetInfoDir()):
 623     os.mkdir(GetInfoDir())
 624
 625   command = argv[1]
 626   if command == "opened":
 627     Opened()
 628     return 0
 629   if command == "nothave":
 630     UnknownFiles(argv[2:])
 631     return 0
 632   if command == "changes":
 633     Changes()
 634     return 0
 635   if command == "diff" and len(argv) == 2:
 636     files = GetFilesNotInCL()
 637     print GenerateDiff([os.path.join(GetRepositoryRoot(), x[1]) for x in files])
 638     return 0
 639
 640   if len(argv) == 2:
 641     if command == "change":
 642       # Generate a random changelist name.
 643       changename = GenerateChangeName()
 644     elif command == "help":
 645       Help()
 646       return 0
 647     else:
 648       ErrorExit("Need a changelist name.")
 649   else:
 650     changename = argv[2]
 651
 652   fail_on_not_found = command != "change"
 653   change_info = LoadChangelistInfo(changename, fail_on_not_found, True)
 654
 655   if command == "change":
 656     Change(change_info)
 657   elif command == "upload":
 658     UploadCL(change_info, argv[3:])
 659   elif command == "commit":
 660     Commit(change_info)
 661   elif command == "delete":
 662     change_info.Delete()
 663   elif command == "try":
 664     TryChange(change_info, argv[3:])
 665   else:
 666     # Everything else that is passed into gcl we redirect to svn, after adding
 667     # the files. This allows commands such as 'gcl diff xxx' to work.
 668     args =["svn", command]
 669     root = GetRepositoryRoot()
 670     args.extend([os.path.join(root, x) for x in change_info.FileList()])
 671     RunShell(args, True)
 672   return 0
 673
 674
 675 if __name__ == "__main__":
 676     sys.exit(main())