tools/vcs/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import logging
   7 import os
   8 import re
   9 import subprocess
  10 import sys
  11
  12 import mozpack.path as mozpath
  13 from mach.decorators import Command, CommandArgument
  14
  15 GITHUB_ROOT = "https://github.com/"
  16 PR_REPOSITORIES = {
  17     "webrender": {
  18         "github": "servo/webrender",
  19         "path": "gfx/wr",
  20         "bugzilla_product": "Core",
  21         "bugzilla_component": "Graphics: WebRender",
  22     },
  23     "webgpu": {
  24         "github": "gfx-rs/wgpu",
  25         "path": "gfx/wgpu",
  26         "bugzilla_product": "Core",
  27         "bugzilla_component": "Graphics: WebGPU",
  28     },
  29     "debugger": {
  30         "github": "firefox-devtools/debugger",
  31         "path": "devtools/client/debugger",
  32         "bugzilla_product": "DevTools",
  33         "bugzilla_component": "Debugger",
  34     },
  35 }
  36
  37
  38 @Command(
  39     "import-pr",
  40     category="misc",
  41     description="Import a pull request from Github to the local repo.",
  42 )
  43 @CommandArgument("-b", "--bug-number", help="Bug number to use in the commit messages.")
  44 @CommandArgument(
  45     "-t",
  46     "--bugzilla-token",
  47     help="Bugzilla API token used to file a new bug if no bug number is provided.",
  48 )
  49 @CommandArgument("-r", "--reviewer", help="Reviewer nick to apply to commit messages.")
  50 @CommandArgument(
  51     "pull_request",
  52     help="URL to the pull request to import (e.g. "
  53     "https://github.com/servo/webrender/pull/3665).",
  54 )
  55 def import_pr(
  56     command_context,
  57     pull_request,
  58     bug_number=None,
  59     bugzilla_token=None,
  60     reviewer=None,
  61 ):
  62     import requests
  63
  64     pr_number = None
  65     repository = None
  66     for r in PR_REPOSITORIES.values():
  67         if pull_request.startswith(GITHUB_ROOT + r["github"] + "/pull/"):
  68             # sanitize URL, dropping anything after the PR number
  69             pr_number = int(re.search("/pull/([0-9]+)", pull_request).group(1))
  70             pull_request = GITHUB_ROOT + r["github"] + "/pull/" + str(pr_number)
  71             repository = r
  72             break
  73
  74     if repository is None:
  75         command_context.log(
  76             logging.ERROR,
  77             "unrecognized_repo",
  78             {},
  79             "The pull request URL was not recognized; add it to the list of "
  80             "recognized repos in PR_REPOSITORIES in %s" % __file__,
  81         )
  82         sys.exit(1)
  83
  84     command_context.log(
  85         logging.INFO,
  86         "import_pr",
  87         {"pr_url": pull_request},
  88         "Attempting to import {pr_url}",
  89     )
  90     dirty = [
  91         f
  92         for f in command_context.repository.get_changed_files(mode="all")
  93         if f.startswith(repository["path"])
  94     ]
  95     if dirty:
  96         command_context.log(
  97             logging.ERROR,
  98             "dirty_tree",
  99             repository,
 100             "Local {path} tree is dirty; aborting!",
 101         )
 102         sys.exit(1)
 103     target_dir = mozpath.join(
 104         command_context.topsrcdir, os.path.normpath(repository["path"])
 105     )
 106
 107     if bug_number is None:
 108         if bugzilla_token is None:
 109             command_context.log(
 110                 logging.WARNING,
 111                 "no_token",
 112                 {},
 113                 "No bug number or bugzilla API token provided; bug number will not "
 114                 "be added to commit messages.",
 115             )
 116         else:
 117             bug_number = _file_bug(
 118                 command_context, bugzilla_token, repository, pr_number
 119             )
 120     elif bugzilla_token is not None:
 121         command_context.log(
 122             logging.WARNING,
 123             "too_much_bug",
 124             {},
 125             "Providing a bugzilla token is unnecessary when a bug number is provided. "
 126             "Using bug number; ignoring token.",
 127         )
 128
 129     pr_patch = requests.get(pull_request + ".patch")
 130     pr_patch.raise_for_status()
 131     for patch in _split_patches(pr_patch.content, bug_number, pull_request, reviewer):
 132         command_context.log(
 133             logging.INFO,
 134             "commit_msg",
 135             patch,
 136             "Processing commit [{commit_summary}] by [{author}] at [{date}]",
 137         )
 138         patch_cmd = subprocess.Popen(
 139             ["patch", "-p1", "-s"], stdin=subprocess.PIPE, cwd=target_dir
 140         )
 141         patch_cmd.stdin.write(patch["diff"].encode("utf-8"))
 142         patch_cmd.stdin.close()
 143         patch_cmd.wait()
 144         if patch_cmd.returncode != 0:
 145             command_context.log(
 146                 logging.ERROR,
 147                 "commit_fail",
 148                 {},
 149                 'Error applying diff from commit via "patch -p1 -s". Aborting...',
 150             )
 151             sys.exit(patch_cmd.returncode)
 152         command_context.repository.commit(
 153             patch["commit_msg"], patch["author"], patch["date"], [target_dir]
 154         )
 155         command_context.log(logging.INFO, "commit_pass", {}, "Committed successfully.")
 156
 157
 158 def _file_bug(command_context, token, repo, pr_number):
 159     import requests
 160
 161     bug = requests.post(
 162         "https://bugzilla.mozilla.org/rest/bug?api_key=%s" % token,
 163         json={
 164             "product": repo["bugzilla_product"],
 165             "component": repo["bugzilla_component"],
 166             "summary": "Land %s#%s in mozilla-central" % (repo["github"], pr_number),
 167             "version": "unspecified",
 168         },
 169     )
 170     bug.raise_for_status()
 171     command_context.log(logging.DEBUG, "new_bug", {}, bug.content)
 172     bugnumber = json.loads(bug.content)["id"]
 173     command_context.log(
 174         logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}"
 175     )
 176     return bugnumber
 177
 178
 179 def _split_patches(patchfile, bug_number, pull_request, reviewer):
 180     INITIAL = 0
 181     HEADERS = 1
 182     STAT_AND_DIFF = 2
 183
 184     patch = b""
 185     state = INITIAL
 186     for line in patchfile.splitlines():
 187         if state == INITIAL:
 188             if line.startswith(b"From "):
 189                 state = HEADERS
 190         elif state == HEADERS:
 191             patch += line + b"\n"
 192             if line == b"---":
 193                 state = STAT_AND_DIFF
 194         elif state == STAT_AND_DIFF:
 195             if line.startswith(b"From "):
 196                 yield _parse_patch(patch, bug_number, pull_request, reviewer)
 197                 patch = b""
 198                 state = HEADERS
 199             else:
 200                 patch += line + b"\n"
 201     if len(patch) > 0:
 202         yield _parse_patch(patch, bug_number, pull_request, reviewer)
 203     return
 204
 205
 206 def _parse_patch(patch, bug_number, pull_request, reviewer):
 207     import email
 208     from email import header, policy
 209
 210     parse_policy = policy.compat32.clone(max_line_length=None)
 211     parsed_mail = email.message_from_bytes(patch, policy=parse_policy)
 212
 213     def header_as_unicode(key):
 214         decoded = header.decode_header(parsed_mail[key])
 215         return str(header.make_header(decoded))
 216
 217     author = header_as_unicode("From")
 218     date = header_as_unicode("Date")
 219     commit_summary = header_as_unicode("Subject")
 220     email_body = parsed_mail.get_payload(decode=True).decode("utf-8")
 221     (commit_body, diff) = ("\n" + email_body).rsplit("\n---\n", 1)
 222
 223     bug_prefix = ""
 224     if bug_number is not None:
 225         bug_prefix = "Bug %s - " % bug_number
 226     commit_summary = re.sub(r"^\[PATCH[0-9 /]*\] ", bug_prefix, commit_summary)
 227     if reviewer is not None:
 228         commit_summary += " r=" + reviewer
 229
 230     commit_msg = commit_summary + "\n"
 231     if len(commit_body) > 0:
 232         commit_msg += commit_body + "\n"
 233     commit_msg += "\n[import_pr] From " + pull_request + "\n"
 234
 235     patch_obj = {
 236         "author": author,
 237         "date": date,
 238         "commit_summary": commit_summary,
 239         "commit_msg": commit_msg,
 240         "diff": diff,
 241     }
 242     return patch_obj