dev-support/submit-patch.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Makes a patch for the current branch, creates/updates the review board request and uploads new
  20 # patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming
  21 # rules. If no jira is specified, patch will be named (branch name).patch and jira and review board
  22 # are not updated. Review board id is retrieved from the remote link in the jira.
  23 # Print help: submit-patch.py --h
  24 import argparse
  25 import getpass
  26 import git
  27 import json
  28 import logging
  29 import os
  30 import re
  31 import requests
  32 import subprocess
  33 import sys
  34
  35 parser = argparse.ArgumentParser(
  36     epilog = "To avoid having to enter jira/review board username/password every time, setup an "
  37              "encrypted ~/.apache-cred files as follows:\n"
  38              "1) Create a file with following single "
  39              "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
  40              "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
  41              "2) Encrypt it with openssl.\n"
  42              "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
  43              "3) Delete original file.\n"
  44              "Now onwards, you'll need to enter this encryption key only once per run. If you "
  45              "forget the key, simply regenerate ~/.apache-cred file again.",
  46     formatter_class=argparse.RawTextHelpFormatter
  47 )
  48 parser.add_argument("-b", "--branch",
  49                     help = "Branch to use for generating diff. If not specified, tracking branch "
  50                          "is used. If there is no tracking branch, error will be thrown.")
  51
  52 # Arguments related to Jira.
  53 parser.add_argument("-jid", "--jira-id",
  54                     help = "Jira id of the issue. If set, we deduce next patch version from "
  55                            "attachments in the jira and also upload the new patch. Script will "
  56                            "ask for jira username/password for authentication. If not set, "
  57                            "patch is named <branch>.patch.")
  58
  59 # Arguments related to Review Board.
  60 parser.add_argument("-srb", "--skip-review-board",
  61                     help = "Don't create/update the review board.",
  62                     default = False, action = "store_true")
  63 parser.add_argument("--reviewers",
  64                     help = "Comma separated list of users to add as reviewers.")
  65
  66 # Misc arguments
  67 parser.add_argument("--patch-dir", default = "~/patches",
  68                     help = "Directory to store patch files. If it doesn't exist, it will be "
  69                           "created. Default: ~/patches")
  70 parser.add_argument("--rb-repo", default = "hbase-git",
  71                     help = "Review board repository. Default: hbase-git")
  72 args = parser.parse_args()
  73
  74 # Setup logger
  75 logging.basicConfig()
  76 logger = logging.getLogger("submit-patch")
  77 logger.setLevel(logging.INFO)
  78
  79
  80 def log_fatal_and_exit(*arg):
  81     logger.fatal(*arg)
  82     sys.exit(1)
  83
  84
  85 def assert_status_code(response, expected_status_code, description):
  86     if response.status_code != expected_status_code:
  87         log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
  88                            description, response.status_code, response.reason)
  89
  90
  91 # Make repo instance to interact with git repo.
  92 try:
  93     repo = git.Repo(os.getcwd())
  94     git = repo.git
  95 except git.exc.InvalidGitRepositoryError as e:
  96     log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of "
  97                        "HBase's git repo.", e)
  98
  99 logger.info(" Active branch: %s", repo.active_branch.name)
 100 # Do not proceed if there are uncommitted changes.
 101 if repo.is_dirty():
 102     log_fatal_and_exit(" Git status is dirty. Commit locally first.")
 103
 104
 105 # Returns base branch for creating diff.
 106 def get_base_branch():
 107     # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch.
 108     if args.branch is not None:
 109         base_branch = args.branch
 110         # Check that given branch exists.
 111         for ref in repo.refs:
 112             if ref.name == base_branch:
 113                 return base_branch
 114         log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
 115     else:
 116         # if --branch is not set, use tracking branch as base branch for computing diff.
 117         # If there is no tracking branch, log error and quit.
 118         tracking_branch = repo.active_branch.tracking_branch()
 119         if tracking_branch is None:
 120             log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base "
 121                                " branch for computing diff using --branch flag.")
 122         logger.info(" Using tracking branch as base branch")
 123         return tracking_branch.name
 124
 125
 126 # Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is
 127 # specified, patch is name (branch name).patch.
 128 def get_patch_name(branch):
 129     if args.jira_id is None:
 130         return branch + ".patch"
 131
 132     patch_name_prefix = args.jira_id.upper() + "." + branch
 133     return get_patch_name_with_version(patch_name_prefix)
 134
 135
 136 # Fetches list of attachments from the jira, deduces next version for the patch and returns final
 137 # patch name.
 138 def get_patch_name_with_version(patch_name_prefix):
 139     # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
 140     # Using crude way to get list of attachments.
 141     url = "https://issues.apache.org/jira/browse/" + args.jira_id
 142     logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
 143     html = requests.get(url)
 144     if html.status_code == 404:
 145         log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
 146     if html.status_code != 200:
 147         log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
 148     # Iterate over patch names starting from version 1 and return when name is not already used.
 149     content = unicode(html.content, 'utf-8')
 150     for i in range(1, 1000):
 151         name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
 152         if name not in content:
 153             return name
 154
 155
 156 # Validates that patch directory exists, if not, creates it.
 157 def validate_patch_dir(patch_dir):
 158     # Create patch_dir if it doesn't exist.
 159     if not os.path.exists(patch_dir):
 160         logger.warn(" Patch directory doesn't exist. Creating it.")
 161         os.mkdir(patch_dir)
 162     else:
 163         # If patch_dir exists, make sure it's a directory.
 164         if not os.path.isdir(patch_dir):
 165             log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
 166                                patch_dir)
 167
 168
 169 # Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
 170 # - base_branch has commits not in current branch
 171 # - current branch is same as base branch
 172 # - current branch is ahead of base_branch by more than 1 commits
 173 def check_diff_between_branches(base_branch):
 174     only_in_base_branch = git.log("HEAD.." + base_branch, oneline = True)
 175     only_in_active_branch = git.log(base_branch + "..HEAD", oneline = True)
 176     if len(only_in_base_branch) != 0:
 177         log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
 178                            "and try again.", base_branch, len(only_in_base_branch.split("\n")))
 179     if len(only_in_active_branch) == 0:
 180         log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
 181     if len(only_in_active_branch.split("\n")) > 1:
 182         log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single "
 183                            "commit and try again.",
 184                            base_branch, len(only_in_active_branch.split("\n")))
 185
 186
 187 # If ~/.apache-creds is present, load credentials from it otherwise prompt user.
 188 def get_credentials():
 189     creds = dict()
 190     creds_filepath = os.path.expanduser("~/.apache-creds")
 191     if os.path.exists(creds_filepath):
 192         try:
 193             logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
 194             content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
 195                                               shell=True)
 196         except subprocess.CalledProcessError as e:
 197             log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
 198         creds = json.loads(content)
 199     else:
 200         creds['jira_username'] = raw_input("Jira username:")
 201         creds['jira_password'] = getpass.getpass("Jira password:")
 202         if not args.skip_review_board:
 203             creds['rb_username'] = raw_input("Review Board username:")
 204             creds['rb_password'] = getpass.getpass("Review Board password:")
 205     return creds
 206
 207
 208 def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds):
 209     # Upload patch to jira using REST API.
 210     headers = {'X-Atlassian-Token': 'no-check'}
 211     files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')}
 212     jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 213     attachment_url = issue_url + "/attachments"
 214     r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
 215     assert_status_code(r, 200, "uploading patch to jira")
 216
 217
 218 def get_jira_summary(issue_url):
 219     r = requests.get(issue_url + "?fields=summary")
 220     assert_status_code(r, 200, "fetching jira summary")
 221     return json.loads(r.content)["fields"]["summary"]
 222
 223
 224 def get_review_board_id_if_present(issue_url, rb_link_title):
 225     r = requests.get(issue_url + "/remotelink")
 226     assert_status_code(r, 200, "fetching remote links")
 227     links = json.loads(r.content)
 228     for link in links:
 229         if link["object"]["title"] == rb_link_title:
 230             res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
 231             return res.group(1)
 232     return None
 233
 234
 235 base_branch = get_base_branch()
 236 # Remove remote repo name from branch name if present. This assumes that we don't use '/' in
 237 # actual branch names.
 238 base_branch_without_remote = base_branch.split('/')[-1]
 239 logger.info(" Base branch: %s", base_branch)
 240
 241 check_diff_between_branches(base_branch)
 242
 243 patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
 244 logger.info(" Patch directory: %s", patch_dir)
 245 validate_patch_dir(patch_dir)
 246
 247 patch_filename = get_patch_name(base_branch_without_remote)
 248 logger.info(" Patch name: %s", patch_filename)
 249 patch_filepath = os.path.join(patch_dir, patch_filename)
 250
 251 diff = git.format_patch(base_branch, stdout = True)
 252 with open(patch_filepath, "wb") as f:
 253     f.write(diff.encode('utf8'))
 254
 255 if args.jira_id is not None:
 256     creds = get_credentials()
 257     issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id
 258
 259     attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds)
 260
 261     if not args.skip_review_board:
 262         rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])
 263
 264         rb_link_title = "Review Board (" + base_branch_without_remote + ")"
 265         rb_id = get_review_board_id_if_present(issue_url, rb_link_title)
 266
 267         # If no review board link found, create new review request and add its link to jira.
 268         if rb_id is None:
 269             reviews_url = "https://reviews.apache.org/api/review-requests/"
 270             data = {"repository" : "hbase-git"}
 271             r = requests.post(reviews_url, data = data, auth = rb_auth)
 272             assert_status_code(r, 201, "creating new review request")
 273             review_request = json.loads(r.content)["review_request"]
 274             absolute_url = review_request["absolute_url"]
 275             logger.info(" Created new review request: %s", absolute_url)
 276
 277             # Use jira summary as review's summary too.
 278             summary = get_jira_summary(issue_url)
 279             # Use commit message as description.
 280             description = git.log("-1", pretty="%B")
 281             update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase",
 282                                  "target_people" : args.reviewers, "summary" : summary,
 283                                  "description" : description }
 284             draft_url = review_request["links"]["draft"]["href"]
 285             r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
 286             assert_status_code(r, 200, "updating review draft")
 287
 288             draft_request = json.loads(r.content)["draft"]
 289             diff_url = draft_request["links"]["draft_diffs"]["href"]
 290             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 291             r = requests.post(diff_url, files = files, auth = rb_auth)
 292             assert_status_code(r, 201, "uploading diff to review draft")
 293
 294             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 295             assert_status_code(r, 200, "publishing review request")
 296
 297             # Add link to review board in the jira.
 298             remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
 299             jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 300             r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
 301                               headers={'Content-Type':'application/json'})
 302         else:
 303             logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id)
 304             draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
 305             diff_url = draft_url + "diffs/"
 306             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 307             r = requests.post(diff_url, files = files, auth = rb_auth)
 308             assert_status_code(r, 201, "uploading diff to review draft")
 309
 310             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 311             assert_status_code(r, 200, "publishing review request")