dev-support/submit-patch.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Makes a patch for the current branch, creates/updates the review board request and uploads new
  20 # patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming
  21 # rules. If no jira is specified, patch will be named (branch name).patch and jira and review board
  22 # are not updated. Review board id is retrieved from the remote link in the jira.
  23 # Print help: submit-patch.py --h
  24 import argparse
  25 import getpass
  26 import git
  27 import json
  28 import logging
  29 import os
  30 import re
  31 import requests
  32 import subprocess
  33 import sys
  34
  35 parser = argparse.ArgumentParser(
  36     epilog = "To avoid having to enter jira/review board username/password every time, setup an "
  37              "encrypted ~/.apache-cred files as follows:\n"
  38              "1) Create a file with following single "
  39              "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
  40              "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
  41              "2) Encrypt it with openssl.\n"
  42              "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
  43              "3) Delete original file.\n"
  44              "Now onwards, you'll need to enter this encryption key only once per run. If you "
  45              "forget the key, simply regenerate ~/.apache-cred file again.",
  46     formatter_class=argparse.RawTextHelpFormatter
  47 )
  48 parser.add_argument("-b", "--branch",
  49                     help = "Branch to use for generating diff. If not specified, tracking branch "
  50                          "is used. If there is no tracking branch, error will be thrown.")
  51
  52 # Arguments related to Jira.
  53 parser.add_argument("-jid", "--jira-id",
  54                     help = "Jira id of the issue. If set, we deduce next patch version from "
  55                            "attachments in the jira and also upload the new patch. Script will "
  56                            "ask for jira username/password for authentication. If not set, "
  57                            "patch is named <branch>.patch.")
  58
  59 # Arguments related to Review Board.
  60 parser.add_argument("-srb", "--skip-review-board",
  61                     help = "Don't create/update the review board.",
  62                     default = False, action = "store_true")
  63 parser.add_argument("--reviewers",
  64                     help = "Comma separated list of users to add as reviewers.")
  65
  66 # Misc arguments
  67 parser.add_argument("--patch-dir", default = "~/patches",
  68                     help = "Directory to store patch files. If it doesn't exist, it will be "
  69                           "created. Default: ~/patches")
  70 parser.add_argument("--rb-repo", default = "hbase-git",
  71                     help = "Review board repository. Default: hbase-git")
  72 args = parser.parse_args()
  73
  74 # Setup logger
  75 logging.basicConfig()
  76 logger = logging.getLogger("submit-patch")
  77 logger.setLevel(logging.INFO)
  78
  79
  80 def log_fatal_and_exit(*arg):
  81     logger.fatal(*arg)
  82     sys.exit(1)
  83
  84
  85 def assert_status_code(response, expected_status_code, description):
  86     if response.status_code != expected_status_code:
  87         log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
  88                            description, response.status_code, response.reason)
  89
  90
  91 # Make repo instance to interact with git repo.
  92 try:
  93     repo = git.Repo(os.getcwd())
  94     git = repo.git
  95 except git.exc.InvalidGitRepositoryError as e:
  96     log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of "
  97                        "HBase's git repo.", e)
  98
  99 logger.info(" Active branch: %s", repo.active_branch.name)
 100 # Do not proceed if there are uncommitted changes.
 101 if repo.is_dirty():
 102     log_fatal_and_exit(" Git status is dirty. Commit locally first.")
 103
 104
 105 # Returns base branch for creating diff.
 106 def get_base_branch():
 107     # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch.
 108     if args.branch is not None:
 109         base_branch = args.branch
 110         # Check that given branch exists.
 111         for ref in repo.refs:
 112             if ref.name == base_branch:
 113                 return base_branch
 114         log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
 115     else:
 116         # if --branch is not set, use tracking branch as base branch for computing diff.
 117         # If there is no tracking branch, log error and quit.
 118         tracking_branch = repo.active_branch.tracking_branch()
 119         if tracking_branch is None:
 120             log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base "
 121                                " branch for computing diff using --branch flag.")
 122         logger.info(" Using tracking branch as base branch")
 123         return tracking_branch.name
 124
 125
 126 # Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is
 127 # specified, patch is name (branch name).patch.
 128 def get_patch_name(branch):
 129     if args.jira_id is None:
 130         return branch + ".patch"
 131
 132     patch_name_prefix = args.jira_id.upper() + "." + branch
 133     return get_patch_name_with_version(patch_name_prefix)
 134
 135
 136 # Fetches list of attachments from the jira, deduces next version for the patch and returns final
 137 # patch name.
 138 def get_patch_name_with_version(patch_name_prefix):
 139     # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
 140     # Using crude way to get list of attachments.
 141     url = "https://issues.apache.org/jira/browse/" + args.jira_id
 142     logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
 143     html = requests.get(url)
 144     if html.status_code == 404:
 145         log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
 146     if html.status_code != 200:
 147         log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
 148     # Iterate over patch names starting from version 1 and return when name is not already used.
 149     content = unicode(html.content, 'utf-8')
 150     for i in range(1, 1000):
 151         name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
 152         if name not in content:
 153             return name
 154
 155
 156 # Validates that patch directory exists, if not, creates it.
 157 def validate_patch_dir(patch_dir):
 158     # Create patch_dir if it doesn't exist.
 159     if not os.path.exists(patch_dir):
 160         logger.warn(" Patch directory doesn't exist. Creating it.")
 161         os.mkdir(patch_dir)
 162     else:
 163         # If patch_dir exists, make sure it's a directory.
 164         if not os.path.isdir(patch_dir):
 165             log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
 166                                patch_dir)
 167
 168
 169 # Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
 170 # - base_branch has commits not in current branch
 171 # - current branch is same as base branch
 172 # - current branch is ahead of base_branch by more than 1 commits
 173 def check_diff_between_branches(base_branch):
 174     only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch))
 175     only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD"))
 176     if len(only_in_base_branch) != 0:
 177         log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
 178                            "and try again.", base_branch, len(only_in_base_branch))
 179     if len(only_in_active_branch) == 0:
 180         log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
 181     if len(only_in_active_branch) > 1:
 182         log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single "
 183                            "commit and try again.", base_branch, len(only_in_active_branch))
 184
 185
 186 # If ~/.apache-creds is present, load credentials from it otherwise prompt user.
 187 def get_credentials():
 188     creds = dict()
 189     creds_filepath = os.path.expanduser("~/.apache-creds")
 190     if os.path.exists(creds_filepath):
 191         try:
 192             logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
 193             content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
 194                                               shell=True)
 195         except subprocess.CalledProcessError as e:
 196             log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
 197         creds = json.loads(content)
 198     else:
 199         creds['jira_username'] = raw_input("Jira username:")
 200         creds['jira_password'] = getpass.getpass("Jira password:")
 201         if not args.skip_review_board:
 202             creds['rb_username'] = raw_input("Review Board username:")
 203             creds['rb_password'] = getpass.getpass("Review Board password:")
 204     return creds
 205
 206
 207 def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds):
 208     # Upload patch to jira using REST API.
 209     headers = {'X-Atlassian-Token': 'no-check'}
 210     files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')}
 211     jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 212     attachment_url = issue_url + "/attachments"
 213     r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
 214     assert_status_code(r, 200, "uploading patch to jira")
 215
 216
 217 def get_jira_summary(issue_url):
 218     r = requests.get(issue_url + "?fields=summary")
 219     assert_status_code(r, 200, "fetching jira summary")
 220     return json.loads(r.content)["fields"]["summary"]
 221
 222
 223 def get_review_board_id_if_present(issue_url, rb_link_title):
 224     r = requests.get(issue_url + "/remotelink")
 225     assert_status_code(r, 200, "fetching remote links")
 226     links = json.loads(r.content)
 227     for link in links:
 228         if link["object"]["title"] == rb_link_title:
 229             res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
 230             return res.group(1)
 231     return None
 232
 233
 234 base_branch = get_base_branch()
 235 # Remove remote repo name from branch name if present. This assumes that we don't use '/' in
 236 # actual branch names.
 237 base_branch_without_remote = base_branch.split('/')[-1]
 238 logger.info(" Base branch: %s", base_branch)
 239
 240 check_diff_between_branches(base_branch)
 241
 242 patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
 243 logger.info(" Patch directory: %s", patch_dir)
 244 validate_patch_dir(patch_dir)
 245
 246 patch_filename = get_patch_name(base_branch_without_remote)
 247 logger.info(" Patch name: %s", patch_filename)
 248 patch_filepath = os.path.join(patch_dir, patch_filename)
 249
 250 diff = git.format_patch(base_branch, stdout = True)
 251 with open(patch_filepath, "wb") as f:
 252     f.write(diff.encode('utf8'))
 253
 254 if args.jira_id is not None:
 255     creds = get_credentials()
 256     issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id
 257
 258     attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds)
 259
 260     if not args.skip_review_board:
 261         rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])
 262
 263         rb_link_title = "Review Board (" + base_branch_without_remote + ")"
 264         rb_id = get_review_board_id_if_present(issue_url, rb_link_title)
 265
 266         # If no review board link found, create new review request and add its link to jira.
 267         if rb_id is None:
 268             reviews_url = "https://reviews.apache.org/api/review-requests/"
 269             data = {"repository" : "hbase-git"}
 270             r = requests.post(reviews_url, data = data, auth = rb_auth)
 271             assert_status_code(r, 201, "creating new review request")
 272             review_request = json.loads(r.content)["review_request"]
 273             absolute_url = review_request["absolute_url"]
 274             logger.info(" Created new review request: %s", absolute_url)
 275
 276             # Use jira summary as review's summary too.
 277             summary = get_jira_summary(issue_url)
 278             # Use commit message as description.
 279             description = repo.head.commit.message
 280             update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase",
 281                                  "target_people" : args.reviewers, "summary" : summary,
 282                                  "description" : description }
 283             draft_url = review_request["links"]["draft"]["href"]
 284             r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
 285             assert_status_code(r, 200, "updating review draft")
 286
 287             draft_request = json.loads(r.content)["draft"]
 288             diff_url = draft_request["links"]["draft_diffs"]["href"]
 289             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 290             r = requests.post(diff_url, files = files, auth = rb_auth)
 291             assert_status_code(r, 201, "uploading diff to review draft")
 292
 293             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 294             assert_status_code(r, 200, "publishing review request")
 295
 296             # Add link to review board in the jira.
 297             remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
 298             jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 299             r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
 300                               headers={'Content-Type':'application/json'})
 301         else:
 302             logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id)
 303             draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
 304             diff_url = draft_url + "diffs/"
 305             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 306             r = requests.post(diff_url, files = files, auth = rb_auth)
 307             assert_status_code(r, 201, "uploading diff to review draft")
 308
 309             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 310             assert_status_code(r, 200, "publishing review request")