dev-support/submit-patch.py

   1 #!/usr/bin/env python
   2 ##
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Makes a patch for the current branch, creates/updates the review board request and uploads new
  20 # patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming
  21 # rules. If no jira is specified, patch will be named (branch name).patch and jira and review board
  22 # are not updated. Review board id is retrieved from the remote link in the jira.
  23 # Print help: submit-patch.py --h
  24 import argparse
  25 from builtins import input, str
  26 import getpass
  27 import git
  28 import json
  29 import logging
  30 import os
  31 import re
  32 import requests
  33 import subprocess
  34 import sys
  35
  36 parser = argparse.ArgumentParser(
  37     epilog = "To avoid having to enter jira/review board username/password every time, setup an "
  38              "encrypted ~/.apache-cred files as follows:\n"
  39              "1) Create a file with following single "
  40              "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
  41              "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
  42              "2) Encrypt it with openssl.\n"
  43              "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
  44              "3) Delete original file.\n"
  45              "Now onwards, you'll need to enter this encryption key only once per run. If you "
  46              "forget the key, simply regenerate ~/.apache-cred file again.",
  47     formatter_class=argparse.RawTextHelpFormatter
  48 )
  49 parser.add_argument("-b", "--branch",
  50                     help = "Branch to use for generating diff. If not specified, tracking branch "
  51                          "is used. If there is no tracking branch, error will be thrown.")
  52
  53 # Arguments related to Jira.
  54 parser.add_argument("-jid", "--jira-id",
  55                     help = "Jira id of the issue. If set, we deduce next patch version from "
  56                            "attachments in the jira and also upload the new patch. Script will "
  57                            "ask for jira username/password for authentication. If not set, "
  58                            "patch is named <branch>.patch.")
  59
  60 # Arguments related to Review Board.
  61 parser.add_argument("-srb", "--skip-review-board",
  62                     help = "Don't create/update the review board.",
  63                     default = False, action = "store_true")
  64 parser.add_argument("--reviewers",
  65                     help = "Comma separated list of users to add as reviewers.")
  66
  67 # Misc arguments
  68 parser.add_argument("--patch-dir", default = "~/patches",
  69                     help = "Directory to store patch files. If it doesn't exist, it will be "
  70                           "created. Default: ~/patches")
  71 parser.add_argument("--rb-repo", default = "hbase-git",
  72                     help = "Review board repository. Default: hbase-git")
  73 args = parser.parse_args()
  74
  75 # Setup logger
  76 logging.basicConfig()
  77 logger = logging.getLogger("submit-patch")
  78 logger.setLevel(logging.INFO)
  79
  80
  81 def log_fatal_and_exit(*arg):
  82     logger.fatal(*arg)
  83     sys.exit(1)
  84
  85
  86 def assert_status_code(response, expected_status_code, description):
  87     if response.status_code != expected_status_code:
  88         log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
  89                            description, response.status_code, response.reason)
  90
  91
  92 # Make repo instance to interact with git repo.
  93 try:
  94     repo = git.Repo(os.getcwd())
  95     git = repo.git
  96 except git.exc.InvalidGitRepositoryError as e:
  97     log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of "
  98                        "HBase's git repo.", e)
  99
 100 logger.info(" Active branch: %s", repo.active_branch.name)
 101 # Do not proceed if there are uncommitted changes.
 102 if repo.is_dirty():
 103     log_fatal_and_exit(" Git status is dirty. Commit locally first.")
 104
 105
 106 # Returns base branch for creating diff.
 107 def get_base_branch():
 108     # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch.
 109     if args.branch is not None:
 110         base_branch = args.branch
 111         # Check that given branch exists.
 112         for ref in repo.refs:
 113             if ref.name == base_branch:
 114                 return base_branch
 115         log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
 116     else:
 117         # if --branch is not set, use tracking branch as base branch for computing diff.
 118         # If there is no tracking branch, log error and quit.
 119         tracking_branch = repo.active_branch.tracking_branch()
 120         if tracking_branch is None:
 121             log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base "
 122                                " branch for computing diff using --branch flag.")
 123         logger.info(" Using tracking branch as base branch")
 124         return tracking_branch.name
 125
 126
 127 # Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is
 128 # specified, patch is name (branch name).patch.
 129 def get_patch_name(branch):
 130     if args.jira_id is None:
 131         return branch + ".patch"
 132
 133     patch_name_prefix = args.jira_id.upper() + "." + branch
 134     return get_patch_name_with_version(patch_name_prefix)
 135
 136
 137 # Fetches list of attachments from the jira, deduces next version for the patch and returns final
 138 # patch name.
 139 def get_patch_name_with_version(patch_name_prefix):
 140     # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
 141     # Using crude way to get list of attachments.
 142     url = "https://issues.apache.org/jira/browse/" + args.jira_id
 143     logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
 144     html = requests.get(url)
 145     if html.status_code == 404:
 146         log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
 147     if html.status_code != 200:
 148         log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
 149     # Iterate over patch names starting from version 1 and return when name is not already used.
 150     content = str(html.content, 'utf-8')
 151     for i in range(1, 1000):
 152         name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
 153         if name not in content:
 154             return name
 155
 156
 157 # Validates that patch directory exists, if not, creates it.
 158 def validate_patch_dir(patch_dir):
 159     # Create patch_dir if it doesn't exist.
 160     if not os.path.exists(patch_dir):
 161         logger.warn(" Patch directory doesn't exist. Creating it.")
 162         os.mkdir(patch_dir)
 163     else:
 164         # If patch_dir exists, make sure it's a directory.
 165         if not os.path.isdir(patch_dir):
 166             log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
 167                                patch_dir)
 168
 169
 170 # Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
 171 # - base_branch has commits not in current branch
 172 # - current branch is same as base branch
 173 # - current branch is ahead of base_branch by more than 1 commits
 174 def check_diff_between_branches(base_branch):
 175     only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch))
 176     only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD"))
 177     if len(only_in_base_branch) != 0:
 178         log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
 179                            "and try again.", base_branch, len(only_in_base_branch))
 180     if len(only_in_active_branch) == 0:
 181         log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
 182     if len(only_in_active_branch) > 1:
 183         log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single "
 184                            "commit and try again.", base_branch, len(only_in_active_branch))
 185
 186
 187 # If ~/.apache-creds is present, load credentials from it otherwise prompt user.
 188 def get_credentials():
 189     creds = dict()
 190     creds_filepath = os.path.expanduser("~/.apache-creds")
 191     if os.path.exists(creds_filepath):
 192         try:
 193             logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
 194             content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
 195                                               shell=True)
 196         except subprocess.CalledProcessError as e:
 197             log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
 198         creds = json.loads(content)
 199     else:
 200         creds['jira_username'] = input("Jira username:")
 201         creds['jira_password'] = getpass.getpass("Jira password:")
 202         if not args.skip_review_board:
 203             creds['rb_username'] = input("Review Board username:")
 204             creds['rb_password'] = getpass.getpass("Review Board password:")
 205     return creds
 206
 207
 208 def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds):
 209     # Upload patch to jira using REST API.
 210     headers = {'X-Atlassian-Token': 'no-check'}
 211     files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')}
 212     jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 213     attachment_url = issue_url + "/attachments"
 214     r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
 215     assert_status_code(r, 200, "uploading patch to jira")
 216
 217
 218 def get_jira_summary(issue_url):
 219     r = requests.get(issue_url + "?fields=summary")
 220     assert_status_code(r, 200, "fetching jira summary")
 221     return json.loads(r.content)["fields"]["summary"]
 222
 223
 224 def get_review_board_id_if_present(issue_url, rb_link_title):
 225     r = requests.get(issue_url + "/remotelink")
 226     assert_status_code(r, 200, "fetching remote links")
 227     links = json.loads(r.content)
 228     for link in links:
 229         if link["object"]["title"] == rb_link_title:
 230             res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
 231             return res.group(1)
 232     return None
 233
 234
 235 base_branch = get_base_branch()
 236 # Remove remote repo name from branch name if present. This assumes that we don't use '/' in
 237 # actual branch names.
 238 base_branch_without_remote = base_branch.split('/')[-1]
 239 logger.info(" Base branch: %s", base_branch)
 240
 241 check_diff_between_branches(base_branch)
 242
 243 patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
 244 logger.info(" Patch directory: %s", patch_dir)
 245 validate_patch_dir(patch_dir)
 246
 247 patch_filename = get_patch_name(base_branch_without_remote)
 248 logger.info(" Patch name: %s", patch_filename)
 249 patch_filepath = os.path.join(patch_dir, patch_filename)
 250
 251 diff = git.format_patch(base_branch, stdout = True)
 252 with open(patch_filepath, "w") as f:
 253     f.write(diff.encode('utf8'))
 254
 255 if args.jira_id is not None:
 256     creds = get_credentials()
 257     issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id
 258
 259     attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds)
 260
 261     if not args.skip_review_board:
 262         rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])
 263
 264         rb_link_title = "Review Board (" + base_branch_without_remote + ")"
 265         rb_id = get_review_board_id_if_present(issue_url, rb_link_title)
 266
 267         # If no review board link found, create new review request and add its link to jira.
 268         if rb_id is None:
 269             reviews_url = "https://reviews.apache.org/api/review-requests/"
 270             data = {"repository" : "hbase-git"}
 271             r = requests.post(reviews_url, data = data, auth = rb_auth)
 272             assert_status_code(r, 201, "creating new review request")
 273             review_request = json.loads(r.content)["review_request"]
 274             absolute_url = review_request["absolute_url"]
 275             logger.info(" Created new review request: %s", absolute_url)
 276
 277             # Use jira summary as review's summary too.
 278             summary = get_jira_summary(issue_url)
 279             # Use commit message as description.
 280             description = repo.head.commit.message
 281             update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase",
 282                                  "target_people" : args.reviewers, "summary" : summary,
 283                                  "description" : description }
 284             draft_url = review_request["links"]["draft"]["href"]
 285             r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
 286             assert_status_code(r, 200, "updating review draft")
 287
 288             draft_request = json.loads(r.content)["draft"]
 289             diff_url = draft_request["links"]["draft_diffs"]["href"]
 290             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 291             r = requests.post(diff_url, files = files, auth = rb_auth)
 292             assert_status_code(r, 201, "uploading diff to review draft")
 293
 294             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 295             assert_status_code(r, 200, "publishing review request")
 296
 297             # Add link to review board in the jira.
 298             remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
 299             jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
 300             r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
 301                               headers={'Content-Type':'application/json'})
 302         else:
 303             logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id)
 304             draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
 305             diff_url = draft_url + "diffs/"
 306             files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
 307             r = requests.post(diff_url, files = files, auth = rb_auth)
 308             assert_status_code(r, 201, "uploading diff to review draft")
 309
 310             r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
 311             assert_status_code(r, 200, "publishing review request")