llvm/utils/git/github-automation.py

   1 #!/usr/bin/env python3
   2 #
   3 # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
   4 #
   5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   6 # See https://llvm.org/LICENSE.txt for license information.
   7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   8 #
   9 # ==-------------------------------------------------------------------------==#
  10
  11 import argparse
  12 from git import Repo # type: ignore
  13 import github
  14 import os
  15 import re
  16 import requests
  17 import sys
  18 import time
  19 from typing import *
  20
  21 class IssueSubscriber:
  22
  23     @property
  24     def team_name(self) -> str:
  25         return self._team_name
  26
  27     def __init__(self, token:str, repo:str, issue_number:int, label_name:str):
  28         self.repo = github.Github(token).get_repo(repo)
  29         self.org = github.Github(token).get_organization(self.repo.organization.login)
  30         self.issue = self.repo.get_issue(issue_number)
  31         self._team_name = 'issue-subscribers-{}'.format(label_name).lower()
  32
  33     def run(self) -> bool:
  34         for team in self.org.get_teams():
  35             if self.team_name != team.name.lower():
  36                 continue
  37             comment = '@llvm/{}'.format(team.slug)
  38             self.issue.create_comment(comment)
  39             return True
  40         return False
  41
  42 def setup_llvmbot_git(git_dir = '.'):
  43     """
  44     Configure the git repo in `git_dir` with the llvmbot account so
  45     commits are attributed to llvmbot.
  46     """
  47     repo = Repo(git_dir)
  48     with repo.config_writer() as config:
  49         config.set_value('user', 'name', 'llvmbot')
  50         config.set_value('user', 'email', 'llvmbot@llvm.org')
  51
  52 def phab_api_call(phab_token:str, url:str, args:dict) -> dict:
  53     """
  54     Make an API call to the Phabricator web service and return a dictionary
  55     containing the json response.
  56     """
  57     data = { "api.token" : phab_token }
  58     data.update(args)
  59     response = requests.post(url, data = data)
  60     return response.json()
  61
  62
  63 def phab_login_to_github_login(phab_token:str, repo:github.Repository.Repository, phab_login:str) -> str:
  64     """
  65     Tries to translate a Phabricator login to a github login by
  66     finding a commit made in Phabricator's Differential.
  67     The commit's SHA1 is then looked up in the github repo and
  68     the committer's login associated with that commit is returned.
  69
  70     :param str phab_token: The Conduit API token to use for communication with Pabricator
  71     :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
  72     :param str phab_login: The Phabricator login to be translated.
  73     """
  74
  75     args = {
  76         "constraints[authors][0]" : phab_login,
  77         # PHID for "LLVM Github Monorepo" repository
  78         "constraints[repositories][0]" : "PHID-REPO-f4scjekhnkmh7qilxlcy",
  79         "limit" : 1
  80     }
  81     # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
  82     r = phab_api_call(phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args)
  83     data = r['result']['data']
  84     if len(data) == 0:
  85         # Can't find any commits associated with this user
  86         return None
  87
  88     commit_sha = data[0]['fields']['identifier']
  89     committer = repo.get_commit(commit_sha).committer
  90     if not committer:
  91         # This committer had an email address GitHub could not recognize, so
  92         # it can't link the user to a GitHub account.
  93         print(f"Warning: Can't find github account for {phab_login}")
  94         return None
  95     return committer.login
  96
  97 def phab_get_commit_approvers(phab_token:str, repo:github.Repository.Repository, commit:github.Commit.Commit) -> list:
  98     args = { "corpus" : commit.commit.message }
  99     # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
 100     r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args)
 101     review_id = r['result']['revisionIDFieldInfo']['value']
 102     if not review_id:
 103         # No Phabricator revision for this commit
 104         return []
 105
 106     args = {
 107         'constraints[ids][0]' : review_id,
 108         'attachments[reviewers]' : True
 109     }
 110     # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
 111     r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.revision.search", args)
 112     reviewers = r['result']['data'][0]['attachments']['reviewers']['reviewers']
 113     accepted = []
 114     for reviewer in reviewers:
 115         if reviewer['status'] != 'accepted':
 116             continue
 117         phid = reviewer['reviewerPHID']
 118         args = { 'constraints[phids][0]' : phid }
 119         # API documentation: https://reviews.llvm.org/conduit/method/user.search/
 120         r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args)
 121         accepted.append(r['result']['data'][0]['fields']['username'])
 122     return accepted
 123
 124 class ReleaseWorkflow:
 125
 126     CHERRY_PICK_FAILED_LABEL = 'release:cherry-pick-failed'
 127
 128     """
 129     This class implements the sub-commands for the release-workflow command.
 130     The current sub-commands are:
 131         * create-branch
 132         * create-pull-request
 133
 134     The execute_command method will automatically choose the correct sub-command
 135     based on the text in stdin.
 136     """
 137
 138     def __init__(self, token:str, repo:str, issue_number:int,
 139                        branch_repo_name:str, branch_repo_token:str,
 140                        llvm_project_dir:str, phab_token:str) -> None:
 141         self._token = token
 142         self._repo_name = repo
 143         self._issue_number = issue_number
 144         self._branch_repo_name = branch_repo_name
 145         if branch_repo_token:
 146             self._branch_repo_token = branch_repo_token
 147         else:
 148             self._branch_repo_token = self.token
 149         self._llvm_project_dir = llvm_project_dir
 150         self._phab_token = phab_token
 151
 152     @property
 153     def token(self) -> str:
 154         return self._token
 155
 156     @property
 157     def repo_name(self) -> str:
 158         return self._repo_name
 159
 160     @property
 161     def issue_number(self) -> int:
 162         return self._issue_number
 163
 164     @property
 165     def branch_repo_name(self) -> str:
 166         return self._branch_repo_name
 167
 168     @property
 169     def branch_repo_token(self) -> str:
 170         return self._branch_repo_token
 171
 172     @property
 173     def llvm_project_dir(self) -> str:
 174         return self._llvm_project_dir
 175
 176     @property
 177     def phab_token(self) -> str:
 178         return self._phab_token
 179
 180     @property
 181     def repo(self) -> github.Repository.Repository:
 182         return github.Github(self.token).get_repo(self.repo_name)
 183
 184     @property
 185     def issue(self) -> github.Issue.Issue:
 186         return self.repo.get_issue(self.issue_number)
 187
 188     @property
 189     def push_url(self) -> str:
 190         return 'https://{}@github.com/{}'.format(self.branch_repo_token, self.branch_repo_name)
 191
 192     @property
 193     def branch_name(self) -> str:
 194         return 'issue{}'.format(self.issue_number)
 195
 196     @property
 197     def release_branch_for_issue(self) -> Optional[str]:
 198         issue = self.issue
 199         milestone = issue.milestone
 200         if milestone is None:
 201             return None
 202         m = re.search('branch: (.+)',milestone.description)
 203         if m:
 204             return m.group(1)
 205         return None
 206
 207     def print_release_branch(self) -> None:
 208         print(self.release_branch_for_issue)
 209
 210     def issue_notify_branch(self) -> None:
 211         self.issue.create_comment('/branch {}/{}'.format(self.branch_repo_name, self.branch_name))
 212
 213     def issue_notify_pull_request(self, pull:github.PullRequest.PullRequest) -> None:
 214         self.issue.create_comment('/pull-request {}#{}'.format(self.branch_repo_name, pull.number))
 215
 216     def make_ignore_comment(self, comment: str) -> str:
 217         """
 218         Returns the comment string with a prefix that will cause
 219         a Github workflow to skip parsing this comment.
 220
 221         :param str comment: The comment to ignore
 222         """
 223         return "<!--IGNORE-->\n"+comment
 224
 225     def issue_notify_no_milestone(self, comment:List[str]) -> None:
 226         message = "{}\n\nError: Command failed due to missing milestone.".format(''.join(['>' + line for line in comment]))
 227         self.issue.create_comment(self.make_ignore_comment(message))
 228
 229     @property
 230     def action_url(self) -> str:
 231         if os.getenv('CI'):
 232             return 'https://github.com/{}/actions/runs/{}'.format(os.getenv('GITHUB_REPOSITORY'), os.getenv('GITHUB_RUN_ID'))
 233         return ""
 234
 235     def issue_notify_cherry_pick_failure(self, commit:str) -> github.IssueComment.IssueComment:
 236         message = self.make_ignore_comment("Failed to cherry-pick: {}\n\n".format(commit))
 237         action_url = self.action_url
 238         if action_url:
 239             message += action_url + "\n\n"
 240         message += "Please manually backport the fix and push it to your github fork.  Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
 241         issue = self.issue
 242         comment = issue.create_comment(message)
 243         issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL)
 244         return comment
 245
 246     def issue_notify_pull_request_failure(self, branch:str) -> github.IssueComment.IssueComment:
 247         message = "Failed to create pull request for {} ".format(branch)
 248         message += self.action_url
 249         return self.issue.create_comment(message)
 250
 251     def issue_remove_cherry_pick_failed_label(self):
 252         if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]:
 253             self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL)
 254
 255     def pr_request_review(self, pr:github.PullRequest.PullRequest):
 256         """
 257         This function will try to find the best reviewers for `commits` and
 258         then add a comment requesting review of the backport and assign the
 259         pull request to the selected reviewers.
 260
 261         The reviewers selected are those users who approved the patch in
 262         Phabricator.
 263         """
 264         reviewers = []
 265         for commit in pr.get_commits():
 266             approvers = phab_get_commit_approvers(self.phab_token, self.repo, commit)
 267             for a in approvers:
 268                 login = phab_login_to_github_login(self.phab_token, self.repo, a)
 269                 if not login:
 270                     continue
 271                 reviewers.append(login)
 272         if len(reviewers):
 273             message = "{} What do you think about merging this PR to the release branch?".format(
 274                     " ".join(["@" + r for r in reviewers]))
 275             pr.create_issue_comment(message)
 276             pr.add_to_assignees(*reviewers)
 277
 278     def create_branch(self, commits:List[str]) -> bool:
 279         """
 280         This function attempts to backport `commits` into the branch associated
 281         with `self.issue_number`.
 282
 283         If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
 284         a comment is added to the issue saying that the cherry-pick failed.
 285
 286         :param list commits: List of commits to cherry-pick.
 287
 288         """
 289         print('cherry-picking', commits)
 290         branch_name = self.branch_name
 291         local_repo = Repo(self.llvm_project_dir)
 292         local_repo.git.checkout(self.release_branch_for_issue)
 293
 294         for c in commits:
 295             try:
 296                 local_repo.git.cherry_pick('-x', c)
 297             except Exception as e:
 298                 self.issue_notify_cherry_pick_failure(c)
 299                 raise e
 300
 301         push_url = self.push_url
 302         print('Pushing to {} {}'.format(push_url, branch_name))
 303         local_repo.git.push(push_url, 'HEAD:{}'.format(branch_name), force=True)
 304
 305         self.issue_notify_branch()
 306         self.issue_remove_cherry_pick_failed_label()
 307         return True
 308
 309     def check_if_pull_request_exists(self, repo:github.Repository.Repository, head:str) -> bool:
 310         pulls = repo.get_pulls(head=head)
 311         return pulls.totalCount != 0
 312
 313     def create_pull_request(self, owner:str, repo_name:str, branch:str) -> bool:
 314         """
 315         reate a pull request in `self.branch_repo_name`.  The base branch of the
 316         pull request will be choosen based on the the milestone attached to
 317         the issue represented by `self.issue_number`  For example if the milestone
 318         is Release 13.0.1, then the base branch will be release/13.x. `branch`
 319         will be used as the compare branch.
 320         https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
 321         https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
 322         """
 323         repo = github.Github(self.token).get_repo(self.branch_repo_name)
 324         issue_ref = '{}#{}'.format(self.repo_name, self.issue_number)
 325         pull = None
 326         release_branch_for_issue = self.release_branch_for_issue
 327         if release_branch_for_issue is None:
 328             return False
 329         head_branch = branch
 330         if not repo.fork:
 331             # If the target repo is not a fork of llvm-project, we need to copy
 332             # the branch into the target repo.  GitHub only supports cross-repo pull
 333             # requests on forked repos.
 334             head_branch = f'{owner}-{branch}'
 335             local_repo = Repo(self.llvm_project_dir)
 336             push_done = False
 337             for i in range(0,5):
 338                 try:
 339                     local_repo.git.fetch(f'https://github.com/{owner}/{repo_name}', f'{branch}:{branch}')
 340                     local_repo.git.push(self.push_url, f'{branch}:{head_branch}', force=True)
 341                     push_done = True
 342                     break
 343                 except Exception as e:
 344                     print(e)
 345                     time.sleep(30)
 346                     continue
 347             if not push_done:
 348                 raise Exception("Failed to mirror branch into {}".format(self.push_url))
 349             owner = repo.owner.login
 350
 351         head = f"{owner}:{head_branch}"
 352         if self.check_if_pull_request_exists(repo, head):
 353             print("PR already exists...")
 354             return True
 355         try:
 356             pull = repo.create_pull(title=f"PR for {issue_ref}",
 357                                     body='resolves {}'.format(issue_ref),
 358                                     base=release_branch_for_issue,
 359                                     head=head,
 360                                     maintainer_can_modify=False)
 361
 362             try:
 363                 if self.phab_token:
 364                     self.pr_request_review(pull)
 365             except Exception as e:
 366                 print("error: Failed while searching for reviewers", e)
 367
 368         except Exception as e:
 369             self.issue_notify_pull_request_failure(branch)
 370             raise e
 371
 372         if pull is None:
 373             return False
 374
 375         self.issue_notify_pull_request(pull)
 376         self.issue_remove_cherry_pick_failed_label()
 377
 378         # TODO(tstellar): Do you really want to always return True?
 379         return True
 380
 381
 382     def execute_command(self) -> bool:
 383         """
 384         This function reads lines from STDIN and executes the first command
 385         that it finds.  The 2 supported commands are:
 386         /cherry-pick commit0 <commit1> <commit2> <...>
 387         /branch <owner>/<repo>/<branch>
 388         """
 389         for line in sys.stdin:
 390             line.rstrip()
 391             m = re.search("/([a-z-]+)\s(.+)", line)
 392             if not m:
 393                 continue
 394             command = m.group(1)
 395             args = m.group(2)
 396
 397             if command == 'cherry-pick':
 398                 return self.create_branch(args.split())
 399
 400             if command == 'branch':
 401                 m = re.match('([^/]+)/([^/]+)/(.+)', args)
 402                 if m:
 403                     owner = m.group(1)
 404                     repo = m.group(2)
 405                     branch = m.group(3)
 406                     return self.create_pull_request(owner, repo, branch)
 407
 408         print("Do not understand input:")
 409         print(sys.stdin.readlines())
 410         return False
 411
 412 parser = argparse.ArgumentParser()
 413 parser.add_argument('--token', type=str, required=True, help='GitHub authentiation token')
 414 parser.add_argument('--repo', type=str, default=os.getenv('GITHUB_REPOSITORY', 'llvm/llvm-project'),
 415                     help='The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)')
 416 subparsers = parser.add_subparsers(dest='command')
 417
 418 issue_subscriber_parser = subparsers.add_parser('issue-subscriber')
 419 issue_subscriber_parser.add_argument('--label-name', type=str, required=True)
 420 issue_subscriber_parser.add_argument('--issue-number', type=int, required=True)
 421
 422 release_workflow_parser = subparsers.add_parser('release-workflow')
 423 release_workflow_parser.add_argument('--llvm-project-dir', type=str, default='.', help='directory containing the llvm-project checout')
 424 release_workflow_parser.add_argument('--issue-number', type=int, required=True, help='The issue number to update')
 425 release_workflow_parser.add_argument('--phab-token', type=str, help='Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/')
 426 release_workflow_parser.add_argument('--branch-repo-token', type=str,
 427                                      help='GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.')
 428 release_workflow_parser.add_argument('--branch-repo', type=str, default='llvm/llvm-project-release-prs',
 429                                      help='The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)')
 430 release_workflow_parser.add_argument('sub_command', type=str, choices=['print-release-branch', 'auto'],
 431                                      help='Print to stdout the name of the release branch ISSUE_NUMBER should be backported to')
 432
 433 llvmbot_git_config_parser = subparsers.add_parser('setup-llvmbot-git', help='Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot')
 434
 435 args = parser.parse_args()
 436
 437 if args.command == 'issue-subscriber':
 438     issue_subscriber = IssueSubscriber(args.token, args.repo, args.issue_number, args.label_name)
 439     issue_subscriber.run()
 440 elif args.command == 'release-workflow':
 441     release_workflow = ReleaseWorkflow(args.token, args.repo, args.issue_number,
 442                                        args.branch_repo, args.branch_repo_token,
 443                                        args.llvm_project_dir, args.phab_token)
 444     if not release_workflow.release_branch_for_issue:
 445         release_workflow.issue_notify_no_milestone(sys.stdin.readlines())
 446         sys.exit(1)
 447     if args.sub_command == 'print-release-branch':
 448         release_workflow.print_release_branch()
 449     else:
 450         if not release_workflow.execute_command():
 451             sys.exit(1)
 452 elif args.command == 'setup-llvmbot-git':
 453     setup_llvmbot_git()