3 # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==-------------------------------------------------------------------------==#
12 from git
import Repo
# type: ignore
20 from typing
import List
, Optional
22 beginner_comment
= """
25 This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are:
27 1) Assign the issue to you.
28 2) Fix the issue locally.
29 3) [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally.
30 3.1) Remember that the subdirectories under `test/` create fine-grained testing targets, so you can
31 e.g. use `make check-clang-ast` to only run Clang's AST tests.
32 4) Create a `git` commit
33 5) Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes.
34 6) Submit the patch to [Phabricator](https://reviews.llvm.org/).
35 6.1) Detailed instructions can be found [here](https://llvm.org/docs/Phabricator.html#requesting-a-review-via-the-web-interface)
37 For more instructions on how to submit a patch to LLVM, see our [documentation](https://llvm.org/docs/Contributing.html).
39 If you have any further questions about this issue, don't hesitate to ask via a comment on this Github issue.
43 def _get_curent_team(team_name
, teams
) -> Optional
[github
.Team
.Team
]:
45 if team_name
== team
.name
.lower():
50 def escape_description(str):
51 # https://github.com/github/markup/issues/1168#issuecomment-494946168
52 str = html
.escape(str, False)
53 # '@' followed by alphanum is a user name
54 str = re
.sub("@(?=\w)", "@<!-- -->", str)
55 # '#' followed by digits is considered an issue number
56 str = re
.sub("#(?=\d)", "#<!-- -->", str)
60 class IssueSubscriber
:
62 def team_name(self
) -> str:
63 return self
._team
_name
65 def __init__(self
, token
: str, repo
: str, issue_number
: int, label_name
: str):
66 self
.repo
= github
.Github(token
).get_repo(repo
)
67 self
.org
= github
.Github(token
).get_organization(self
.repo
.organization
.login
)
68 self
.issue
= self
.repo
.get_issue(issue_number
)
69 self
._team
_name
= "issue-subscribers-{}".format(label_name
).lower()
71 def run(self
) -> bool:
72 team
= _get_curent_team(self
.team_name
, self
.org
.get_teams())
74 print(f
"couldn't find team named {self.team_name}")
77 if team
.slug
== "issue-subscribers-good-first-issue":
78 comment
= "{}\n".format(beginner_comment
)
80 body
= escape_description(self
.issue
.body
)
85 Author: {self.issue.user.name} ({self.issue.user.login})
92 self
.issue
.create_comment(comment
)
96 def human_readable_size(size
, decimal_places
=2):
97 for unit
in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
98 if size
< 1024.0 or unit
== "PiB":
101 return f
"{size:.{decimal_places}f} {unit}"
106 def team_name(self
) -> str:
107 return self
._team
_name
109 def __init__(self
, token
: str, repo
: str, pr_number
: int, label_name
: str):
110 self
.repo
= github
.Github(token
).get_repo(repo
)
111 self
.org
= github
.Github(token
).get_organization(self
.repo
.organization
.login
)
112 self
.pr
= self
.repo
.get_issue(pr_number
).as_pull_request()
113 self
._team
_name
= "pr-subscribers-{}".format(
114 label_name
.replace("+", "x")
116 self
.COMMENT_TAG
= "<!--LLVM PR SUMMARY COMMENT-->\n"
118 def get_summary_comment(self
) -> github
.IssueComment
.IssueComment
:
119 for comment
in self
.pr
.as_issue().get_comments():
120 if self
.COMMENT_TAG
in comment
.body
:
124 def run(self
) -> bool:
126 team
= _get_curent_team(self
.team_name
, self
.org
.get_teams())
128 print(f
"couldn't find team named {self.team_name}")
131 # GitHub limits comments to 65,536 characters, let's limit the diff
132 # and the file list to 20kB each.
133 STAT_LIMIT
= 20 * 1024
134 DIFF_LIMIT
= 20 * 1024
136 # Get statistics for each file
137 diff_stats
= f
"{self.pr.changed_files} Files Affected:\n\n"
138 for file in self
.pr
.get_files():
139 diff_stats
+= f
"- ({file.status}) {file.filename} ("
141 diff_stats
+= f
"+{file.additions}"
143 diff_stats
+= f
"-{file.deletions}"
145 if file.status
== "renamed":
146 print(f
"(from {file.previous_filename})")
148 if len(diff_stats
) > STAT_LIMIT
:
153 patch
= requests
.get(self
.pr
.diff_url
).text
157 patch_link
= f
"Full diff: {self.pr.diff_url}\n"
158 if len(patch
) > DIFF_LIMIT
:
159 patch_link
= f
"\nPatch is {human_readable_size(len(patch))}, truncated to {human_readable_size(DIFF_LIMIT)} below, full version: {self.pr.diff_url}\n"
160 patch
= patch
[0:DIFF_LIMIT
] + "...\n[truncated]\n"
161 team_mention
= "@llvm/{}".format(team
.slug
)
163 body
= escape_description(self
.pr
.body
)
164 # Note: the comment is in markdown and the code below
165 # is sensible to line break
170 Author: {self.pr.user.name} ({self.pr.user.login})
173 <summary>Changes</summary>
189 summary_comment
= self
.get_summary_comment()
190 if not summary_comment
:
191 self
.pr
.as_issue().create_comment(comment
)
192 elif team_mention
+ "\n" in summary_comment
.body
:
193 print("Team {} already mentioned.".format(team
.slug
))
195 summary_comment
.edit(
196 summary_comment
.body
.replace(
197 self
.COMMENT_TAG
, self
.COMMENT_TAG
+ team_mention
+ "\n"
202 def _get_curent_team(self
) -> Optional
[github
.Team
.Team
]:
203 for team
in self
.org
.get_teams():
204 if self
.team_name
== team
.name
.lower():
209 def setup_llvmbot_git(git_dir
="."):
211 Configure the git repo in `git_dir` with the llvmbot account so
212 commits are attributed to llvmbot.
215 with repo
.config_writer() as config
:
216 config
.set_value("user", "name", "llvmbot")
217 config
.set_value("user", "email", "llvmbot@llvm.org")
220 def phab_api_call(phab_token
: str, url
: str, args
: dict) -> dict:
222 Make an API call to the Phabricator web service and return a dictionary
223 containing the json response.
225 data
= {"api.token": phab_token
}
227 response
= requests
.post(url
, data
=data
)
228 return response
.json()
231 def phab_login_to_github_login(
232 phab_token
: str, repo
: github
.Repository
.Repository
, phab_login
: str
235 Tries to translate a Phabricator login to a github login by
236 finding a commit made in Phabricator's Differential.
237 The commit's SHA1 is then looked up in the github repo and
238 the committer's login associated with that commit is returned.
240 :param str phab_token: The Conduit API token to use for communication with Pabricator
241 :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
242 :param str phab_login: The Phabricator login to be translated.
246 "constraints[authors][0]": phab_login
,
247 # PHID for "LLVM Github Monorepo" repository
248 "constraints[repositories][0]": "PHID-REPO-f4scjekhnkmh7qilxlcy",
251 # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
253 phab_token
, "https://reviews.llvm.org/api/diffusion.commit.search", args
255 data
= r
["result"]["data"]
257 # Can't find any commits associated with this user
260 commit_sha
= data
[0]["fields"]["identifier"]
261 committer
= repo
.get_commit(commit_sha
).committer
263 # This committer had an email address GitHub could not recognize, so
264 # it can't link the user to a GitHub account.
265 print(f
"Warning: Can't find github account for {phab_login}")
267 return committer
.login
270 def phab_get_commit_approvers(phab_token
: str, commit
: github
.Commit
.Commit
) -> list:
271 args
= {"corpus": commit
.commit
.message
}
272 # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
274 phab_token
, "https://reviews.llvm.org/api/differential.parsecommitmessage", args
276 review_id
= r
["result"]["revisionIDFieldInfo"]["value"]
278 # No Phabricator revision for this commit
281 args
= {"constraints[ids][0]": review_id
, "attachments[reviewers]": True}
282 # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
284 phab_token
, "https://reviews.llvm.org/api/differential.revision.search", args
286 reviewers
= r
["result"]["data"][0]["attachments"]["reviewers"]["reviewers"]
288 for reviewer
in reviewers
:
289 if reviewer
["status"] != "accepted":
291 phid
= reviewer
["reviewerPHID"]
292 args
= {"constraints[phids][0]": phid
}
293 # API documentation: https://reviews.llvm.org/conduit/method/user.search/
294 r
= phab_api_call(phab_token
, "https://reviews.llvm.org/api/user.search", args
)
295 accepted
.append(r
["result"]["data"][0]["fields"]["username"])
299 def extract_commit_hash(arg
: str):
301 Extract the commit hash from the argument passed to /action github
302 comment actions. We currently only support passing the commit hash
303 directly or use the github URL, such as
304 https://github.com/llvm/llvm-project/commit/2832d7941f4207f1fcf813b27cf08cecc3086959
306 github_prefix
= "https://github.com/llvm/llvm-project/commit/"
307 if arg
.startswith(github_prefix
):
308 return arg
[len(github_prefix
) :]
312 class ReleaseWorkflow
:
313 CHERRY_PICK_FAILED_LABEL
= "release:cherry-pick-failed"
316 This class implements the sub-commands for the release-workflow command.
317 The current sub-commands are:
319 * create-pull-request
321 The execute_command method will automatically choose the correct sub-command
322 based on the text in stdin.
330 branch_repo_name
: str,
331 branch_repo_token
: str,
332 llvm_project_dir
: str,
336 self
._repo
_name
= repo
337 self
._issue
_number
= issue_number
338 self
._branch
_repo
_name
= branch_repo_name
339 if branch_repo_token
:
340 self
._branch
_repo
_token
= branch_repo_token
342 self
._branch
_repo
_token
= self
.token
343 self
._llvm
_project
_dir
= llvm_project_dir
344 self
._phab
_token
= phab_token
347 def token(self
) -> str:
351 def repo_name(self
) -> str:
352 return self
._repo
_name
355 def issue_number(self
) -> int:
356 return self
._issue
_number
359 def branch_repo_name(self
) -> str:
360 return self
._branch
_repo
_name
363 def branch_repo_token(self
) -> str:
364 return self
._branch
_repo
_token
367 def llvm_project_dir(self
) -> str:
368 return self
._llvm
_project
_dir
371 def phab_token(self
) -> str:
372 return self
._phab
_token
375 def repo(self
) -> github
.Repository
.Repository
:
376 return github
.Github(self
.token
).get_repo(self
.repo_name
)
379 def issue(self
) -> github
.Issue
.Issue
:
380 return self
.repo
.get_issue(self
.issue_number
)
383 def push_url(self
) -> str:
384 return "https://{}@github.com/{}".format(
385 self
.branch_repo_token
, self
.branch_repo_name
389 def branch_name(self
) -> str:
390 return "issue{}".format(self
.issue_number
)
393 def release_branch_for_issue(self
) -> Optional
[str]:
395 milestone
= issue
.milestone
396 if milestone
is None:
398 m
= re
.search("branch: (.+)", milestone
.description
)
403 def print_release_branch(self
) -> None:
404 print(self
.release_branch_for_issue
)
406 def issue_notify_branch(self
) -> None:
407 self
.issue
.create_comment(
408 "/branch {}/{}".format(self
.branch_repo_name
, self
.branch_name
)
411 def issue_notify_pull_request(self
, pull
: github
.PullRequest
.PullRequest
) -> None:
412 self
.issue
.create_comment(
413 "/pull-request {}#{}".format(self
.branch_repo_name
, pull
.number
)
416 def make_ignore_comment(self
, comment
: str) -> str:
418 Returns the comment string with a prefix that will cause
419 a Github workflow to skip parsing this comment.
421 :param str comment: The comment to ignore
423 return "<!--IGNORE-->\n" + comment
425 def issue_notify_no_milestone(self
, comment
: List
[str]) -> None:
426 message
= "{}\n\nError: Command failed due to missing milestone.".format(
427 "".join([">" + line
for line
in comment
])
429 self
.issue
.create_comment(self
.make_ignore_comment(message
))
432 def action_url(self
) -> str:
434 return "https://github.com/{}/actions/runs/{}".format(
435 os
.getenv("GITHUB_REPOSITORY"), os
.getenv("GITHUB_RUN_ID")
439 def issue_notify_cherry_pick_failure(
441 ) -> github
.IssueComment
.IssueComment
:
442 message
= self
.make_ignore_comment(
443 "Failed to cherry-pick: {}\n\n".format(commit
)
445 action_url
= self
.action_url
447 message
+= action_url
+ "\n\n"
448 message
+= "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
450 comment
= issue
.create_comment(message
)
451 issue
.add_to_labels(self
.CHERRY_PICK_FAILED_LABEL
)
454 def issue_notify_pull_request_failure(
456 ) -> github
.IssueComment
.IssueComment
:
457 message
= "Failed to create pull request for {} ".format(branch
)
458 message
+= self
.action_url
459 return self
.issue
.create_comment(message
)
461 def issue_remove_cherry_pick_failed_label(self
):
462 if self
.CHERRY_PICK_FAILED_LABEL
in [l
.name
for l
in self
.issue
.labels
]:
463 self
.issue
.remove_from_labels(self
.CHERRY_PICK_FAILED_LABEL
)
465 def pr_request_review(self
, pr
: github
.PullRequest
.PullRequest
):
467 This function will try to find the best reviewers for `commits` and
468 then add a comment requesting review of the backport and assign the
469 pull request to the selected reviewers.
471 The reviewers selected are those users who approved the patch in
475 for commit
in pr
.get_commits():
476 approvers
= phab_get_commit_approvers(self
.phab_token
, commit
)
478 login
= phab_login_to_github_login(self
.phab_token
, self
.repo
, a
)
481 reviewers
.append(login
)
483 message
= "{} What do you think about merging this PR to the release branch?".format(
484 " ".join(["@" + r
for r
in reviewers
])
486 pr
.create_issue_comment(message
)
487 pr
.add_to_assignees(*reviewers
)
489 def create_branch(self
, commits
: List
[str]) -> bool:
491 This function attempts to backport `commits` into the branch associated
492 with `self.issue_number`.
494 If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
495 a comment is added to the issue saying that the cherry-pick failed.
497 :param list commits: List of commits to cherry-pick.
500 print("cherry-picking", commits
)
501 branch_name
= self
.branch_name
502 local_repo
= Repo(self
.llvm_project_dir
)
503 local_repo
.git
.checkout(self
.release_branch_for_issue
)
507 local_repo
.git
.cherry_pick("-x", c
)
508 except Exception as e
:
509 self
.issue_notify_cherry_pick_failure(c
)
512 push_url
= self
.push_url
513 print("Pushing to {} {}".format(push_url
, branch_name
))
514 local_repo
.git
.push(push_url
, "HEAD:{}".format(branch_name
), force
=True)
516 self
.issue_notify_branch()
517 self
.issue_remove_cherry_pick_failed_label()
520 def check_if_pull_request_exists(
521 self
, repo
: github
.Repository
.Repository
, head
: str
523 pulls
= repo
.get_pulls(head
=head
)
524 return pulls
.totalCount
!= 0
526 def create_pull_request(self
, owner
: str, repo_name
: str, branch
: str) -> bool:
528 reate a pull request in `self.branch_repo_name`. The base branch of the
529 pull request will be chosen based on the the milestone attached to
530 the issue represented by `self.issue_number` For example if the milestone
531 is Release 13.0.1, then the base branch will be release/13.x. `branch`
532 will be used as the compare branch.
533 https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
534 https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
536 repo
= github
.Github(self
.token
).get_repo(self
.branch_repo_name
)
537 issue_ref
= "{}#{}".format(self
.repo_name
, self
.issue_number
)
539 release_branch_for_issue
= self
.release_branch_for_issue
540 if release_branch_for_issue
is None:
544 # If the target repo is not a fork of llvm-project, we need to copy
545 # the branch into the target repo. GitHub only supports cross-repo pull
546 # requests on forked repos.
547 head_branch
= f
"{owner}-{branch}"
548 local_repo
= Repo(self
.llvm_project_dir
)
550 for _
in range(0, 5):
552 local_repo
.git
.fetch(
553 f
"https://github.com/{owner}/{repo_name}", f
"{branch}:{branch}"
556 self
.push_url
, f
"{branch}:{head_branch}", force
=True
560 except Exception as e
:
565 raise Exception("Failed to mirror branch into {}".format(self
.push_url
))
566 owner
= repo
.owner
.login
568 head
= f
"{owner}:{head_branch}"
569 if self
.check_if_pull_request_exists(repo
, head
):
570 print("PR already exists...")
573 pull
= repo
.create_pull(
574 title
=f
"PR for {issue_ref}",
575 body
="resolves {}".format(issue_ref
),
576 base
=release_branch_for_issue
,
578 maintainer_can_modify
=False,
583 self
.pr_request_review(pull
)
584 except Exception as e
:
585 print("error: Failed while searching for reviewers", e
)
587 except Exception as e
:
588 self
.issue_notify_pull_request_failure(branch
)
594 self
.issue_notify_pull_request(pull
)
595 self
.issue_remove_cherry_pick_failed_label()
597 # TODO(tstellar): Do you really want to always return True?
600 def execute_command(self
) -> bool:
602 This function reads lines from STDIN and executes the first command
603 that it finds. The 2 supported commands are:
604 /cherry-pick commit0 <commit1> <commit2> <...>
605 /branch <owner>/<repo>/<branch>
607 for line
in sys
.stdin
:
609 m
= re
.search(r
"/([a-z-]+)\s(.+)", line
)
615 if command
== "cherry-pick":
616 arg_list
= args
.split()
617 commits
= list(map(lambda a
: extract_commit_hash(a
), arg_list
))
618 return self
.create_branch(commits
)
620 if command
== "branch":
621 m
= re
.match("([^/]+)/([^/]+)/(.+)", args
)
626 return self
.create_pull_request(owner
, repo
, branch
)
628 print("Do not understand input:")
629 print(sys
.stdin
.readlines())
633 parser
= argparse
.ArgumentParser()
635 "--token", type=str, required
=True, help="GitHub authentiation token"
640 default
=os
.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"),
641 help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)",
643 subparsers
= parser
.add_subparsers(dest
="command")
645 issue_subscriber_parser
= subparsers
.add_parser("issue-subscriber")
646 issue_subscriber_parser
.add_argument("--label-name", type=str, required
=True)
647 issue_subscriber_parser
.add_argument("--issue-number", type=int, required
=True)
649 pr_subscriber_parser
= subparsers
.add_parser("pr-subscriber")
650 pr_subscriber_parser
.add_argument("--label-name", type=str, required
=True)
651 pr_subscriber_parser
.add_argument("--issue-number", type=int, required
=True)
653 release_workflow_parser
= subparsers
.add_parser("release-workflow")
654 release_workflow_parser
.add_argument(
655 "--llvm-project-dir",
658 help="directory containing the llvm-project checout",
660 release_workflow_parser
.add_argument(
661 "--issue-number", type=int, required
=True, help="The issue number to update"
663 release_workflow_parser
.add_argument(
666 help="Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/",
668 release_workflow_parser
.add_argument(
669 "--branch-repo-token",
671 help="GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.",
673 release_workflow_parser
.add_argument(
676 default
="llvm/llvm-project-release-prs",
677 help="The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)",
679 release_workflow_parser
.add_argument(
682 choices
=["print-release-branch", "auto"],
683 help="Print to stdout the name of the release branch ISSUE_NUMBER should be backported to",
686 llvmbot_git_config_parser
= subparsers
.add_parser(
688 help="Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot",
691 args
= parser
.parse_args()
693 if args
.command
== "issue-subscriber":
694 issue_subscriber
= IssueSubscriber(
695 args
.token
, args
.repo
, args
.issue_number
, args
.label_name
697 issue_subscriber
.run()
698 elif args
.command
== "pr-subscriber":
699 pr_subscriber
= PRSubscriber(
700 args
.token
, args
.repo
, args
.issue_number
, args
.label_name
703 elif args
.command
== "release-workflow":
704 release_workflow
= ReleaseWorkflow(
709 args
.branch_repo_token
,
710 args
.llvm_project_dir
,
713 if not release_workflow
.release_branch_for_issue
:
714 release_workflow
.issue_notify_no_milestone(sys
.stdin
.readlines())
716 if args
.sub_command
== "print-release-branch":
717 release_workflow
.print_release_branch()
719 if not release_workflow
.execute_command():
721 elif args
.command
== "setup-llvmbot-git":