3 # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==-------------------------------------------------------------------------==#
12 from git
import Repo
# type: ignore
20 from typing
import List
, Optional
22 beginner_comment
= """
25 This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are:
27 1. In the comments of the issue, request for it to be assigned to you.
28 2. Fix the issue locally.
29 3. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests.
30 4. Create a Git commit.
31 5. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes.
32 6. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request).
34 If you have any further questions about this issue, don't hesitate to ask via a comment in the thread below.
38 def _get_curent_team(team_name
, teams
) -> Optional
[github
.Team
.Team
]:
40 if team_name
== team
.name
.lower():
45 def escape_description(str):
46 # If the description of an issue/pull request is empty, the Github API
47 # library returns None instead of an empty string. Handle this here to
48 # avoid failures from trying to manipulate None.
51 # https://github.com/github/markup/issues/1168#issuecomment-494946168
52 str = html
.escape(str, False)
53 # '@' followed by alphanum is a user name
54 str = re
.sub("@(?=\w)", "@<!-- -->", str)
55 # '#' followed by digits is considered an issue number
56 str = re
.sub("#(?=\d)", "#<!-- -->", str)
60 class IssueSubscriber
:
62 def team_name(self
) -> str:
63 return self
._team
_name
65 def __init__(self
, token
: str, repo
: str, issue_number
: int, label_name
: str):
66 self
.repo
= github
.Github(token
).get_repo(repo
)
67 self
.org
= github
.Github(token
).get_organization(self
.repo
.organization
.login
)
68 self
.issue
= self
.repo
.get_issue(issue_number
)
69 self
._team
_name
= "issue-subscribers-{}".format(label_name
).lower()
71 def run(self
) -> bool:
72 team
= _get_curent_team(self
.team_name
, self
.org
.get_teams())
74 print(f
"couldn't find team named {self.team_name}")
78 if team
.slug
== "issue-subscribers-good-first-issue":
79 comment
= "{}\n".format(beginner_comment
)
80 self
.issue
.create_comment(comment
)
82 body
= escape_description(self
.issue
.body
)
86 Author: {self.issue.user.name} ({self.issue.user.login})
93 self
.issue
.create_comment(comment
)
97 def human_readable_size(size
, decimal_places
=2):
98 for unit
in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
99 if size
< 1024.0 or unit
== "PiB":
102 return f
"{size:.{decimal_places}f} {unit}"
107 def team_name(self
) -> str:
108 return self
._team
_name
110 def __init__(self
, token
: str, repo
: str, pr_number
: int, label_name
: str):
111 self
.repo
= github
.Github(token
).get_repo(repo
)
112 self
.org
= github
.Github(token
).get_organization(self
.repo
.organization
.login
)
113 self
.pr
= self
.repo
.get_issue(pr_number
).as_pull_request()
114 self
._team
_name
= "pr-subscribers-{}".format(
115 label_name
.replace("+", "x")
117 self
.COMMENT_TAG
= "<!--LLVM PR SUMMARY COMMENT-->\n"
119 def get_summary_comment(self
) -> github
.IssueComment
.IssueComment
:
120 for comment
in self
.pr
.as_issue().get_comments():
121 if self
.COMMENT_TAG
in comment
.body
:
125 def run(self
) -> bool:
127 team
= _get_curent_team(self
.team_name
, self
.org
.get_teams())
129 print(f
"couldn't find team named {self.team_name}")
132 # GitHub limits comments to 65,536 characters, let's limit the diff
133 # and the file list to 20kB each.
134 STAT_LIMIT
= 20 * 1024
135 DIFF_LIMIT
= 20 * 1024
137 # Get statistics for each file
138 diff_stats
= f
"{self.pr.changed_files} Files Affected:\n\n"
139 for file in self
.pr
.get_files():
140 diff_stats
+= f
"- ({file.status}) {file.filename} ("
142 diff_stats
+= f
"+{file.additions}"
144 diff_stats
+= f
"-{file.deletions}"
146 if file.status
== "renamed":
147 print(f
"(from {file.previous_filename})")
149 if len(diff_stats
) > STAT_LIMIT
:
154 patch
= requests
.get(self
.pr
.diff_url
).text
158 patch_link
= f
"Full diff: {self.pr.diff_url}\n"
159 if len(patch
) > DIFF_LIMIT
:
160 patch_link
= f
"\nPatch is {human_readable_size(len(patch))}, truncated to {human_readable_size(DIFF_LIMIT)} below, full version: {self.pr.diff_url}\n"
161 patch
= patch
[0:DIFF_LIMIT
] + "...\n[truncated]\n"
162 team_mention
= "@llvm/{}".format(team
.slug
)
164 body
= escape_description(self
.pr
.body
)
165 # Note: the comment is in markdown and the code below
166 # is sensible to line break
171 Author: {self.pr.user.name} ({self.pr.user.login})
174 <summary>Changes</summary>
190 summary_comment
= self
.get_summary_comment()
191 if not summary_comment
:
192 self
.pr
.as_issue().create_comment(comment
)
193 elif team_mention
+ "\n" in summary_comment
.body
:
194 print("Team {} already mentioned.".format(team
.slug
))
196 summary_comment
.edit(
197 summary_comment
.body
.replace(
198 self
.COMMENT_TAG
, self
.COMMENT_TAG
+ team_mention
+ "\n"
203 def _get_curent_team(self
) -> Optional
[github
.Team
.Team
]:
204 for team
in self
.org
.get_teams():
205 if self
.team_name
== team
.name
.lower():
211 def __init__(self
, token
: str, repo
: str, pr_number
: int):
212 repo
= github
.Github(token
).get_repo(repo
)
213 self
.pr
= repo
.get_issue(pr_number
).as_pull_request()
215 def run(self
) -> bool:
216 # We assume that this is only called for a PR that has just been opened
217 # by a user new to LLVM and/or GitHub itself.
219 # This text is using Markdown formatting.
221 Thank you for submitting a Pull Request (PR) to the LLVM Project!
223 This PR will be automatically labeled and the relevant teams will be
226 If you wish to, you can add reviewers by using the "Reviewers" section on this page.
228 If this is not working for you, it is probably because you do not have write
229 permissions for the repository. In which case you can instead tag reviewers by
230 name in a comment by using `@` followed by their GitHub username.
232 If you have received no comments on your PR for a week, you can request a review
233 by "ping"ing the PR by adding a comment “Ping”. The common courtesy "ping" rate
234 is once a week. Please remember that you are asking for valuable time from other developers.
236 If you have further questions, they may be answered by the [LLVM GitHub User Guide](https://llvm.org/docs/GitHub.html).
238 You can also ask questions in a comment on this PR, on the [LLVM Discord](https://discord.com/invite/xS7Z362) or on the [forums](https://discourse.llvm.org/)."""
239 self
.pr
.as_issue().create_comment(comment
)
243 def setup_llvmbot_git(git_dir
="."):
245 Configure the git repo in `git_dir` with the llvmbot account so
246 commits are attributed to llvmbot.
249 with repo
.config_writer() as config
:
250 config
.set_value("user", "name", "llvmbot")
251 config
.set_value("user", "email", "llvmbot@llvm.org")
254 def phab_api_call(phab_token
: str, url
: str, args
: dict) -> dict:
256 Make an API call to the Phabricator web service and return a dictionary
257 containing the json response.
259 data
= {"api.token": phab_token
}
261 response
= requests
.post(url
, data
=data
)
262 return response
.json()
265 def phab_login_to_github_login(
266 phab_token
: str, repo
: github
.Repository
.Repository
, phab_login
: str
269 Tries to translate a Phabricator login to a github login by
270 finding a commit made in Phabricator's Differential.
271 The commit's SHA1 is then looked up in the github repo and
272 the committer's login associated with that commit is returned.
274 :param str phab_token: The Conduit API token to use for communication with Pabricator
275 :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
276 :param str phab_login: The Phabricator login to be translated.
280 "constraints[authors][0]": phab_login
,
281 # PHID for "LLVM Github Monorepo" repository
282 "constraints[repositories][0]": "PHID-REPO-f4scjekhnkmh7qilxlcy",
285 # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
287 phab_token
, "https://reviews.llvm.org/api/diffusion.commit.search", args
289 data
= r
["result"]["data"]
291 # Can't find any commits associated with this user
294 commit_sha
= data
[0]["fields"]["identifier"]
295 committer
= repo
.get_commit(commit_sha
).committer
297 # This committer had an email address GitHub could not recognize, so
298 # it can't link the user to a GitHub account.
299 print(f
"Warning: Can't find github account for {phab_login}")
301 return committer
.login
304 def phab_get_commit_approvers(phab_token
: str, commit
: github
.Commit
.Commit
) -> list:
305 args
= {"corpus": commit
.commit
.message
}
306 # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
308 phab_token
, "https://reviews.llvm.org/api/differential.parsecommitmessage", args
310 review_id
= r
["result"]["revisionIDFieldInfo"]["value"]
312 # No Phabricator revision for this commit
315 args
= {"constraints[ids][0]": review_id
, "attachments[reviewers]": True}
316 # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
318 phab_token
, "https://reviews.llvm.org/api/differential.revision.search", args
320 reviewers
= r
["result"]["data"][0]["attachments"]["reviewers"]["reviewers"]
322 for reviewer
in reviewers
:
323 if reviewer
["status"] != "accepted":
325 phid
= reviewer
["reviewerPHID"]
326 args
= {"constraints[phids][0]": phid
}
327 # API documentation: https://reviews.llvm.org/conduit/method/user.search/
328 r
= phab_api_call(phab_token
, "https://reviews.llvm.org/api/user.search", args
)
329 accepted
.append(r
["result"]["data"][0]["fields"]["username"])
333 def extract_commit_hash(arg
: str):
335 Extract the commit hash from the argument passed to /action github
336 comment actions. We currently only support passing the commit hash
337 directly or use the github URL, such as
338 https://github.com/llvm/llvm-project/commit/2832d7941f4207f1fcf813b27cf08cecc3086959
340 github_prefix
= "https://github.com/llvm/llvm-project/commit/"
341 if arg
.startswith(github_prefix
):
342 return arg
[len(github_prefix
) :]
346 class ReleaseWorkflow
:
347 CHERRY_PICK_FAILED_LABEL
= "release:cherry-pick-failed"
350 This class implements the sub-commands for the release-workflow command.
351 The current sub-commands are:
353 * create-pull-request
355 The execute_command method will automatically choose the correct sub-command
356 based on the text in stdin.
364 branch_repo_name
: str,
365 branch_repo_token
: str,
366 llvm_project_dir
: str,
370 self
._repo
_name
= repo
371 self
._issue
_number
= issue_number
372 self
._branch
_repo
_name
= branch_repo_name
373 if branch_repo_token
:
374 self
._branch
_repo
_token
= branch_repo_token
376 self
._branch
_repo
_token
= self
.token
377 self
._llvm
_project
_dir
= llvm_project_dir
378 self
._phab
_token
= phab_token
381 def token(self
) -> str:
385 def repo_name(self
) -> str:
386 return self
._repo
_name
389 def issue_number(self
) -> int:
390 return self
._issue
_number
393 def branch_repo_name(self
) -> str:
394 return self
._branch
_repo
_name
397 def branch_repo_token(self
) -> str:
398 return self
._branch
_repo
_token
401 def llvm_project_dir(self
) -> str:
402 return self
._llvm
_project
_dir
405 def phab_token(self
) -> str:
406 return self
._phab
_token
409 def repo(self
) -> github
.Repository
.Repository
:
410 return github
.Github(self
.token
).get_repo(self
.repo_name
)
413 def issue(self
) -> github
.Issue
.Issue
:
414 return self
.repo
.get_issue(self
.issue_number
)
417 def push_url(self
) -> str:
418 return "https://{}@github.com/{}".format(
419 self
.branch_repo_token
, self
.branch_repo_name
423 def branch_name(self
) -> str:
424 return "issue{}".format(self
.issue_number
)
427 def release_branch_for_issue(self
) -> Optional
[str]:
429 milestone
= issue
.milestone
430 if milestone
is None:
432 m
= re
.search("branch: (.+)", milestone
.description
)
437 def print_release_branch(self
) -> None:
438 print(self
.release_branch_for_issue
)
440 def issue_notify_branch(self
) -> None:
441 self
.issue
.create_comment(
442 "/branch {}/{}".format(self
.branch_repo_name
, self
.branch_name
)
445 def issue_notify_pull_request(self
, pull
: github
.PullRequest
.PullRequest
) -> None:
446 self
.issue
.create_comment(
447 "/pull-request {}#{}".format(self
.branch_repo_name
, pull
.number
)
450 def make_ignore_comment(self
, comment
: str) -> str:
452 Returns the comment string with a prefix that will cause
453 a Github workflow to skip parsing this comment.
455 :param str comment: The comment to ignore
457 return "<!--IGNORE-->\n" + comment
459 def issue_notify_no_milestone(self
, comment
: List
[str]) -> None:
460 message
= "{}\n\nError: Command failed due to missing milestone.".format(
461 "".join([">" + line
for line
in comment
])
463 self
.issue
.create_comment(self
.make_ignore_comment(message
))
466 def action_url(self
) -> str:
468 return "https://github.com/{}/actions/runs/{}".format(
469 os
.getenv("GITHUB_REPOSITORY"), os
.getenv("GITHUB_RUN_ID")
473 def issue_notify_cherry_pick_failure(
475 ) -> github
.IssueComment
.IssueComment
:
476 message
= self
.make_ignore_comment(
477 "Failed to cherry-pick: {}\n\n".format(commit
)
479 action_url
= self
.action_url
481 message
+= action_url
+ "\n\n"
482 message
+= "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
484 comment
= issue
.create_comment(message
)
485 issue
.add_to_labels(self
.CHERRY_PICK_FAILED_LABEL
)
488 def issue_notify_pull_request_failure(
490 ) -> github
.IssueComment
.IssueComment
:
491 message
= "Failed to create pull request for {} ".format(branch
)
492 message
+= self
.action_url
493 return self
.issue
.create_comment(message
)
495 def issue_remove_cherry_pick_failed_label(self
):
496 if self
.CHERRY_PICK_FAILED_LABEL
in [l
.name
for l
in self
.issue
.labels
]:
497 self
.issue
.remove_from_labels(self
.CHERRY_PICK_FAILED_LABEL
)
499 def pr_request_review(self
, pr
: github
.PullRequest
.PullRequest
):
501 This function will try to find the best reviewers for `commits` and
502 then add a comment requesting review of the backport and assign the
503 pull request to the selected reviewers.
505 The reviewers selected are those users who approved the patch in
509 for commit
in pr
.get_commits():
510 approvers
= phab_get_commit_approvers(self
.phab_token
, commit
)
512 login
= phab_login_to_github_login(self
.phab_token
, self
.repo
, a
)
515 reviewers
.append(login
)
517 message
= "{} What do you think about merging this PR to the release branch?".format(
518 " ".join(["@" + r
for r
in reviewers
])
520 pr
.create_issue_comment(message
)
521 pr
.add_to_assignees(*reviewers
)
523 def create_branch(self
, commits
: List
[str]) -> bool:
525 This function attempts to backport `commits` into the branch associated
526 with `self.issue_number`.
528 If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
529 a comment is added to the issue saying that the cherry-pick failed.
531 :param list commits: List of commits to cherry-pick.
534 print("cherry-picking", commits
)
535 branch_name
= self
.branch_name
536 local_repo
= Repo(self
.llvm_project_dir
)
537 local_repo
.git
.checkout(self
.release_branch_for_issue
)
541 local_repo
.git
.cherry_pick("-x", c
)
542 except Exception as e
:
543 self
.issue_notify_cherry_pick_failure(c
)
546 push_url
= self
.push_url
547 print("Pushing to {} {}".format(push_url
, branch_name
))
548 local_repo
.git
.push(push_url
, "HEAD:{}".format(branch_name
), force
=True)
550 self
.issue_notify_branch()
551 self
.issue_remove_cherry_pick_failed_label()
554 def check_if_pull_request_exists(
555 self
, repo
: github
.Repository
.Repository
, head
: str
557 pulls
= repo
.get_pulls(head
=head
)
558 return pulls
.totalCount
!= 0
560 def create_pull_request(self
, owner
: str, repo_name
: str, branch
: str) -> bool:
562 reate a pull request in `self.branch_repo_name`. The base branch of the
563 pull request will be chosen based on the the milestone attached to
564 the issue represented by `self.issue_number` For example if the milestone
565 is Release 13.0.1, then the base branch will be release/13.x. `branch`
566 will be used as the compare branch.
567 https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
568 https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
570 repo
= github
.Github(self
.token
).get_repo(self
.branch_repo_name
)
571 issue_ref
= "{}#{}".format(self
.repo_name
, self
.issue_number
)
573 release_branch_for_issue
= self
.release_branch_for_issue
574 if release_branch_for_issue
is None:
578 # If the target repo is not a fork of llvm-project, we need to copy
579 # the branch into the target repo. GitHub only supports cross-repo pull
580 # requests on forked repos.
581 head_branch
= f
"{owner}-{branch}"
582 local_repo
= Repo(self
.llvm_project_dir
)
584 for _
in range(0, 5):
586 local_repo
.git
.fetch(
587 f
"https://github.com/{owner}/{repo_name}", f
"{branch}:{branch}"
590 self
.push_url
, f
"{branch}:{head_branch}", force
=True
594 except Exception as e
:
599 raise Exception("Failed to mirror branch into {}".format(self
.push_url
))
600 owner
= repo
.owner
.login
602 head
= f
"{owner}:{head_branch}"
603 if self
.check_if_pull_request_exists(repo
, head
):
604 print("PR already exists...")
607 pull
= repo
.create_pull(
608 title
=f
"PR for {issue_ref}",
609 body
="resolves {}".format(issue_ref
),
610 base
=release_branch_for_issue
,
612 maintainer_can_modify
=False,
617 self
.pr_request_review(pull
)
618 except Exception as e
:
619 print("error: Failed while searching for reviewers", e
)
621 except Exception as e
:
622 self
.issue_notify_pull_request_failure(branch
)
628 self
.issue_notify_pull_request(pull
)
629 self
.issue_remove_cherry_pick_failed_label()
631 # TODO(tstellar): Do you really want to always return True?
634 def execute_command(self
) -> bool:
636 This function reads lines from STDIN and executes the first command
637 that it finds. The 2 supported commands are:
638 /cherry-pick commit0 <commit1> <commit2> <...>
639 /branch <owner>/<repo>/<branch>
641 for line
in sys
.stdin
:
643 m
= re
.search(r
"/([a-z-]+)\s(.+)", line
)
649 if command
== "cherry-pick":
650 arg_list
= args
.split()
651 commits
= list(map(lambda a
: extract_commit_hash(a
), arg_list
))
652 return self
.create_branch(commits
)
654 if command
== "branch":
655 m
= re
.match("([^/]+)/([^/]+)/(.+)", args
)
660 return self
.create_pull_request(owner
, repo
, branch
)
662 print("Do not understand input:")
663 print(sys
.stdin
.readlines())
667 parser
= argparse
.ArgumentParser()
669 "--token", type=str, required
=True, help="GitHub authentiation token"
674 default
=os
.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"),
675 help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)",
677 subparsers
= parser
.add_subparsers(dest
="command")
679 issue_subscriber_parser
= subparsers
.add_parser("issue-subscriber")
680 issue_subscriber_parser
.add_argument("--label-name", type=str, required
=True)
681 issue_subscriber_parser
.add_argument("--issue-number", type=int, required
=True)
683 pr_subscriber_parser
= subparsers
.add_parser("pr-subscriber")
684 pr_subscriber_parser
.add_argument("--label-name", type=str, required
=True)
685 pr_subscriber_parser
.add_argument("--issue-number", type=int, required
=True)
687 pr_greeter_parser
= subparsers
.add_parser("pr-greeter")
688 pr_greeter_parser
.add_argument("--issue-number", type=int, required
=True)
690 release_workflow_parser
= subparsers
.add_parser("release-workflow")
691 release_workflow_parser
.add_argument(
692 "--llvm-project-dir",
695 help="directory containing the llvm-project checout",
697 release_workflow_parser
.add_argument(
698 "--issue-number", type=int, required
=True, help="The issue number to update"
700 release_workflow_parser
.add_argument(
703 help="Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/",
705 release_workflow_parser
.add_argument(
706 "--branch-repo-token",
708 help="GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.",
710 release_workflow_parser
.add_argument(
713 default
="llvm/llvm-project-release-prs",
714 help="The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)",
716 release_workflow_parser
.add_argument(
719 choices
=["print-release-branch", "auto"],
720 help="Print to stdout the name of the release branch ISSUE_NUMBER should be backported to",
723 llvmbot_git_config_parser
= subparsers
.add_parser(
725 help="Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot",
728 args
= parser
.parse_args()
730 if args
.command
== "issue-subscriber":
731 issue_subscriber
= IssueSubscriber(
732 args
.token
, args
.repo
, args
.issue_number
, args
.label_name
734 issue_subscriber
.run()
735 elif args
.command
== "pr-subscriber":
736 pr_subscriber
= PRSubscriber(
737 args
.token
, args
.repo
, args
.issue_number
, args
.label_name
740 elif args
.command
== "pr-greeter":
741 pr_greeter
= PRGreeter(args
.token
, args
.repo
, args
.issue_number
)
743 elif args
.command
== "release-workflow":
744 release_workflow
= ReleaseWorkflow(
749 args
.branch_repo_token
,
750 args
.llvm_project_dir
,
753 if not release_workflow
.release_branch_for_issue
:
754 release_workflow
.issue_notify_no_milestone(sys
.stdin
.readlines())
756 if args
.sub_command
== "print-release-branch":
757 release_workflow
.print_release_branch()
759 if not release_workflow
.execute_command():
761 elif args
.command
== "setup-llvmbot-git":