Run DCE after a LoopFlatten test to reduce spurious output [nfc]
[llvm-project.git] / llvm / utils / git / github-automation.py
blob52523704fe82dc816ba211c47c6c7d96a9c3fe40
1 #!/usr/bin/env python3
3 # ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
5 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6 # See https://llvm.org/LICENSE.txt for license information.
7 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
9 # ==-------------------------------------------------------------------------==#
11 import argparse
12 from git import Repo # type: ignore
13 import html
14 import github
15 import os
16 import re
17 import requests
18 import sys
19 import time
20 from typing import List, Optional
22 beginner_comment = """
23 Hi!
25 This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are:
27 1) Assign the issue to you.
28 2) Fix the issue locally.
29 3) [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally.
30 3.1) Remember that the subdirectories under `test/` create fine-grained testing targets, so you can
31 e.g. use `make check-clang-ast` to only run Clang's AST tests.
32 4) Create a `git` commit
33 5) Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes.
34 6) Submit the patch to [Phabricator](https://reviews.llvm.org/).
35 6.1) Detailed instructions can be found [here](https://llvm.org/docs/Phabricator.html#requesting-a-review-via-the-web-interface)
37 For more instructions on how to submit a patch to LLVM, see our [documentation](https://llvm.org/docs/Contributing.html).
39 If you have any further questions about this issue, don't hesitate to ask via a comment on this Github issue.
40 """
43 def _get_curent_team(team_name, teams) -> Optional[github.Team.Team]:
44 for team in teams:
45 if team_name == team.name.lower():
46 return team
47 return None
50 def escape_description(str):
51 # https://github.com/github/markup/issues/1168#issuecomment-494946168
52 str = html.escape(str, False)
53 # '@' followed by alphanum is a user name
54 str = re.sub("@(?=\w)", "@<!-- -->", str)
55 # '#' followed by digits is considered an issue number
56 str = re.sub("#(?=\d)", "#<!-- -->", str)
57 return str
60 class IssueSubscriber:
61 @property
62 def team_name(self) -> str:
63 return self._team_name
65 def __init__(self, token: str, repo: str, issue_number: int, label_name: str):
66 self.repo = github.Github(token).get_repo(repo)
67 self.org = github.Github(token).get_organization(self.repo.organization.login)
68 self.issue = self.repo.get_issue(issue_number)
69 self._team_name = "issue-subscribers-{}".format(label_name).lower()
71 def run(self) -> bool:
72 team = _get_curent_team(self.team_name, self.org.get_teams())
73 if not team:
74 print(f"couldn't find team named {self.team_name}")
75 return False
76 comment = ""
77 if team.slug == "issue-subscribers-good-first-issue":
78 comment = "{}\n".format(beginner_comment)
80 body = escape_description(self.issue.body)
82 comment = f"""
83 @llvm/{team.slug}
85 Author: {self.issue.user.name} ({self.issue.user.login})
87 <details>
88 {body}
89 </details>
90 """
92 self.issue.create_comment(comment)
93 return True
96 def human_readable_size(size, decimal_places=2):
97 for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
98 if size < 1024.0 or unit == "PiB":
99 break
100 size /= 1024.0
101 return f"{size:.{decimal_places}f} {unit}"
104 class PRSubscriber:
105 @property
106 def team_name(self) -> str:
107 return self._team_name
109 def __init__(self, token: str, repo: str, pr_number: int, label_name: str):
110 self.repo = github.Github(token).get_repo(repo)
111 self.org = github.Github(token).get_organization(self.repo.organization.login)
112 self.pr = self.repo.get_issue(pr_number).as_pull_request()
113 self._team_name = "pr-subscribers-{}".format(
114 label_name.replace("+", "x")
115 ).lower()
116 self.COMMENT_TAG = "<!--LLVM PR SUMMARY COMMENT-->\n"
118 def get_summary_comment(self) -> github.IssueComment.IssueComment:
119 for comment in self.pr.as_issue().get_comments():
120 if self.COMMENT_TAG in comment.body:
121 return comment
122 return None
124 def run(self) -> bool:
125 patch = None
126 team = _get_curent_team(self.team_name, self.org.get_teams())
127 if not team:
128 print(f"couldn't find team named {self.team_name}")
129 return False
131 # GitHub limits comments to 65,536 characters, let's limit the diff
132 # and the file list to 20kB each.
133 STAT_LIMIT = 20 * 1024
134 DIFF_LIMIT = 20 * 1024
136 # Get statistics for each file
137 diff_stats = f"{self.pr.changed_files} Files Affected:\n\n"
138 for file in self.pr.get_files():
139 diff_stats += f"- ({file.status}) {file.filename} ("
140 if file.additions:
141 diff_stats += f"+{file.additions}"
142 if file.deletions:
143 diff_stats += f"-{file.deletions}"
144 diff_stats += ") "
145 if file.status == "renamed":
146 print(f"(from {file.previous_filename})")
147 diff_stats += "\n"
148 if len(diff_stats) > STAT_LIMIT:
149 break
151 # Get the diff
152 try:
153 patch = requests.get(self.pr.diff_url).text
154 except:
155 patch = ""
157 patch_link = f"Full diff: {self.pr.diff_url}\n"
158 if len(patch) > DIFF_LIMIT:
159 patch_link = f"\nPatch is {human_readable_size(len(patch))}, truncated to {human_readable_size(DIFF_LIMIT)} below, full version: {self.pr.diff_url}\n"
160 patch = patch[0:DIFF_LIMIT] + "...\n[truncated]\n"
161 team_mention = "@llvm/{}".format(team.slug)
163 body = escape_description(self.pr.body)
164 # Note: the comment is in markdown and the code below
165 # is sensible to line break
166 comment = f"""
167 {self.COMMENT_TAG}
168 {team_mention}
170 Author: {self.pr.user.name} ({self.pr.user.login})
172 <details>
173 <summary>Changes</summary>
175 {body}
178 {patch_link}
180 {diff_stats}
182 ``````````diff
183 {patch}
184 ``````````
186 </details>
189 summary_comment = self.get_summary_comment()
190 if not summary_comment:
191 self.pr.as_issue().create_comment(comment)
192 elif team_mention + "\n" in summary_comment.body:
193 print("Team {} already mentioned.".format(team.slug))
194 else:
195 summary_comment.edit(
196 summary_comment.body.replace(
197 self.COMMENT_TAG, self.COMMENT_TAG + team_mention + "\n"
200 return True
202 def _get_curent_team(self) -> Optional[github.Team.Team]:
203 for team in self.org.get_teams():
204 if self.team_name == team.name.lower():
205 return team
206 return None
209 def setup_llvmbot_git(git_dir="."):
211 Configure the git repo in `git_dir` with the llvmbot account so
212 commits are attributed to llvmbot.
214 repo = Repo(git_dir)
215 with repo.config_writer() as config:
216 config.set_value("user", "name", "llvmbot")
217 config.set_value("user", "email", "llvmbot@llvm.org")
220 def phab_api_call(phab_token: str, url: str, args: dict) -> dict:
222 Make an API call to the Phabricator web service and return a dictionary
223 containing the json response.
225 data = {"api.token": phab_token}
226 data.update(args)
227 response = requests.post(url, data=data)
228 return response.json()
231 def phab_login_to_github_login(
232 phab_token: str, repo: github.Repository.Repository, phab_login: str
233 ) -> Optional[str]:
235 Tries to translate a Phabricator login to a github login by
236 finding a commit made in Phabricator's Differential.
237 The commit's SHA1 is then looked up in the github repo and
238 the committer's login associated with that commit is returned.
240 :param str phab_token: The Conduit API token to use for communication with Pabricator
241 :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
242 :param str phab_login: The Phabricator login to be translated.
245 args = {
246 "constraints[authors][0]": phab_login,
247 # PHID for "LLVM Github Monorepo" repository
248 "constraints[repositories][0]": "PHID-REPO-f4scjekhnkmh7qilxlcy",
249 "limit": 1,
251 # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
252 r = phab_api_call(
253 phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args
255 data = r["result"]["data"]
256 if len(data) == 0:
257 # Can't find any commits associated with this user
258 return None
260 commit_sha = data[0]["fields"]["identifier"]
261 committer = repo.get_commit(commit_sha).committer
262 if not committer:
263 # This committer had an email address GitHub could not recognize, so
264 # it can't link the user to a GitHub account.
265 print(f"Warning: Can't find github account for {phab_login}")
266 return None
267 return committer.login
270 def phab_get_commit_approvers(phab_token: str, commit: github.Commit.Commit) -> list:
271 args = {"corpus": commit.commit.message}
272 # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
273 r = phab_api_call(
274 phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args
276 review_id = r["result"]["revisionIDFieldInfo"]["value"]
277 if not review_id:
278 # No Phabricator revision for this commit
279 return []
281 args = {"constraints[ids][0]": review_id, "attachments[reviewers]": True}
282 # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
283 r = phab_api_call(
284 phab_token, "https://reviews.llvm.org/api/differential.revision.search", args
286 reviewers = r["result"]["data"][0]["attachments"]["reviewers"]["reviewers"]
287 accepted = []
288 for reviewer in reviewers:
289 if reviewer["status"] != "accepted":
290 continue
291 phid = reviewer["reviewerPHID"]
292 args = {"constraints[phids][0]": phid}
293 # API documentation: https://reviews.llvm.org/conduit/method/user.search/
294 r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args)
295 accepted.append(r["result"]["data"][0]["fields"]["username"])
296 return accepted
299 def extract_commit_hash(arg: str):
301 Extract the commit hash from the argument passed to /action github
302 comment actions. We currently only support passing the commit hash
303 directly or use the github URL, such as
304 https://github.com/llvm/llvm-project/commit/2832d7941f4207f1fcf813b27cf08cecc3086959
306 github_prefix = "https://github.com/llvm/llvm-project/commit/"
307 if arg.startswith(github_prefix):
308 return arg[len(github_prefix) :]
309 return arg
312 class ReleaseWorkflow:
313 CHERRY_PICK_FAILED_LABEL = "release:cherry-pick-failed"
316 This class implements the sub-commands for the release-workflow command.
317 The current sub-commands are:
318 * create-branch
319 * create-pull-request
321 The execute_command method will automatically choose the correct sub-command
322 based on the text in stdin.
325 def __init__(
326 self,
327 token: str,
328 repo: str,
329 issue_number: int,
330 branch_repo_name: str,
331 branch_repo_token: str,
332 llvm_project_dir: str,
333 phab_token: str,
334 ) -> None:
335 self._token = token
336 self._repo_name = repo
337 self._issue_number = issue_number
338 self._branch_repo_name = branch_repo_name
339 if branch_repo_token:
340 self._branch_repo_token = branch_repo_token
341 else:
342 self._branch_repo_token = self.token
343 self._llvm_project_dir = llvm_project_dir
344 self._phab_token = phab_token
346 @property
347 def token(self) -> str:
348 return self._token
350 @property
351 def repo_name(self) -> str:
352 return self._repo_name
354 @property
355 def issue_number(self) -> int:
356 return self._issue_number
358 @property
359 def branch_repo_name(self) -> str:
360 return self._branch_repo_name
362 @property
363 def branch_repo_token(self) -> str:
364 return self._branch_repo_token
366 @property
367 def llvm_project_dir(self) -> str:
368 return self._llvm_project_dir
370 @property
371 def phab_token(self) -> str:
372 return self._phab_token
374 @property
375 def repo(self) -> github.Repository.Repository:
376 return github.Github(self.token).get_repo(self.repo_name)
378 @property
379 def issue(self) -> github.Issue.Issue:
380 return self.repo.get_issue(self.issue_number)
382 @property
383 def push_url(self) -> str:
384 return "https://{}@github.com/{}".format(
385 self.branch_repo_token, self.branch_repo_name
388 @property
389 def branch_name(self) -> str:
390 return "issue{}".format(self.issue_number)
392 @property
393 def release_branch_for_issue(self) -> Optional[str]:
394 issue = self.issue
395 milestone = issue.milestone
396 if milestone is None:
397 return None
398 m = re.search("branch: (.+)", milestone.description)
399 if m:
400 return m.group(1)
401 return None
403 def print_release_branch(self) -> None:
404 print(self.release_branch_for_issue)
406 def issue_notify_branch(self) -> None:
407 self.issue.create_comment(
408 "/branch {}/{}".format(self.branch_repo_name, self.branch_name)
411 def issue_notify_pull_request(self, pull: github.PullRequest.PullRequest) -> None:
412 self.issue.create_comment(
413 "/pull-request {}#{}".format(self.branch_repo_name, pull.number)
416 def make_ignore_comment(self, comment: str) -> str:
418 Returns the comment string with a prefix that will cause
419 a Github workflow to skip parsing this comment.
421 :param str comment: The comment to ignore
423 return "<!--IGNORE-->\n" + comment
425 def issue_notify_no_milestone(self, comment: List[str]) -> None:
426 message = "{}\n\nError: Command failed due to missing milestone.".format(
427 "".join([">" + line for line in comment])
429 self.issue.create_comment(self.make_ignore_comment(message))
431 @property
432 def action_url(self) -> str:
433 if os.getenv("CI"):
434 return "https://github.com/{}/actions/runs/{}".format(
435 os.getenv("GITHUB_REPOSITORY"), os.getenv("GITHUB_RUN_ID")
437 return ""
439 def issue_notify_cherry_pick_failure(
440 self, commit: str
441 ) -> github.IssueComment.IssueComment:
442 message = self.make_ignore_comment(
443 "Failed to cherry-pick: {}\n\n".format(commit)
445 action_url = self.action_url
446 if action_url:
447 message += action_url + "\n\n"
448 message += "Please manually backport the fix and push it to your github fork. Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
449 issue = self.issue
450 comment = issue.create_comment(message)
451 issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL)
452 return comment
454 def issue_notify_pull_request_failure(
455 self, branch: str
456 ) -> github.IssueComment.IssueComment:
457 message = "Failed to create pull request for {} ".format(branch)
458 message += self.action_url
459 return self.issue.create_comment(message)
461 def issue_remove_cherry_pick_failed_label(self):
462 if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]:
463 self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL)
465 def pr_request_review(self, pr: github.PullRequest.PullRequest):
467 This function will try to find the best reviewers for `commits` and
468 then add a comment requesting review of the backport and assign the
469 pull request to the selected reviewers.
471 The reviewers selected are those users who approved the patch in
472 Phabricator.
474 reviewers = []
475 for commit in pr.get_commits():
476 approvers = phab_get_commit_approvers(self.phab_token, commit)
477 for a in approvers:
478 login = phab_login_to_github_login(self.phab_token, self.repo, a)
479 if not login:
480 continue
481 reviewers.append(login)
482 if len(reviewers):
483 message = "{} What do you think about merging this PR to the release branch?".format(
484 " ".join(["@" + r for r in reviewers])
486 pr.create_issue_comment(message)
487 pr.add_to_assignees(*reviewers)
489 def create_branch(self, commits: List[str]) -> bool:
491 This function attempts to backport `commits` into the branch associated
492 with `self.issue_number`.
494 If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
495 a comment is added to the issue saying that the cherry-pick failed.
497 :param list commits: List of commits to cherry-pick.
500 print("cherry-picking", commits)
501 branch_name = self.branch_name
502 local_repo = Repo(self.llvm_project_dir)
503 local_repo.git.checkout(self.release_branch_for_issue)
505 for c in commits:
506 try:
507 local_repo.git.cherry_pick("-x", c)
508 except Exception as e:
509 self.issue_notify_cherry_pick_failure(c)
510 raise e
512 push_url = self.push_url
513 print("Pushing to {} {}".format(push_url, branch_name))
514 local_repo.git.push(push_url, "HEAD:{}".format(branch_name), force=True)
516 self.issue_notify_branch()
517 self.issue_remove_cherry_pick_failed_label()
518 return True
520 def check_if_pull_request_exists(
521 self, repo: github.Repository.Repository, head: str
522 ) -> bool:
523 pulls = repo.get_pulls(head=head)
524 return pulls.totalCount != 0
526 def create_pull_request(self, owner: str, repo_name: str, branch: str) -> bool:
528 reate a pull request in `self.branch_repo_name`. The base branch of the
529 pull request will be chosen based on the the milestone attached to
530 the issue represented by `self.issue_number` For example if the milestone
531 is Release 13.0.1, then the base branch will be release/13.x. `branch`
532 will be used as the compare branch.
533 https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
534 https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
536 repo = github.Github(self.token).get_repo(self.branch_repo_name)
537 issue_ref = "{}#{}".format(self.repo_name, self.issue_number)
538 pull = None
539 release_branch_for_issue = self.release_branch_for_issue
540 if release_branch_for_issue is None:
541 return False
542 head_branch = branch
543 if not repo.fork:
544 # If the target repo is not a fork of llvm-project, we need to copy
545 # the branch into the target repo. GitHub only supports cross-repo pull
546 # requests on forked repos.
547 head_branch = f"{owner}-{branch}"
548 local_repo = Repo(self.llvm_project_dir)
549 push_done = False
550 for _ in range(0, 5):
551 try:
552 local_repo.git.fetch(
553 f"https://github.com/{owner}/{repo_name}", f"{branch}:{branch}"
555 local_repo.git.push(
556 self.push_url, f"{branch}:{head_branch}", force=True
558 push_done = True
559 break
560 except Exception as e:
561 print(e)
562 time.sleep(30)
563 continue
564 if not push_done:
565 raise Exception("Failed to mirror branch into {}".format(self.push_url))
566 owner = repo.owner.login
568 head = f"{owner}:{head_branch}"
569 if self.check_if_pull_request_exists(repo, head):
570 print("PR already exists...")
571 return True
572 try:
573 pull = repo.create_pull(
574 title=f"PR for {issue_ref}",
575 body="resolves {}".format(issue_ref),
576 base=release_branch_for_issue,
577 head=head,
578 maintainer_can_modify=False,
581 try:
582 if self.phab_token:
583 self.pr_request_review(pull)
584 except Exception as e:
585 print("error: Failed while searching for reviewers", e)
587 except Exception as e:
588 self.issue_notify_pull_request_failure(branch)
589 raise e
591 if pull is None:
592 return False
594 self.issue_notify_pull_request(pull)
595 self.issue_remove_cherry_pick_failed_label()
597 # TODO(tstellar): Do you really want to always return True?
598 return True
600 def execute_command(self) -> bool:
602 This function reads lines from STDIN and executes the first command
603 that it finds. The 2 supported commands are:
604 /cherry-pick commit0 <commit1> <commit2> <...>
605 /branch <owner>/<repo>/<branch>
607 for line in sys.stdin:
608 line.rstrip()
609 m = re.search(r"/([a-z-]+)\s(.+)", line)
610 if not m:
611 continue
612 command = m.group(1)
613 args = m.group(2)
615 if command == "cherry-pick":
616 arg_list = args.split()
617 commits = list(map(lambda a: extract_commit_hash(a), arg_list))
618 return self.create_branch(commits)
620 if command == "branch":
621 m = re.match("([^/]+)/([^/]+)/(.+)", args)
622 if m:
623 owner = m.group(1)
624 repo = m.group(2)
625 branch = m.group(3)
626 return self.create_pull_request(owner, repo, branch)
628 print("Do not understand input:")
629 print(sys.stdin.readlines())
630 return False
633 parser = argparse.ArgumentParser()
634 parser.add_argument(
635 "--token", type=str, required=True, help="GitHub authentiation token"
637 parser.add_argument(
638 "--repo",
639 type=str,
640 default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"),
641 help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)",
643 subparsers = parser.add_subparsers(dest="command")
645 issue_subscriber_parser = subparsers.add_parser("issue-subscriber")
646 issue_subscriber_parser.add_argument("--label-name", type=str, required=True)
647 issue_subscriber_parser.add_argument("--issue-number", type=int, required=True)
649 pr_subscriber_parser = subparsers.add_parser("pr-subscriber")
650 pr_subscriber_parser.add_argument("--label-name", type=str, required=True)
651 pr_subscriber_parser.add_argument("--issue-number", type=int, required=True)
653 release_workflow_parser = subparsers.add_parser("release-workflow")
654 release_workflow_parser.add_argument(
655 "--llvm-project-dir",
656 type=str,
657 default=".",
658 help="directory containing the llvm-project checout",
660 release_workflow_parser.add_argument(
661 "--issue-number", type=int, required=True, help="The issue number to update"
663 release_workflow_parser.add_argument(
664 "--phab-token",
665 type=str,
666 help="Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/",
668 release_workflow_parser.add_argument(
669 "--branch-repo-token",
670 type=str,
671 help="GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.",
673 release_workflow_parser.add_argument(
674 "--branch-repo",
675 type=str,
676 default="llvm/llvm-project-release-prs",
677 help="The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)",
679 release_workflow_parser.add_argument(
680 "sub_command",
681 type=str,
682 choices=["print-release-branch", "auto"],
683 help="Print to stdout the name of the release branch ISSUE_NUMBER should be backported to",
686 llvmbot_git_config_parser = subparsers.add_parser(
687 "setup-llvmbot-git",
688 help="Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot",
691 args = parser.parse_args()
693 if args.command == "issue-subscriber":
694 issue_subscriber = IssueSubscriber(
695 args.token, args.repo, args.issue_number, args.label_name
697 issue_subscriber.run()
698 elif args.command == "pr-subscriber":
699 pr_subscriber = PRSubscriber(
700 args.token, args.repo, args.issue_number, args.label_name
702 pr_subscriber.run()
703 elif args.command == "release-workflow":
704 release_workflow = ReleaseWorkflow(
705 args.token,
706 args.repo,
707 args.issue_number,
708 args.branch_repo,
709 args.branch_repo_token,
710 args.llvm_project_dir,
711 args.phab_token,
713 if not release_workflow.release_branch_for_issue:
714 release_workflow.issue_notify_no_milestone(sys.stdin.readlines())
715 sys.exit(1)
716 if args.sub_command == "print-release-branch":
717 release_workflow.print_release_branch()
718 else:
719 if not release_workflow.execute_command():
720 sys.exit(1)
721 elif args.command == "setup-llvmbot-git":
722 setup_llvmbot_git()