1#!/usr/bin/env python3
2#
3# ======- github-automation - LLVM GitHub Automation Routines--*- python -*--==#
4#
5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6# See https://llvm.org/LICENSE.txt for license information.
7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8#
9# ==-------------------------------------------------------------------------==#
10
11import argparse
12from git import Repo # type: ignore
13import github
14import os
15import re
16import requests
17import sys
18import time
19from typing import *
20
21class IssueSubscriber:
22
23    @property
24    def team_name(self) -> str:
25        return self._team_name
26
27    def __init__(self, token:str, repo:str, issue_number:int, label_name:str):
28        self.repo = github.Github(token).get_repo(repo)
29        self.org = github.Github(token).get_organization(self.repo.organization.login)
30        self.issue = self.repo.get_issue(issue_number)
31        self._team_name = 'issue-subscribers-{}'.format(label_name).lower()
32
33    def run(self) -> bool:
34        for team in self.org.get_teams():
35            if self.team_name != team.name.lower():
36                continue
37            comment = '@llvm/{}'.format(team.slug)
38            self.issue.create_comment(comment)
39            return True
40        return False
41
42def setup_llvmbot_git(git_dir = '.'):
43    """
44    Configure the git repo in `git_dir` with the llvmbot account so
45    commits are attributed to llvmbot.
46    """
47    repo = Repo(git_dir)
48    with repo.config_writer() as config:
49        config.set_value('user', 'name', 'llvmbot')
50        config.set_value('user', 'email', '[email protected]')
51
52def phab_api_call(phab_token:str, url:str, args:dict) -> dict:
53    """
54    Make an API call to the Phabricator web service and return a dictionary
55    containing the json response.
56    """
57    data = { "api.token" : phab_token }
58    data.update(args)
59    response = requests.post(url, data = data)
60    return response.json()
61
62
63def phab_login_to_github_login(phab_token:str, repo:github.Repository.Repository, phab_login:str) -> str:
64    """
65    Tries to translate a Phabricator login to a github login by
66    finding a commit made in Phabricator's Differential.
67    The commit's SHA1 is then looked up in the github repo and
68    the committer's login associated with that commit is returned.
69
70    :param str phab_token: The Conduit API token to use for communication with Pabricator
71    :param github.Repository.Repository repo: The github repo to use when looking for the SHA1 found in Differential
72    :param str phab_login: The Phabricator login to be translated.
73    """
74
75    args = {
76        "constraints[authors][0]" : phab_login,
77        # PHID for "LLVM Github Monorepo" repository
78        "constraints[repositories][0]" : "PHID-REPO-f4scjekhnkmh7qilxlcy",
79        "limit" : 1
80    }
81    # API documentation: https://reviews.llvm.org/conduit/method/diffusion.commit.search/
82    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/diffusion.commit.search", args)
83    data = r['result']['data']
84    if len(data) == 0:
85        # Can't find any commits associated with this user
86        return None
87
88    commit_sha = data[0]['fields']['identifier']
89    return repo.get_commit(commit_sha).committer.login
90
91def phab_get_commit_approvers(phab_token:str, repo:github.Repository.Repository, commit:github.Commit.Commit) -> list:
92    args = { "corpus" : commit.commit.message }
93    # API documentation: https://reviews.llvm.org/conduit/method/differential.parsecommitmessage/
94    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.parsecommitmessage", args)
95    review_id = r['result']['revisionIDFieldInfo']['value']
96
97    args = {
98        'constraints[ids][0]' : review_id,
99        'attachments[reviewers]' : True
100    }
101    # API documentation: https://reviews.llvm.org/conduit/method/differential.revision.search/
102    r = phab_api_call(phab_token, "https://reviews.llvm.org/api/differential.revision.search", args)
103    reviewers = r['result']['data'][0]['attachments']['reviewers']['reviewers']
104    accepted = []
105    for reviewer in reviewers:
106        if reviewer['status'] != 'accepted':
107            continue
108        phid = reviewer['reviewerPHID']
109        args = { 'constraints[phids][0]' : phid }
110        # API documentation: https://reviews.llvm.org/conduit/method/user.search/
111        r = phab_api_call(phab_token, "https://reviews.llvm.org/api/user.search", args)
112        accepted.append(r['result']['data'][0]['fields']['username'])
113    return accepted
114
115class ReleaseWorkflow:
116
117    CHERRY_PICK_FAILED_LABEL = 'release:cherry-pick-failed'
118
119    """
120    This class implements the sub-commands for the release-workflow command.
121    The current sub-commands are:
122        * create-branch
123        * create-pull-request
124
125    The execute_command method will automatically choose the correct sub-command
126    based on the text in stdin.
127    """
128
129    def __init__(self, token:str, repo:str, issue_number:int,
130                       branch_repo_name:str, branch_repo_token:str,
131                       llvm_project_dir:str, phab_token:str) -> None:
132        self._token = token
133        self._repo_name = repo
134        self._issue_number = issue_number
135        self._branch_repo_name = branch_repo_name
136        if branch_repo_token:
137            self._branch_repo_token = branch_repo_token
138        else:
139            self._branch_repo_token = self.token
140        self._llvm_project_dir = llvm_project_dir
141        self._phab_token = phab_token
142
143    @property
144    def token(self) -> str:
145        return self._token
146
147    @property
148    def repo_name(self) -> str:
149        return self._repo_name
150
151    @property
152    def issue_number(self) -> int:
153        return self._issue_number
154
155    @property
156    def branch_repo_name(self) -> str:
157        return self._branch_repo_name
158
159    @property
160    def branch_repo_token(self) -> str:
161        return self._branch_repo_token
162
163    @property
164    def llvm_project_dir(self) -> str:
165        return self._llvm_project_dir
166
167    @property
168    def phab_token(self) -> str:
169        return self._phab_token
170
171    @property
172    def repo(self) -> github.Repository.Repository:
173        return github.Github(self.token).get_repo(self.repo_name)
174
175    @property
176    def issue(self) -> github.Issue.Issue:
177        return self.repo.get_issue(self.issue_number)
178
179    @property
180    def push_url(self) -> str:
181        return 'https://{}@github.com/{}'.format(self.branch_repo_token, self.branch_repo_name)
182
183    @property
184    def branch_name(self) -> str:
185        return 'issue{}'.format(self.issue_number)
186
187    @property
188    def release_branch_for_issue(self) -> Optional[str]:
189        issue = self.issue
190        milestone = issue.milestone
191        if milestone is None:
192            return None
193        m = re.search('branch: (.+)',milestone.description)
194        if m:
195            return m.group(1)
196        return None
197
198    def print_release_branch(self) -> None:
199        print(self.release_branch_for_issue)
200
201    def issue_notify_branch(self) -> None:
202        self.issue.create_comment('/branch {}/{}'.format(self.branch_repo_name, self.branch_name))
203
204    def issue_notify_pull_request(self, pull:github.PullRequest.PullRequest) -> None:
205        self.issue.create_comment('/pull-request {}#{}'.format(self.branch_repo_name, pull.number))
206
207    def make_ignore_comment(self, comment: str) -> str:
208        """
209        Returns the comment string with a prefix that will cause
210        a Github workflow to skip parsing this comment.
211
212        :param str comment: The comment to ignore
213        """
214        return "<!--IGNORE-->\n"+comment
215
216    def issue_notify_no_milestone(self, comment:List[str]) -> None:
217        message = "{}\n\nError: Command failed due to missing milestone.".format(''.join(['>' + line for line in comment]))
218        self.issue.create_comment(self.make_ignore_comment(message))
219
220    @property
221    def action_url(self) -> str:
222        if os.getenv('CI'):
223            return 'https://github.com/{}/actions/runs/{}'.format(os.getenv('GITHUB_REPOSITORY'), os.getenv('GITHUB_RUN_ID'))
224        return ""
225
226    def issue_notify_cherry_pick_failure(self, commit:str) -> github.IssueComment.IssueComment:
227        message = self.make_ignore_comment("Failed to cherry-pick: {}\n\n".format(commit))
228        action_url = self.action_url
229        if action_url:
230            message += action_url + "\n\n"
231        message += "Please manually backport the fix and push it to your github fork.  Once this is done, please add a comment like this:\n\n`/branch <user>/<repo>/<branch>`"
232        issue = self.issue
233        comment = issue.create_comment(message)
234        issue.add_to_labels(self.CHERRY_PICK_FAILED_LABEL)
235        return comment
236
237    def issue_notify_pull_request_failure(self, branch:str) -> github.IssueComment.IssueComment:
238        message = "Failed to create pull request for {} ".format(branch)
239        message += self.action_url
240        return self.issue.create_comment(message)
241
242    def issue_remove_cherry_pick_failed_label(self):
243        if self.CHERRY_PICK_FAILED_LABEL in [l.name for l in self.issue.labels]:
244            self.issue.remove_from_labels(self.CHERRY_PICK_FAILED_LABEL)
245
246    def pr_request_review(self, pr:github.PullRequest.PullRequest):
247        """
248        This function will try to find the best reviewers for `commits` and
249        then add a comment requesting review of the backport and assign the
250        pull request to the selected reviewers.
251
252        The reviewers selected are those users who approved the patch in
253        Phabricator.
254        """
255        reviewers = []
256        for commit in pr.get_commits():
257            approvers = phab_get_commit_approvers(self.phab_token, self.repo, commit)
258            for a in approvers:
259                login = phab_login_to_github_login(self.phab_token, self.repo, a)
260                if not login:
261                    continue
262                reviewers.append(login)
263        if len(reviewers):
264            message = "{} What do you think about merging this PR to the release branch?".format(
265                    " ".join(["@" + r for r in reviewers]))
266            pr.create_issue_comment(message)
267            pr.add_to_assignees(*reviewers)
268
269    def create_branch(self, commits:List[str]) -> bool:
270        """
271        This function attempts to backport `commits` into the branch associated
272        with `self.issue_number`.
273
274        If this is successful, then the branch is pushed to `self.branch_repo_name`, if not,
275        a comment is added to the issue saying that the cherry-pick failed.
276
277        :param list commits: List of commits to cherry-pick.
278
279        """
280        print('cherry-picking', commits)
281        branch_name = self.branch_name
282        local_repo = Repo(self.llvm_project_dir)
283        local_repo.git.checkout(self.release_branch_for_issue)
284
285        for c in commits:
286            try:
287                local_repo.git.cherry_pick('-x', c)
288            except Exception as e:
289                self.issue_notify_cherry_pick_failure(c)
290                raise e
291
292        push_url = self.push_url
293        print('Pushing to {} {}'.format(push_url, branch_name))
294        local_repo.git.push(push_url, 'HEAD:{}'.format(branch_name), force=True)
295
296        self.issue_notify_branch()
297        self.issue_remove_cherry_pick_failed_label()
298        return True
299
300    def check_if_pull_request_exists(self, repo:github.Repository.Repository, head:str) -> bool:
301        pulls = repo.get_pulls(head=head)
302        return pulls.totalCount != 0
303
304    def create_pull_request(self, owner:str, branch:str) -> bool:
305        """
306        reate a pull request in `self.branch_repo_name`.  The base branch of the
307        pull request will be choosen based on the the milestone attached to
308        the issue represented by `self.issue_number`  For example if the milestone
309        is Release 13.0.1, then the base branch will be release/13.x. `branch`
310        will be used as the compare branch.
311        https://docs.github.com/en/get-started/quickstart/github-glossary#base-branch
312        https://docs.github.com/en/get-started/quickstart/github-glossary#compare-branch
313        """
314        repo = github.Github(self.token).get_repo(self.branch_repo_name)
315        issue_ref = '{}#{}'.format(self.repo_name, self.issue_number)
316        pull = None
317        release_branch_for_issue = self.release_branch_for_issue
318        if release_branch_for_issue is None:
319            return False
320        head_branch = branch
321        if not repo.fork:
322            # If the target repo is not a fork of llvm-project, we need to copy
323            # the branch into the target repo.  GitHub only supports cross-repo pull
324            # requests on forked repos.
325            head_branch = f'{owner}-{branch}'
326            local_repo = Repo(self.llvm_project_dir)
327            push_done = False
328            for i in range(0,5):
329                try:
330                    local_repo.git.fetch(f'https://github.com/{owner}/llvm-project', f'{branch}:{branch}')
331                    local_repo.git.push(self.push_url, f'{branch}:{head_branch}', force=True)
332                    push_done = True
333                    break
334                except Exception as e:
335                    print(e)
336                    time.sleep(30)
337                    continue
338            if not push_done:
339                raise Exception("Failed to mirror branch into {}".format(self.push_url))
340            owner = repo.owner.login
341
342        head = f"{owner}:{head_branch}"
343        if self.check_if_pull_request_exists(repo, head):
344            print("PR already exists...")
345            return True
346        try:
347            pull = repo.create_pull(title=f"PR for {issue_ref}",
348                                    body='resolves {}'.format(issue_ref),
349                                    base=release_branch_for_issue,
350                                    head=head,
351                                    maintainer_can_modify=False)
352
353            try:
354                if self.phab_token:
355                    self.pr_request_review(pull)
356            except Exception as e:
357                print("error: Failed while searching for reviewers", e)
358
359        except Exception as e:
360            self.issue_notify_pull_request_failure(branch)
361            raise e
362
363        if pull is None:
364            return False
365
366        self.issue_notify_pull_request(pull)
367        self.issue_remove_cherry_pick_failed_label()
368
369        # TODO(tstellar): Do you really want to always return True?
370        return True
371
372
373    def execute_command(self) -> bool:
374        """
375        This function reads lines from STDIN and executes the first command
376        that it finds.  The 2 supported commands are:
377        /cherry-pick commit0 <commit1> <commit2> <...>
378        /branch <owner>/<repo>/<branch>
379        """
380        for line in sys.stdin:
381            line.rstrip()
382            m = re.search("/([a-z-]+)\s(.+)", line)
383            if not m:
384                continue
385            command = m.group(1)
386            args = m.group(2)
387
388            if command == 'cherry-pick':
389                return self.create_branch(args.split())
390
391            if command == 'branch':
392                m = re.match('([^/]+)/([^/]+)/(.+)', args)
393                if m:
394                    owner = m.group(1)
395                    branch = m.group(3)
396                    return self.create_pull_request(owner, branch)
397
398        print("Do not understand input:")
399        print(sys.stdin.readlines())
400        return False
401
402parser = argparse.ArgumentParser()
403parser.add_argument('--token', type=str, required=True, help='GitHub authentiation token')
404parser.add_argument('--repo', type=str, default=os.getenv('GITHUB_REPOSITORY', 'llvm/llvm-project'),
405                    help='The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)')
406subparsers = parser.add_subparsers(dest='command')
407
408issue_subscriber_parser = subparsers.add_parser('issue-subscriber')
409issue_subscriber_parser.add_argument('--label-name', type=str, required=True)
410issue_subscriber_parser.add_argument('--issue-number', type=int, required=True)
411
412release_workflow_parser = subparsers.add_parser('release-workflow')
413release_workflow_parser.add_argument('--llvm-project-dir', type=str, default='.', help='directory containing the llvm-project checout')
414release_workflow_parser.add_argument('--issue-number', type=int, required=True, help='The issue number to update')
415release_workflow_parser.add_argument('--phab-token', type=str, help='Phabricator conduit API token. See https://reviews.llvm.org/settings/user/<USER>/page/apitokens/')
416release_workflow_parser.add_argument('--branch-repo-token', type=str,
417                                     help='GitHub authentication token to use for the repository where new branches will be pushed. Defaults to TOKEN.')
418release_workflow_parser.add_argument('--branch-repo', type=str, default='llvm/llvm-project-release-prs',
419                                     help='The name of the repo where new branches will be pushed (e.g. llvm/llvm-project)')
420release_workflow_parser.add_argument('sub_command', type=str, choices=['print-release-branch', 'auto'],
421                                     help='Print to stdout the name of the release branch ISSUE_NUMBER should be backported to')
422
423llvmbot_git_config_parser = subparsers.add_parser('setup-llvmbot-git', help='Set the default user and email for the git repo in LLVM_PROJECT_DIR to llvmbot')
424
425args = parser.parse_args()
426
427if args.command == 'issue-subscriber':
428    issue_subscriber = IssueSubscriber(args.token, args.repo, args.issue_number, args.label_name)
429    issue_subscriber.run()
430elif args.command == 'release-workflow':
431    release_workflow = ReleaseWorkflow(args.token, args.repo, args.issue_number,
432                                       args.branch_repo, args.branch_repo_token,
433                                       args.llvm_project_dir, args.phab_token)
434    if not release_workflow.release_branch_for_issue:
435        release_workflow.issue_notify_no_milestone(sys.stdin.readlines())
436        sys.exit(1)
437    if args.sub_command == 'print-release-branch':
438        release_workflow.print_release_branch()
439    else:
440        if not release_workflow.execute_command():
441            sys.exit(1)
442elif args.command == 'setup-llvmbot-git':
443    setup_llvmbot_git()
444