From 22d7adbd04dfb684e9091a711050acb4ff43a486 Mon Sep 17 00:00:00 2001 From: Brian Clifton Date: Mon, 4 Feb 2019 16:09:14 -0700 Subject: [PATCH 1/4] Create new `npm run pr` task Only works for brave-core (Desktop). New script will create branches / PRs as needed to uplift to a given channel. Also allows assignment of reviewers / assignees (including login validation) Fixes https://github.com/brave/devops/issues/672 --- script/lib/github.py | 142 +++++++++++++++++++ script/lib/helpers.py | 14 +- script/pr.py | 323 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 477 insertions(+), 2 deletions(-) create mode 100755 script/pr.py diff --git a/script/lib/github.py b/script/lib/github.py index aedcd0808f61..e048f54cc2cb 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -5,6 +5,8 @@ import re import requests import sys +import base64 +from util import execute, scoped_cwd REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'requests')) @@ -80,3 +82,143 @@ def __getattr__(self, attr): name = '%s/%s' % (self._name, attr) return _Callable(self._gh, name) + + +def get_authenticated_user_login(token): + """given a valid GitHub access token, return the associated GitHub user login""" + # for more info see: https://developer.github.com/v3/users/#get-the-authenticated-user + user = GitHub(token).user() + try: + response = user.get() + return response['login'] + except Exception as e: + print('[ERROR] ' + str(e)) + + +def parse_user_logins(token, login_csv, verbose=False): + """given a list of logins in csv format, parse into a list and validate logins""" + if login_csv is None: + return [] + login_csv = login_csv.replace(" ", "") + parsed_logins = login_csv.split(',') + + users = GitHub(token).users() + + invalid_logins = [] + + # check login/username against GitHub + # for more info see: https://developer.github.com/v3/users/#get-a-single-user + for login in parsed_logins: + try: + user = users(login).get() + except Exception as e: + if verbose: + print('Login "' + login + '" does not appear to be valid. ' + str(e)) + invalid_logins.append(login) + + if len(invalid_logins) > 0: + raise Exception('Invalid logins found. Are they misspelled? ' + ','.join(invalid_logins)) + + return parsed_logins + + +def get_file_contents(token, repo_name, filename, branch=None): + # NOTE: API only supports files up to 1MB in size + # for more info see: https://developer.github.com/v3/repos/contents/ + repo = GitHub(token).repos(repo_name) + get_data = {} + if branch: + get_data['ref'] = branch + file = repo.contents(filename).get(params=get_data) + if file['encoding'] == 'base64': + return base64.b64decode(file['content']) + return file['content'] + + +def add_reviewers_to_pull_request(token, repo_name, pr_number, reviewers=[], verbose=False, dryrun=False): + # add reviewers to pull request + # for more info see: https://developer.github.com/v3/pulls/review_requests/ + repo = GitHub(token).repos(repo_name) + patch_data = {} + if len(reviewers) > 0: + patch_data['reviewers'] = reviewers + if dryrun: + print('[INFO] would call `repo.pulls(' + str(pr_number) + + ').requested_reviewers.post(' + str(patch_data) + ')`') + return + response = repo.pulls(pr_number).requested_reviewers.post(data=patch_data) + if verbose: + print('repo.pulls(' + str(pr_number) + ').requested_reviewers.post(data) response:\n' + str(response)) + return response + + +def get_milestones(token, repo_name, verbose=False): + # get all milestones for a repo + # for more info see: https://developer.github.com/v3/issues/milestones/ + repo = GitHub(token).repos(repo_name) + response = repo.milestones.get() + if verbose: + print('repo.milestones.get() response:\n' + str(response)) + return response + + +def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, verbose=False, dryrun=False): + post_data = { + 'title': title, + 'head': branch_src, + 'base': branch_dst, + 'body': body, + 'maintainer_can_modify': True + } + # create the pull request + # for more info see: http://developer.github.com/v3/pulls + if dryrun: + print('[INFO] would call `repo.pulls.post(' + str(post_data) + ')`') + return 1234 + repo = GitHub(token).repos(repo_name) + response = repo.pulls.post(data=post_data) + if verbose: + print('repo.pulls.post(data) response:\n' + str(response)) + return int(response['number']) + + +def set_issue_details(token, repo_name, issue_number, milestone_number=None, assignees=[], verbose=False, dryrun=False): + patch_data = {} + if milestone_number: + patch_data['milestone'] = milestone_number + if len(assignees) > 0: + patch_data['assignees'] = assignees + # TODO: error if no keys in patch_data + + # add milestone and assignee to issue / pull request + # for more info see: https://developer.github.com/v3/issues/#edit-an-issue + if dryrun: + print('[INFO] would call `repo.issues(' + str(issue_number) + ').patch(' + str(patch_data) + ')`') + return + repo = GitHub(token).repos(repo_name) + response = repo.issues(issue_number).patch(data=patch_data) + if verbose: + print('repo.issues(' + str(issue_number) + ').patch(data) response:\n' + str(response)) + + +def fetch_origin_check_staged(path): + """given a path on disk (to a git repo), fetch origin and ensure there aren't unstaged files""" + with scoped_cwd(path): + execute(['git', 'fetch', 'origin']) + status = execute(['git', 'status', '-s']).strip() + if len(status) > 0: + print('[ERROR] There appear to be unstaged changes.\n' + + 'Please resolve these before running (ex: `git status`).') + return 1 + return 0 + + +def push_branches_to_remote(path, branches_to_push, dryrun=False): + if dryrun: + print('[INFO] would push the following local branches to remote: ' + str(branches_to_push)) + else: + with scoped_cwd(path): + for branch_to_push in branches_to_push: + print('- pushing ' + branch_to_push + '...') + # TODO: if they already exist, force push?? or error?? + execute(['git', 'push', '-u', 'origin', branch_to_push]) diff --git a/script/lib/helpers.py b/script/lib/helpers.py index fb9784823ef8..de6d2be8b4f7 100644 --- a/script/lib/helpers.py +++ b/script/lib/helpers.py @@ -8,11 +8,21 @@ from .config import get_raw_version BRAVE_REPO = "brave/brave-browser" +BRAVE_CORE_REPO = "brave/brave-core" + + +def channels(): + return ['nightly', 'dev', 'beta', 'release'] def get_channel_display_name(): - d = {'beta': 'Beta', 'canary': 'Canary', 'dev': 'Developer', - 'release': 'Release'} + raw = channels() + d = { + 'canary': 'Canary', + raw[1]: 'Developer', + raw[2]: 'Beta', + raw[3]: 'Release' + } return d[release_channel()] diff --git a/script/pr.py b/script/pr.py new file mode 100755 index 000000000000..4d9208401edf --- /dev/null +++ b/script/pr.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import errno +import hashlib +import os +import requests +import re +import shutil +import subprocess +import sys +import tempfile +import json + +from io import StringIO +from lib.config import get_env_var, SOURCE_ROOT, BRAVE_CORE_ROOT, get_raw_version +from lib.util import execute, scoped_cwd +from lib.helpers import * +from lib.github import (GitHub, get_authenticated_user_login, parse_user_logins, + get_file_contents, add_reviewers_to_pull_request, + get_milestones, create_pull_request, set_issue_details, + fetch_origin_check_staged, push_branches_to_remote) + + +# TODOs for this version +##### +# - validate channel name (passed with --uplift-to) +# - parse out issue (so it can be included in body). ex: git log --pretty=format:%b +# - add arg for overriding the title +# - open PRs in default browser (shell exec) BY DEFAULT. Offer a quiet that DOESN'T open +# - discover associated issues +# - put them in the uplift / original PR body +# - set milestone! (on the ISSUE) +# - --labels=QA/Yes (and then put those on the issues). And check existance of label + + +class PrConfig: + channel_names = channels() + channels_to_process = channels() + is_verbose = False + is_dryrun = False + branches_to_push = [] + master_pr_number = -1 + github_token = None + parsed_reviewers = [] + parsed_owners = [] + milestones = None + + def initialize(self, args): + try: + self.is_verbose = args.verbose + self.is_dryrun = args.dry_run + # TODO: read github token FIRST from CLI, then from .npmrc + self.github_token = get_env_var('GITHUB_TOKEN') + if len(self.github_token) == 0: + # TODO: check .npmrc + # if not there, error out! + print('[ERROR] no valid GitHub token was found') + return 1 + self.parsed_reviewers = parse_user_logins(self.github_token, args.reviewers, verbose=self.is_verbose) + # if `--owners` is not provided, fall back to user owning token + self.parsed_owners = parse_user_logins(self.github_token, args.owners, verbose=self.is_verbose) + if len(self.parsed_owners) == 0: + self.parsed_owners = [get_authenticated_user_login(self.github_token)] + if self.is_verbose: + print('[INFO] config: ' + str(vars(self))) + return 0 + except Exception as e: + print('[ERROR] error returned from GitHub API while initializing config: ' + str(e)) + return 1 + + +config = PrConfig() + + +def is_nightly(channel): + global config + return config.channel_names[0] == channel + + +# given something like "0.60.2", return branch version ("0.60.x") +def get_current_version_branch(version): + version = str(version) + if version[0] == 'v': + version = version[1:] + parts = version.split('.', 3) + parts[2] = 'x' + return '.'.join(parts) + + +# given something like "0.60.x", get previous version ("0.59.x") +def get_previous_version_branch(version): + version = str(version) + if version[0] == 'v': + version = version[1:] + parts = version.split('.', 3) + parts[1] = str(int(parts[1]) - 1) + parts[2] = 'x' + return '.'.join(parts) + + +def get_remote_channel_branches(raw_nightly_version): + global config + nightly_version = get_current_version_branch(raw_nightly_version) + dev_version = get_previous_version_branch(nightly_version) + beta_version = get_previous_version_branch(dev_version) + release_version = get_previous_version_branch(beta_version) + return { + config.channel_names[0]: nightly_version, + config.channel_names[1]: dev_version, + config.channel_names[2]: beta_version, + config.channel_names[3]: release_version + } + + +def parse_args(): + parser = argparse.ArgumentParser(description='create PRs for all branches given branch against master') + parser.add_argument('--reviewers', + help='comma separated list of GitHub logins to mark as reviewers', + default=None) + parser.add_argument('--owners', + help='comma seperated list of GitHub logins to mark as assignee', + default=None) + parser.add_argument('--uplift-to', + help='starting at nightly (master), how far back to uplift the changes', + default='nightly') + parser.add_argument('--start-from', + help='instead of starting from nightly (default), start from beta/dev/release', + default='nightly') + parser.add_argument('-v', '--verbose', action='store_true', + help='prints the output of the GitHub API calls') + parser.add_argument('-n', '--dry-run', action='store_true', + help='don\'t actually create pull requests; just show a call would be made') + + return parser.parse_args() + + +def get_versions(args): + global config + local_version = get_raw_version() + if not is_nightly(args.start_from): + # TODO: fix me + target_branch = '0.60.x' + else: + target_branch = 'master' + decoded_file = get_file_contents(config.github_token, BRAVE_REPO, 'package.json', target_branch) + json_file = json.loads(decoded_file) + remote_version = json_file['version'] + return local_version, remote_version + + +def main(): + args = parse_args() + if args.verbose: + print('[INFO] args: ' + str(args)) + + global config + result = config.initialize(args) + if result != 0: + return result + + result = fetch_origin_check_staged(BRAVE_CORE_ROOT) + if result != 0: + return result + + # get local version + latest version on remote (master) + # if they don't match, rebase is needed + # TODO: allow for diff NOT against master; in that case, use the start-from branch! + local_version, remote_version = get_versions(args) + if local_version != remote_version: + print('[ERROR] Your branch is out of sync (local=' + local_version + + ', remote=' + remote_version + '); please rebase against master') + return 1 + + # get the branch name and the first commit subject (used as the title of the pull request) + # TODO: allow for diff NOT against master; in that case, use the start-from branch! + with scoped_cwd(BRAVE_CORE_ROOT): + local_branch = execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + title_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%s', '--reverse']) + title_list = title_list.split('\n') + if len(title_list) == 0: + print('[ERROR] No commits found!') + return 1 + title = title_list[0] + + # if starting point is NOT nightly, remove options which aren't desired + if not is_nightly(args.start_from): + try: + start_index = config.channel_names.index(args.start_from) + config.channels_to_process = config.channel_names[start_index:] + except Exception as e: + print('[ERROR] specified `start-from` value "' + args.start_from + '" not found in channel list') + return 1 + + # Create a branch for each channel + print('\nCreating branches...') + remote_branches = get_remote_channel_branches(local_version) + local_branches = {} + try: + for channel in config.channels_to_process: + branch = create_branch(channel, remote_branches[channel], local_branch) + local_branches[channel] = branch + if channel == args.uplift_to: + break + except Exception as e: + print('[ERROR] cherry-pick failed for branch "' + branch + '". Please resolve manually:\n' + str(e)) + return 1 + + print('\nPushing local branches to remote...') + push_branches_to_remote(BRAVE_CORE_ROOT, config.branches_to_push, dryrun=config.is_dryrun) + + try: + print('\nCreating the pull requests...') + for channel in config.channels_to_process: + submit_pr( + channel, + title, + remote_branches[channel], + local_branches[channel]) + if channel == args.uplift_to: + break + print('\nDone!') + except Exception as e: + print('\n[ERROR] Unhandled error while creating pull request; ' + str(e)) + return 1 + + return 0 + + +def create_branch(channel, remote_branch, local_branch): + global config + + if not is_nightly(channel): + channel_branch = remote_branch + '_' + local_branch + + with scoped_cwd(BRAVE_CORE_ROOT): + # get SHA for all commits (in order) + sha_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%h', '--reverse']) + sha_list = sha_list.split('\n') + try: + # check if branch exists already + try: + branch_sha = execute(['git', 'rev-parse', '-q', '--verify', channel_branch]) + except Exception as e: + branch_sha = '' + + if len(branch_sha) > 0: + # branch exists; reset it + print('(' + channel + ') branch "' + channel_branch + '" exists; resetting to origin/' + remote_branch) + execute(['git', 'reset', '--hard', 'origin/' + remote_branch]) + else: + # create the branch + print('(' + channel + ') creating "' + channel_branch + '" from ' + channel) + execute(['git', 'checkout', remote_branch]) + execute(['git', 'checkout', '-b', channel_branch]) + + # TODO: handle errors thrown by cherry-pick + for sha in sha_list: + output = execute(['git', 'cherry-pick', sha]).split('\n') + print('- picked ' + sha + ' (' + output[0] + ')') + + finally: + # switch back to original branch + execute(['git', 'checkout', local_branch]) + execute(['git', 'reset', '--hard', sha_list[-1]]) + + config.branches_to_push.append(channel_branch) + + return channel_branch + return local_branch + + +def get_milestone_for_branch(channel_branch): + global config + if not config.milestones: + config.milestones = get_milestones(config.github_token, BRAVE_CORE_REPO) + for milestone in config.milestones: + if milestone['title'].startswith(channel_branch + ' - '): + return milestone['number'] + return None + + +def submit_pr(channel, title, remote_branch, local_branch): + global config + + try: + milestone_number = get_milestone_for_branch(remote_branch) + if milestone_number is None: + print('milestone for "' + remote_branch + '"" was not found!') + return 0 + + print('(' + channel + ') creating pull request') + pr_title = title + pr_dst = remote_branch + if is_nightly(channel): + pr_dst = 'master' + pr_body = 'TODO: fill me in\n(created using `npm run pr`)' + else: + pr_title += ' (uplift to ' + remote_branch + ')' + pr_body = 'Uplift of #' + str(config.master_pr_number) + number = create_pull_request(config.github_token, BRAVE_CORE_REPO, pr_title, pr_body, + branch_src=local_branch, branch_dst=pr_dst, + verbose=config.is_verbose, dryrun=config.is_dryrun) + + # store the PR number (master only) so that it can be referenced in uplifts + if is_nightly(channel): + config.master_pr_number = number + + # assign milestone / reviewer(s) / owner(s) + add_reviewers_to_pull_request(config.github_token, BRAVE_CORE_REPO, number, config.parsed_reviewers, + verbose=config.is_verbose, dryrun=config.is_dryrun) + set_issue_details(config.github_token, BRAVE_CORE_REPO, number, milestone_number, config.parsed_owners, + verbose=config.is_verbose, dryrun=config.is_dryrun) + except Exception as e: + print('[ERROR] unhandled error occurred:', str(e)) + + +if __name__ == '__main__': + import sys + sys.exit(main()) From 445cde173dedb87b515012ef51df83f5c2d9d473 Mon Sep 17 00:00:00 2001 From: Brian Clifton Date: Tue, 12 Feb 2019 10:53:55 -0700 Subject: [PATCH 2/4] Fixed bugs when using `starts-from` and added `--labels`, `--title`, and `--force` parameters --- script/lib/github.py | 56 +++++++++++++- script/pr.py | 180 ++++++++++++++++++++++++------------------- 2 files changed, 154 insertions(+), 82 deletions(-) diff --git a/script/lib/github.py b/script/lib/github.py index e048f54cc2cb..b70ff5c46fd6 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -110,10 +110,12 @@ def parse_user_logins(token, login_csv, verbose=False): # for more info see: https://developer.github.com/v3/users/#get-a-single-user for login in parsed_logins: try: - user = users(login).get() + response = users(login).get() + if verbose: + print('[INFO] Login "' + login + '" found: ' + str(response)) except Exception as e: if verbose: - print('Login "' + login + '" does not appear to be valid. ' + str(e)) + print('[INFO] Login "' + login + '" does not appear to be valid. ' + str(e)) invalid_logins.append(login) if len(invalid_logins) > 0: @@ -122,6 +124,34 @@ def parse_user_logins(token, login_csv, verbose=False): return parsed_logins +def parse_labels(token, repo_name, label_csv, verbose=False): + global config + if label_csv is None: + return [] + label_csv = label_csv.replace(" ", "") + parsed_labels = label_csv.split(',') + + invalid_labels = [] + + # validate labels passed in are correct + # for more info see: https://developer.github.com/v3/issues/labels/#get-a-single-label + repo = GitHub(token).repos(repo_name) + for label in parsed_labels: + try: + response = repo.labels(label).get() + if verbose: + print('[INFO] Label "' + label + '" found: ' + str(response)) + except Exception as e: + if verbose: + print('[INFO] Label "' + label + '" does not appear to be valid. ' + str(e)) + invalid_labels.append(label) + + if len(invalid_labels) > 0: + raise Exception('Invalid labels found. Are they misspelled? ' + ','.join(invalid_labels)) + + return parsed_labels + + def get_file_contents(token, repo_name, filename, branch=None): # NOTE: API only supports files up to 1MB in size # for more info see: https://developer.github.com/v3/repos/contents/ @@ -182,12 +212,15 @@ def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, v return int(response['number']) -def set_issue_details(token, repo_name, issue_number, milestone_number=None, assignees=[], verbose=False, dryrun=False): +def set_issue_details(token, repo_name, issue_number, milestone_number=None, + assignees=[], labels=[], verbose=False, dryrun=False): patch_data = {} if milestone_number: patch_data['milestone'] = milestone_number if len(assignees) > 0: patch_data['assignees'] = assignees + if len(labels) > 0: + patch_data['labels'] = labels # TODO: error if no keys in patch_data # add milestone and assignee to issue / pull request @@ -213,6 +246,23 @@ def fetch_origin_check_staged(path): return 0 +def get_local_branch_name(path): + with scoped_cwd(path): + return execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + + +def get_title_from_first_commit(path, branch_to_compare): + """get the first commit subject (useful for the title of a pull request)""" + with scoped_cwd(path): + local_branch = execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + title_list = execute(['git', 'log', 'origin/' + branch_to_compare + + '..HEAD', '--pretty=format:%s', '--reverse']) + title_list = title_list.split('\n') + if len(title_list) == 0: + raise Exception('No commits found! Local branch matches "' + branch_to_compare + '"') + return title_list[0] + + def push_branches_to_remote(path, branches_to_push, dryrun=False): if dryrun: print('[INFO] would push the following local branches to remote: ' + str(branches_to_push)) diff --git a/script/pr.py b/script/pr.py index 4d9208401edf..4855ba945368 100755 --- a/script/pr.py +++ b/script/pr.py @@ -20,21 +20,19 @@ from lib.util import execute, scoped_cwd from lib.helpers import * from lib.github import (GitHub, get_authenticated_user_login, parse_user_logins, - get_file_contents, add_reviewers_to_pull_request, + parse_labels, get_file_contents, add_reviewers_to_pull_request, get_milestones, create_pull_request, set_issue_details, - fetch_origin_check_staged, push_branches_to_remote) + fetch_origin_check_staged, get_local_branch_name, + get_title_from_first_commit, push_branches_to_remote) # TODOs for this version ##### -# - validate channel name (passed with --uplift-to) # - parse out issue (so it can be included in body). ex: git log --pretty=format:%b -# - add arg for overriding the title # - open PRs in default browser (shell exec) BY DEFAULT. Offer a quiet that DOESN'T open # - discover associated issues # - put them in the uplift / original PR body # - set milestone! (on the ISSUE) -# - --labels=QA/Yes (and then put those on the issues). And check existance of label class PrConfig: @@ -48,11 +46,17 @@ class PrConfig: parsed_reviewers = [] parsed_owners = [] milestones = None + title = None + labels = [] def initialize(self, args): try: self.is_verbose = args.verbose self.is_dryrun = args.dry_run + self.title = args.title + # validate channel names + validate_channel(args.uplift_to) + validate_channel(args.start_from) # TODO: read github token FIRST from CLI, then from .npmrc self.github_token = get_env_var('GITHUB_TOKEN') if len(self.github_token) == 0: @@ -65,6 +69,7 @@ def initialize(self, args): self.parsed_owners = parse_user_logins(self.github_token, args.owners, verbose=self.is_verbose) if len(self.parsed_owners) == 0: self.parsed_owners = [get_authenticated_user_login(self.github_token)] + self.labels = parse_labels(self.github_token, BRAVE_CORE_REPO, args.labels, verbose=self.is_verbose) if self.is_verbose: print('[INFO] config: ' + str(vars(self))) return 0 @@ -116,6 +121,14 @@ def get_remote_channel_branches(raw_nightly_version): } +def validate_channel(channel): + global config + try: + config.channel_names.index(channel) + except Exception as e: + raise Exception('Channel name "' + channel + '" is not valid!') + + def parse_args(): parser = argparse.ArgumentParser(description='create PRs for all branches given branch against master') parser.add_argument('--reviewers', @@ -134,22 +147,23 @@ def parse_args(): help='prints the output of the GitHub API calls') parser.add_argument('-n', '--dry-run', action='store_true', help='don\'t actually create pull requests; just show a call would be made') + parser.add_argument('--labels', + help='comma seperated list of labels to apply to each pull request', + default=None) + parser.add_argument('--title', + help='title to use (instead of inferring one from the first commit)', + default=None) + parser.add_argument('-f', '--force', action='store_true', + help='use the script forcefully; ignore warnings') return parser.parse_args() -def get_versions(args): +def get_remote_version(branch_to_compare): global config - local_version = get_raw_version() - if not is_nightly(args.start_from): - # TODO: fix me - target_branch = '0.60.x' - else: - target_branch = 'master' - decoded_file = get_file_contents(config.github_token, BRAVE_REPO, 'package.json', target_branch) + decoded_file = get_file_contents(config.github_token, BRAVE_REPO, 'package.json', branch_to_compare) json_file = json.loads(decoded_file) - remote_version = json_file['version'] - return local_version, remote_version + return json_file['version'] def main(): @@ -166,28 +180,13 @@ def main(): if result != 0: return result - # get local version + latest version on remote (master) - # if they don't match, rebase is needed - # TODO: allow for diff NOT against master; in that case, use the start-from branch! - local_version, remote_version = get_versions(args) - if local_version != remote_version: - print('[ERROR] Your branch is out of sync (local=' + local_version + - ', remote=' + remote_version + '); please rebase against master') - return 1 - - # get the branch name and the first commit subject (used as the title of the pull request) - # TODO: allow for diff NOT against master; in that case, use the start-from branch! - with scoped_cwd(BRAVE_CORE_ROOT): - local_branch = execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - title_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%s', '--reverse']) - title_list = title_list.split('\n') - if len(title_list) == 0: - print('[ERROR] No commits found!') - return 1 - title = title_list[0] - # if starting point is NOT nightly, remove options which aren't desired + # also, find the branch which should be used for diffs (for cherry-picking) + local_version = get_raw_version() + remote_branches = get_remote_channel_branches(local_version) + top_level_branch = 'master' if not is_nightly(args.start_from): + top_level_branch = remote_branches[args.start_from] try: start_index = config.channel_names.index(args.start_from) config.channels_to_process = config.channel_names[start_index:] @@ -195,13 +194,29 @@ def main(): print('[ERROR] specified `start-from` value "' + args.start_from + '" not found in channel list') return 1 + # get local version + latest version on remote (master) + # if they don't match, rebase is needed + remote_version = get_remote_version(top_level_branch) + if local_version != remote_version: + if not args.force: + print('[ERROR] Your branch is out of sync (local=' + local_version + + ', remote=' + remote_version + '); please rebase (ex: "git rebase origin/' + + top_level_branch + '"). NOTE: You can bypass this check by using -f') + return 1 + print('[WARNING] Your branch is out of sync (local=' + local_version + + ', remote=' + remote_version + '); continuing since -f was provided') + + local_branch = get_local_branch_name(BRAVE_CORE_ROOT) + if not config.title: + config.title = get_title_from_first_commit(BRAVE_CORE_ROOT, top_level_branch) + # Create a branch for each channel print('\nCreating branches...') remote_branches = get_remote_channel_branches(local_version) local_branches = {} try: for channel in config.channels_to_process: - branch = create_branch(channel, remote_branches[channel], local_branch) + branch = create_branch(channel, top_level_branch, remote_branches[channel], local_branch) local_branches[channel] = branch if channel == args.uplift_to: break @@ -217,7 +232,7 @@ def main(): for channel in config.channels_to_process: submit_pr( channel, - title, + top_level_branch, remote_branches[channel], local_branches[channel]) if channel == args.uplift_to: @@ -230,47 +245,49 @@ def main(): return 0 -def create_branch(channel, remote_branch, local_branch): +def create_branch(channel, top_level_branch, remote_branch, local_branch): global config - if not is_nightly(channel): - channel_branch = remote_branch + '_' + local_branch + if is_nightly(channel): + return local_branch - with scoped_cwd(BRAVE_CORE_ROOT): - # get SHA for all commits (in order) - sha_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%h', '--reverse']) - sha_list = sha_list.split('\n') + channel_branch = remote_branch + '_' + local_branch + + with scoped_cwd(BRAVE_CORE_ROOT): + # get SHA for all commits (in order) + sha_list = execute(['git', 'log', 'origin/' + top_level_branch + '..HEAD', '--pretty=format:%h', '--reverse']) + sha_list = sha_list.split('\n') + try: + # check if branch exists already try: - # check if branch exists already - try: - branch_sha = execute(['git', 'rev-parse', '-q', '--verify', channel_branch]) - except Exception as e: - branch_sha = '' - - if len(branch_sha) > 0: - # branch exists; reset it - print('(' + channel + ') branch "' + channel_branch + '" exists; resetting to origin/' + remote_branch) - execute(['git', 'reset', '--hard', 'origin/' + remote_branch]) - else: - # create the branch - print('(' + channel + ') creating "' + channel_branch + '" from ' + channel) - execute(['git', 'checkout', remote_branch]) - execute(['git', 'checkout', '-b', channel_branch]) - - # TODO: handle errors thrown by cherry-pick - for sha in sha_list: - output = execute(['git', 'cherry-pick', sha]).split('\n') - print('- picked ' + sha + ' (' + output[0] + ')') - - finally: - # switch back to original branch - execute(['git', 'checkout', local_branch]) - execute(['git', 'reset', '--hard', sha_list[-1]]) - - config.branches_to_push.append(channel_branch) - - return channel_branch - return local_branch + branch_sha = execute(['git', 'rev-parse', '-q', '--verify', channel_branch]) + except Exception as e: + branch_sha = '' + + if len(branch_sha) > 0: + # branch exists; reset it + print('(' + channel + ') branch "' + channel_branch + + '" exists; resetting to origin/' + remote_branch) + execute(['git', 'reset', '--hard', 'origin/' + remote_branch]) + else: + # create the branch + print('(' + channel + ') creating "' + channel_branch + '" from ' + channel) + execute(['git', 'checkout', remote_branch]) + execute(['git', 'checkout', '-b', channel_branch]) + + # TODO: handle errors thrown by cherry-pick + for sha in sha_list: + output = execute(['git', 'cherry-pick', sha]).split('\n') + print('- picked ' + sha + ' (' + output[0] + ')') + + finally: + # switch back to original branch + execute(['git', 'checkout', local_branch]) + execute(['git', 'reset', '--hard', sha_list[-1]]) + + config.branches_to_push.append(channel_branch) + + return channel_branch def get_milestone_for_branch(channel_branch): @@ -283,7 +300,7 @@ def get_milestone_for_branch(channel_branch): return None -def submit_pr(channel, title, remote_branch, local_branch): +def submit_pr(channel, top_level_branch, remote_branch, local_branch): global config try: @@ -293,26 +310,31 @@ def submit_pr(channel, title, remote_branch, local_branch): return 0 print('(' + channel + ') creating pull request') - pr_title = title + pr_title = config.title pr_dst = remote_branch if is_nightly(channel): pr_dst = 'master' + + # add uplift specific details (if needed) + if local_branch.startswith(top_level_branch): pr_body = 'TODO: fill me in\n(created using `npm run pr`)' else: pr_title += ' (uplift to ' + remote_branch + ')' pr_body = 'Uplift of #' + str(config.master_pr_number) + number = create_pull_request(config.github_token, BRAVE_CORE_REPO, pr_title, pr_body, branch_src=local_branch, branch_dst=pr_dst, verbose=config.is_verbose, dryrun=config.is_dryrun) - # store the PR number (master only) so that it can be referenced in uplifts - if is_nightly(channel): + # store the original PR number so that it can be referenced in uplifts + if local_branch.startswith(top_level_branch): config.master_pr_number = number # assign milestone / reviewer(s) / owner(s) add_reviewers_to_pull_request(config.github_token, BRAVE_CORE_REPO, number, config.parsed_reviewers, verbose=config.is_verbose, dryrun=config.is_dryrun) - set_issue_details(config.github_token, BRAVE_CORE_REPO, number, milestone_number, config.parsed_owners, + set_issue_details(config.github_token, BRAVE_CORE_REPO, number, milestone_number, + config.parsed_owners, config.labels, verbose=config.is_verbose, dryrun=config.is_dryrun) except Exception as e: print('[ERROR] unhandled error occurred:', str(e)) From 15ddd25be0db801bfe233db6e2dee2d7843e2991 Mon Sep 17 00:00:00 2001 From: Brian Clifton Date: Tue, 12 Feb 2019 11:54:57 -0700 Subject: [PATCH 3/4] Script now considers npm config when looking for GitHub token Also, each PR created is opened in the default browser --- script/lib/github.py | 7 ++++++- script/pr.py | 35 ++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/script/lib/github.py b/script/lib/github.py index b70ff5c46fd6..920877bf407a 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -192,7 +192,7 @@ def get_milestones(token, repo_name, verbose=False): return response -def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, verbose=False, dryrun=False): +def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, open_in_browser=False, verbose=False, dryrun=False): post_data = { 'title': title, 'head': branch_src, @@ -204,11 +204,16 @@ def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, v # for more info see: http://developer.github.com/v3/pulls if dryrun: print('[INFO] would call `repo.pulls.post(' + str(post_data) + ')`') + if open_in_browser: + print('[INFO] would open PR in web browser') return 1234 repo = GitHub(token).repos(repo_name) response = repo.pulls.post(data=post_data) if verbose: print('repo.pulls.post(data) response:\n' + str(response)) + if open_in_browser: + import webbrowser + webbrowser.open(response['html_url']) return int(response['number']) diff --git a/script/pr.py b/script/pr.py index 4855ba945368..e3f2c5dc67a2 100755 --- a/script/pr.py +++ b/script/pr.py @@ -29,7 +29,6 @@ # TODOs for this version ##### # - parse out issue (so it can be included in body). ex: git log --pretty=format:%b -# - open PRs in default browser (shell exec) BY DEFAULT. Offer a quiet that DOESN'T open # - discover associated issues # - put them in the uplift / original PR body # - set milestone! (on the ISSUE) @@ -57,13 +56,17 @@ def initialize(self, args): # validate channel names validate_channel(args.uplift_to) validate_channel(args.start_from) - # TODO: read github token FIRST from CLI, then from .npmrc + # read github token FIRST from CLI, then from .npmrc self.github_token = get_env_var('GITHUB_TOKEN') if len(self.github_token) == 0: - # TODO: check .npmrc - # if not there, error out! - print('[ERROR] no valid GitHub token was found') - return 1 + try: + result = execute(['npm', 'config', 'get', 'BRAVE_GITHUB_TOKEN']).strip() + if result == 'undefined': + raise Exception('`BRAVE_GITHUB_TOKEN` value not found!') + self.github_token = result + except Exception as e: + print('[ERROR] no valid GitHub token was found either in npmrc or via environment variables (BRAVE_GITHUB_TOKEN)') + return 1 self.parsed_reviewers = parse_user_logins(self.github_token, args.reviewers, verbose=self.is_verbose) # if `--owners` is not provided, fall back to user owning token self.parsed_owners = parse_user_logins(self.github_token, args.owners, verbose=self.is_verbose) @@ -196,7 +199,10 @@ def main(): # get local version + latest version on remote (master) # if they don't match, rebase is needed - remote_version = get_remote_version(top_level_branch) + # TODO: FIXME. needs to be changed from 'master' to top_level_branch + # see TODO notes in create_branch() for more info + # remote_version = get_remote_version(top_level_branch) + remote_version = get_remote_version('master') if local_version != remote_version: if not args.force: print('[ERROR] Your branch is out of sync (local=' + local_version + @@ -255,7 +261,14 @@ def create_branch(channel, top_level_branch, remote_branch, local_branch): with scoped_cwd(BRAVE_CORE_ROOT): # get SHA for all commits (in order) - sha_list = execute(['git', 'log', 'origin/' + top_level_branch + '..HEAD', '--pretty=format:%h', '--reverse']) + # TODO: FIXME. needs to be changed from 'master' to top_level_branch + # however... there are complications with cherry-picking when that happens + # (ex: users feature branch would need to be based off 0.60.x for example) + # It would be good to detect if local branch is related (?) to the branch specified. + # ex: try against master- if no ancestor within X commits, then try against top_level_branch instead + # Basically: there needs to be a way to get ONLY the commits in this branch (and nothing more) + # sha_list = execute(['git', 'log', 'origin/' + top_level_branch + '..HEAD', '--pretty=format:%h', '--reverse']) + sha_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%h', '--reverse']) sha_list = sha_list.split('\n') try: # check if branch exists already @@ -316,7 +329,7 @@ def submit_pr(channel, top_level_branch, remote_branch, local_branch): pr_dst = 'master' # add uplift specific details (if needed) - if local_branch.startswith(top_level_branch): + if is_nightly(channel) or local_branch.startswith(top_level_branch): pr_body = 'TODO: fill me in\n(created using `npm run pr`)' else: pr_title += ' (uplift to ' + remote_branch + ')' @@ -324,10 +337,10 @@ def submit_pr(channel, top_level_branch, remote_branch, local_branch): number = create_pull_request(config.github_token, BRAVE_CORE_REPO, pr_title, pr_body, branch_src=local_branch, branch_dst=pr_dst, - verbose=config.is_verbose, dryrun=config.is_dryrun) + open_in_browser=True, verbose=config.is_verbose, dryrun=config.is_dryrun) # store the original PR number so that it can be referenced in uplifts - if local_branch.startswith(top_level_branch): + if is_nightly(channel) or local_branch.startswith(top_level_branch): config.master_pr_number = number # assign milestone / reviewer(s) / owner(s) From 983ce6a2d3b7e0a0c33f4df48ea75467cd300024 Mon Sep 17 00:00:00 2001 From: Brian Clifton Date: Tue, 12 Feb 2019 16:21:24 -0700 Subject: [PATCH 4/4] Added new `uplift_using_pr` option This does the following: - fetches the PR in question - creates a local branch with the content of the PR - does the rest of the process normally Handy since you don't need to actually have the code locally in order to uplift --- script/lib/github.py | 3 +- script/pr.py | 108 ++++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/script/lib/github.py b/script/lib/github.py index 920877bf407a..bfda3b1d0364 100644 --- a/script/lib/github.py +++ b/script/lib/github.py @@ -192,7 +192,8 @@ def get_milestones(token, repo_name, verbose=False): return response -def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, open_in_browser=False, verbose=False, dryrun=False): +def create_pull_request(token, repo_name, title, body, branch_src, branch_dst, + open_in_browser=False, verbose=False, dryrun=False): post_data = { 'title': title, 'head': branch_src, diff --git a/script/pr.py b/script/pr.py index e3f2c5dc67a2..f43b4c0a622a 100755 --- a/script/pr.py +++ b/script/pr.py @@ -65,7 +65,8 @@ def initialize(self, args): raise Exception('`BRAVE_GITHUB_TOKEN` value not found!') self.github_token = result except Exception as e: - print('[ERROR] no valid GitHub token was found either in npmrc or via environment variables (BRAVE_GITHUB_TOKEN)') + print('[ERROR] no valid GitHub token was found either in npmrc or ' + + 'via environment variables (BRAVE_GITHUB_TOKEN)') return 1 self.parsed_reviewers = parse_user_logins(self.github_token, args.reviewers, verbose=self.is_verbose) # if `--owners` is not provided, fall back to user owning token @@ -143,6 +144,9 @@ def parse_args(): parser.add_argument('--uplift-to', help='starting at nightly (master), how far back to uplift the changes', default='nightly') + parser.add_argument('--uplift-using-pr', + help='link to already existing pull request (number) to use as a reference for uplifting', + default=None) parser.add_argument('--start-from', help='instead of starting from nightly (default), start from beta/dev/release', default='nightly') @@ -169,6 +173,12 @@ def get_remote_version(branch_to_compare): return json_file['version'] +def fancy_print(text): + print('#' * len(text)) + print(text) + print('#' * len(text)) + + def main(): args = parse_args() if args.verbose: @@ -187,9 +197,9 @@ def main(): # also, find the branch which should be used for diffs (for cherry-picking) local_version = get_raw_version() remote_branches = get_remote_channel_branches(local_version) - top_level_branch = 'master' + top_level_base = 'master' if not is_nightly(args.start_from): - top_level_branch = remote_branches[args.start_from] + top_level_base = remote_branches[args.start_from] try: start_index = config.channel_names.index(args.start_from) config.channels_to_process = config.channel_names[start_index:] @@ -197,32 +207,75 @@ def main(): print('[ERROR] specified `start-from` value "' + args.start_from + '" not found in channel list') return 1 + # optionally (instead of having a local branch), allow uplifting a specific PR + # this pulls down the pr locally (in a special branch) + if args.uplift_using_pr: + pr_number = int(args.uplift_using_pr) + repo = GitHub(config.github_token).repos(BRAVE_CORE_REPO) + try: + # get enough details from PR to check out locally + response = repo.pulls(pr_number).get() + head = response['head'] + local_branch = 'pr' + str(pr_number) + '_' + head['ref'] + head_sha = head['sha'] + top_level_base = response['base']['ref'] + + except Exception as e: + print('[ERROR] API returned an error when looking up pull request "' + str(pr_number) + '"\n' + str(e)) + return 1 + + # set starting point AHEAD of the PR provided + config.master_pr_number = pr_number + if top_level_base == 'master': + config.channels_to_process = config.channel_names[1:] + else: + branch_index = remote_branches.index(top_level_base) + config.channels_to_process = config.channel_names[branch_index:] + + # create local branch which matches the contents of the PR + with scoped_cwd(BRAVE_CORE_ROOT): + # check if branch exists already + try: + branch_sha = execute(['git', 'rev-parse', '-q', '--verify', local_branch]) + except Exception as e: + branch_sha = '' + if len(branch_sha) > 0: + # branch exists; reset it + print('branch "' + local_branch + '" exists; resetting to origin/' + head['ref'] + ' (' + head_sha + ')') + execute(['git', 'checkout', local_branch]) + execute(['git', 'reset', '--hard', head_sha]) + else: + # create the branch + print('creating branch "' + local_branch + '" using origin/' + head['ref'] + ' (' + head_sha + ')') + execute(['git', 'checkout', '-b', local_branch, head_sha]) + # now that branch exists, switch back to previous local branch + execute(['git', 'checkout', '-']) + # get local version + latest version on remote (master) # if they don't match, rebase is needed - # TODO: FIXME. needs to be changed from 'master' to top_level_branch - # see TODO notes in create_branch() for more info - # remote_version = get_remote_version(top_level_branch) - remote_version = get_remote_version('master') + remote_version = get_remote_version(top_level_base) if local_version != remote_version: if not args.force: print('[ERROR] Your branch is out of sync (local=' + local_version + ', remote=' + remote_version + '); please rebase (ex: "git rebase origin/' + - top_level_branch + '"). NOTE: You can bypass this check by using -f') + top_level_base + '"). NOTE: You can bypass this check by using -f') return 1 print('[WARNING] Your branch is out of sync (local=' + local_version + ', remote=' + remote_version + '); continuing since -f was provided') + # If title isn't set already, generate one from first commit local_branch = get_local_branch_name(BRAVE_CORE_ROOT) if not config.title: - config.title = get_title_from_first_commit(BRAVE_CORE_ROOT, top_level_branch) + config.title = get_title_from_first_commit(BRAVE_CORE_ROOT, top_level_base) # Create a branch for each channel print('\nCreating branches...') + fancy_print('NOTE: Commits are being detected by diffing "' + local_branch + '" against "' + top_level_base + '"') remote_branches = get_remote_channel_branches(local_version) local_branches = {} try: for channel in config.channels_to_process: - branch = create_branch(channel, top_level_branch, remote_branches[channel], local_branch) + branch = create_branch(channel, top_level_base, remote_branches[channel], local_branch) local_branches[channel] = branch if channel == args.uplift_to: break @@ -238,7 +291,7 @@ def main(): for channel in config.channels_to_process: submit_pr( channel, - top_level_branch, + top_level_base, remote_branches[channel], local_branches[channel]) if channel == args.uplift_to: @@ -251,24 +304,17 @@ def main(): return 0 -def create_branch(channel, top_level_branch, remote_branch, local_branch): +def create_branch(channel, top_level_base, remote_base, local_branch): global config if is_nightly(channel): return local_branch - channel_branch = remote_branch + '_' + local_branch + channel_branch = remote_base + '_' + local_branch with scoped_cwd(BRAVE_CORE_ROOT): # get SHA for all commits (in order) - # TODO: FIXME. needs to be changed from 'master' to top_level_branch - # however... there are complications with cherry-picking when that happens - # (ex: users feature branch would need to be based off 0.60.x for example) - # It would be good to detect if local branch is related (?) to the branch specified. - # ex: try against master- if no ancestor within X commits, then try against top_level_branch instead - # Basically: there needs to be a way to get ONLY the commits in this branch (and nothing more) - # sha_list = execute(['git', 'log', 'origin/' + top_level_branch + '..HEAD', '--pretty=format:%h', '--reverse']) - sha_list = execute(['git', 'log', 'origin/master..HEAD', '--pretty=format:%h', '--reverse']) + sha_list = execute(['git', 'log', 'origin/' + top_level_base + '..HEAD', '--pretty=format:%h', '--reverse']) sha_list = sha_list.split('\n') try: # check if branch exists already @@ -280,12 +326,12 @@ def create_branch(channel, top_level_branch, remote_branch, local_branch): if len(branch_sha) > 0: # branch exists; reset it print('(' + channel + ') branch "' + channel_branch + - '" exists; resetting to origin/' + remote_branch) - execute(['git', 'reset', '--hard', 'origin/' + remote_branch]) + '" exists; resetting to origin/' + remote_base) + execute(['git', 'reset', '--hard', 'origin/' + remote_base]) else: # create the branch print('(' + channel + ') creating "' + channel_branch + '" from ' + channel) - execute(['git', 'checkout', remote_branch]) + execute(['git', 'checkout', remote_base]) execute(['git', 'checkout', '-b', channel_branch]) # TODO: handle errors thrown by cherry-pick @@ -313,26 +359,26 @@ def get_milestone_for_branch(channel_branch): return None -def submit_pr(channel, top_level_branch, remote_branch, local_branch): +def submit_pr(channel, top_level_base, remote_base, local_branch): global config try: - milestone_number = get_milestone_for_branch(remote_branch) + milestone_number = get_milestone_for_branch(remote_base) if milestone_number is None: - print('milestone for "' + remote_branch + '"" was not found!') + print('milestone for "' + remote_base + '"" was not found!') return 0 print('(' + channel + ') creating pull request') pr_title = config.title - pr_dst = remote_branch + pr_dst = remote_base if is_nightly(channel): pr_dst = 'master' # add uplift specific details (if needed) - if is_nightly(channel) or local_branch.startswith(top_level_branch): + if is_nightly(channel) or local_branch.startswith(top_level_base): pr_body = 'TODO: fill me in\n(created using `npm run pr`)' else: - pr_title += ' (uplift to ' + remote_branch + ')' + pr_title += ' (uplift to ' + remote_base + ')' pr_body = 'Uplift of #' + str(config.master_pr_number) number = create_pull_request(config.github_token, BRAVE_CORE_REPO, pr_title, pr_body, @@ -340,7 +386,7 @@ def submit_pr(channel, top_level_branch, remote_branch, local_branch): open_in_browser=True, verbose=config.is_verbose, dryrun=config.is_dryrun) # store the original PR number so that it can be referenced in uplifts - if is_nightly(channel) or local_branch.startswith(top_level_branch): + if is_nightly(channel) or local_branch.startswith(top_level_base): config.master_pr_number = number # assign milestone / reviewer(s) / owner(s)