|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# |
| 4 | +# Licensed to the Apache Software Foundation (ASF) under one or more |
| 5 | +# contributor license agreements. See the NOTICE file distributed with |
| 6 | +# this work for additional information regarding copyright ownership. |
| 7 | +# The ASF licenses this file to You under the Apache License, Version 2.0 |
| 8 | +# (the "License"); you may not use this file except in compliance with |
| 9 | +# the License. You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, software |
| 14 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | +# See the License for the specific language governing permissions and |
| 17 | +# limitations under the License. |
| 18 | +# |
| 19 | + |
| 20 | +# This script is inspired by Apache Spark |
| 21 | + |
| 22 | +# This script simplifies the process of creating release notes, it |
| 23 | +# - folds the original and the revert commits |
| 24 | +# - filters out unrelated commits |
| 25 | +# - generates the contributor list |
| 26 | +# - canonicalizes the contributors' name with the known_translations |
| 27 | + |
| 28 | +# TODO |
| 29 | +# - canonicalizes the commits' title |
| 30 | + |
| 31 | +# Usage: |
| 32 | +# set environment variables: RELEASE_TAG and PREVIOUS_RELEASE_TAG, then perform |
| 33 | +# ./pre_gen_release_notes.py |
| 34 | +# Example: |
| 35 | +# RELEASE_TAG=v1.8.1 PREVIOUS_RELEASE_TAG=v1.8.0 ./pre_gen_release_notes.py |
| 36 | + |
| 37 | +# It outputs |
| 38 | +# - commits-${RELEASE_TAG}.txt: the canonical commit list |
| 39 | +# - contributors-${RELEASE_TAG}.txt: the canonical contributor list |
| 40 | + |
| 41 | +import os |
| 42 | +import re |
| 43 | +import sys |
| 44 | + |
| 45 | +from release_utils import ( |
| 46 | + tag_exists, |
| 47 | + get_commits, |
| 48 | + yes_or_no_prompt, |
| 49 | + get_date, |
| 50 | + is_valid_author, |
| 51 | + capitalize_author, |
| 52 | + print_indented |
| 53 | +) |
| 54 | + |
| 55 | +RELEASE_TAG = os.environ.get("RELEASE_TAG") |
| 56 | +if RELEASE_TAG is None: |
| 57 | + sys.exit("RELEASE_TAG is required") |
| 58 | +if not tag_exists(RELEASE_TAG): |
| 59 | + sys.exit("RELEASE_TAG: %s does not exist!" % RELEASE_TAG) |
| 60 | + |
| 61 | +PREVIOUS_RELEASE_TAG = os.environ.get("PREVIOUS_RELEASE_TAG") |
| 62 | +if PREVIOUS_RELEASE_TAG is None: |
| 63 | + sys.exit("PREVIOUS_RELEASE_TAG is required") |
| 64 | +if not tag_exists(PREVIOUS_RELEASE_TAG): |
| 65 | + sys.exit("PREVIOUS_RELEASE_TAG: %s does not exist!" % PREVIOUS_RELEASE_TAG) |
| 66 | + |
| 67 | +release_dir = os.path.dirname(os.path.abspath(__file__)) |
| 68 | +commits_file_name = "commits-%s.txt" % RELEASE_TAG |
| 69 | +contributors_file_name = "contributors-%s.txt" % RELEASE_TAG |
| 70 | + |
| 71 | +# Gather commits found in the new tag but not in the old tag. |
| 72 | +# This filters commits based on both the git hash and the PR number. |
| 73 | +# If either is present in the old tag, then we ignore the commit. |
| 74 | +print("Gathering new commits between tags %s and %s" % (PREVIOUS_RELEASE_TAG, RELEASE_TAG)) |
| 75 | +release_commits = get_commits(RELEASE_TAG) |
| 76 | +previous_release_commits = get_commits(PREVIOUS_RELEASE_TAG) |
| 77 | +previous_release_hashes = set() |
| 78 | +previous_release_prs = set() |
| 79 | +for old_commit in previous_release_commits: |
| 80 | + previous_release_hashes.add(old_commit.get_hash()) |
| 81 | + if old_commit.get_pr_number(): |
| 82 | + previous_release_prs.add(old_commit.get_pr_number()) |
| 83 | +new_commits = [] |
| 84 | +for this_commit in release_commits: |
| 85 | + this_hash = this_commit.get_hash() |
| 86 | + this_pr_number = this_commit.get_pr_number() |
| 87 | + if this_hash in previous_release_hashes: |
| 88 | + continue |
| 89 | + if this_pr_number and this_pr_number in previous_release_prs: |
| 90 | + continue |
| 91 | + new_commits.append(this_commit) |
| 92 | +if not new_commits: |
| 93 | + sys.exit("There are no new commits between %s and %s!" % (PREVIOUS_RELEASE_TAG, RELEASE_TAG)) |
| 94 | + |
| 95 | +# Prompt the user for confirmation that the commit range is correct |
| 96 | +print("\n==================================================================================") |
| 97 | +print("Release tag: %s" % RELEASE_TAG) |
| 98 | +print("Previous release tag: %s" % PREVIOUS_RELEASE_TAG) |
| 99 | +print("Number of commits in this range: %s" % len(new_commits)) |
| 100 | +print("") |
| 101 | + |
| 102 | +if yes_or_no_prompt("Show all commits?"): |
| 103 | + print_indented(new_commits) |
| 104 | +print("==================================================================================\n") |
| 105 | +if not yes_or_no_prompt("Does this look correct?"): |
| 106 | + sys.exit("Ok, exiting") |
| 107 | + |
| 108 | +# Filter out special commits |
| 109 | +releases = [] |
| 110 | +reverts = [] |
| 111 | +no_tickets = [] |
| 112 | +effective_commits = [] |
| 113 | + |
| 114 | +def is_release(commit_title): |
| 115 | + return "[release]" in commit_title.lower() |
| 116 | + |
| 117 | + |
| 118 | +def has_no_ticket(commit_title): |
| 119 | + return not re.findall("\\[KYUUBI\\s\\#[0-9]+\\]", commit_title.upper()) |
| 120 | + |
| 121 | + |
| 122 | +def is_revert(commit_title): |
| 123 | + return "revert" in commit_title.lower() |
| 124 | + |
| 125 | + |
| 126 | +for c in new_commits: |
| 127 | + t = c.get_title() |
| 128 | + if not t: |
| 129 | + continue |
| 130 | + elif is_release(t): |
| 131 | + releases.append(c) |
| 132 | + elif is_revert(t): |
| 133 | + reverts.append(c) |
| 134 | + elif has_no_ticket(t): |
| 135 | + no_tickets.append(c) |
| 136 | + else: |
| 137 | + effective_commits.append(c) |
| 138 | + |
| 139 | + |
| 140 | +# Warn against ignored commits |
| 141 | +if releases or reverts or no_tickets: |
| 142 | + print("\n==================================================================================") |
| 143 | + if releases: |
| 144 | + print("Found %d release commits" % len(releases)) |
| 145 | + if reverts: |
| 146 | + print("Found %d revert commits" % len(reverts)) |
| 147 | + if no_tickets: |
| 148 | + print("Found %d commits with no Ticket" % len(no_tickets)) |
| 149 | + print("==================== Warning: these commits will be ignored ======================\n") |
| 150 | + if releases: |
| 151 | + print("Release (%d)" % len(releases)) |
| 152 | + print_indented(releases) |
| 153 | + if reverts: |
| 154 | + print("Revert (%d)" % len(reverts)) |
| 155 | + print_indented(reverts) |
| 156 | + if no_tickets: |
| 157 | + print("No Ticket (%d)" % len(no_tickets)) |
| 158 | + print_indented(no_tickets) |
| 159 | + print("==================== Warning: the above commits will be ignored ==================\n") |
| 160 | +prompt_msg = "%d effective commits left to process after filtering. OK to proceed?" % len(effective_commits) |
| 161 | +if not yes_or_no_prompt(prompt_msg): |
| 162 | + sys.exit("OK, exiting.") |
| 163 | + |
| 164 | + |
| 165 | +# Load known author translations that are cached locally |
| 166 | +known_translations = {} |
| 167 | +known_translations_file_name = "known_translations" |
| 168 | +known_translations_file = open(os.path.join(release_dir, known_translations_file_name), "r") |
| 169 | +for line in known_translations_file: |
| 170 | + if line.startswith("#") or not line.strip(): |
| 171 | + continue |
| 172 | + [old_name, new_name] = line.strip("\n").split(" - ") |
| 173 | + known_translations[old_name] = new_name |
| 174 | +known_translations_file.close() |
| 175 | + |
| 176 | +# Keep track of warnings to tell the user at the end |
| 177 | +warnings = [] |
| 178 | + |
| 179 | +# Mapping from the invalid author name to its associated tickets |
| 180 | +# E.g. pan3793 -> set("[KYUUBI #1234]", "[KYUUBI #1235]") |
| 181 | +invalid_authors = {} |
| 182 | + |
| 183 | +# Populate a map that groups issues and components by author |
| 184 | +# It takes the form: Author Name -> set() |
| 185 | +# For instance, |
| 186 | +# { |
| 187 | +# 'Cheng Pan' -> set('[KYUUBI #1234]', '[KYUUBI #1235]'), |
| 188 | +# 'Fu Chen' -> set('[KYUUBI #2345]') |
| 189 | +# } |
| 190 | +# |
| 191 | +author_info = {} |
| 192 | +print("\n=========================== Compiling contributor list ===========================") |
| 193 | +for commit in effective_commits: |
| 194 | + _hash = commit.get_hash() |
| 195 | + title = commit.get_title() |
| 196 | + issues = re.findall("\\[KYUUBI\\s\\#[0-9]+\\]", title.upper()) |
| 197 | + author = commit.get_author() |
| 198 | + date = get_date(_hash) |
| 199 | + # Translate the known author name |
| 200 | + if author in known_translations: |
| 201 | + author = known_translations[author] |
| 202 | + elif is_valid_author(author): |
| 203 | + # If the author name is invalid, keep track of it along |
| 204 | + # with all associated issues so we can translate it later |
| 205 | + author = capitalize_author(author) |
| 206 | + else: |
| 207 | + if author not in invalid_authors: |
| 208 | + invalid_authors[author] = set() |
| 209 | + for issue in issues: |
| 210 | + invalid_authors[author].add(issue) |
| 211 | + # Populate or merge an issue into author_info[author] |
| 212 | + def populate(issues): |
| 213 | + if author not in author_info: |
| 214 | + author_info[author] = set() |
| 215 | + for issue in issues: |
| 216 | + author_info[author].add(issue) |
| 217 | + # Find issues associated with this commit |
| 218 | + try: |
| 219 | + populate(issues) |
| 220 | + except Exception as e: |
| 221 | + print("Unexpected error:", e) |
| 222 | + print(" Processed commit %s authored by %s on %s" % (_hash, author, date)) |
| 223 | +print("==================================================================================\n") |
| 224 | + |
| 225 | +commits_file = open(os.path.join(release_dir, commits_file_name), "w") |
| 226 | +for commit in effective_commits: |
| 227 | + if commit.get_hash() not in map(lambda revert: revert.get_revert_hash(), reverts): |
| 228 | + commits_file.write(commit.title + "\n") |
| 229 | +for commit in no_tickets: |
| 230 | + commits_file.write(commit.title + "\n") |
| 231 | +commits_file.close() |
| 232 | +print("Commits list is successfully written to %s!" % commits_file_name) |
| 233 | + |
| 234 | +# Write to contributors file ordered by author names |
| 235 | +# Each line takes the format " * Author Name -- tickets" |
| 236 | +# e.g. * Cheng Pan -- [KYUUBI #1234][KYUUBI #1235] |
| 237 | +# e.g. * Fu Chen -- [KYUUBI #2345] |
| 238 | +contributors_file = open(os.path.join(release_dir, contributors_file_name), "w") |
| 239 | +authors = list(author_info.keys()) |
| 240 | +authors.sort(key=lambda author: author.split(" ")[-1]) |
| 241 | +author_max_len = max(len(author) for author in authors) |
| 242 | +for author in authors: |
| 243 | + contribution = "".join(author_info[author]) |
| 244 | + line = ("* {:<%s}" % author_max_len).format(author) + " -- " + contribution |
| 245 | + contributors_file.write(line + "\n") |
| 246 | +contributors_file.close() |
| 247 | +print("Contributors list is successfully written to %s!" % contributors_file_name) |
| 248 | + |
| 249 | +# Prompt the user to translate author names if necessary |
| 250 | +if invalid_authors: |
| 251 | + warnings.append("Found the following invalid authors:") |
| 252 | + for a in invalid_authors: |
| 253 | + warnings.append("\t%s" % a) |
| 254 | + warnings.append("Please update 'known_translations'.") |
| 255 | + |
| 256 | +# Log any warnings encountered in the process |
| 257 | +if warnings: |
| 258 | + print("\n============ Warnings encountered while creating the contributor list ============") |
| 259 | + for w in warnings: |
| 260 | + print(w) |
| 261 | + print("Please correct these in the final contributors list at %s." % contributors_file_name) |
| 262 | + print("==================================================================================\n") |
0 commit comments