Associate unreleased PRs and PRs with rebased commits

Unreleased PRs will be associated with the next release based on either
the --release-branch or the GitHub default branch if a release branch is
not specified

PRs not found in tags or in the release branch (due to the SHAs changing
during a rebase) may be associated with SHAs specified in GitHub
comments.

lib/github_changelog_generator/generator/generator.rb

@@ -147,7 +147,7 @@ def fetch_issues_and_pr
 
       fetch_events_for_issues_and_pr
       detect_actual_closed_dates(@issues + @pull_requests)
-      @fetcher.add_first_occurring_tag_to_prs(@sorted_tags, @pull_requests)
+      add_first_occurring_tag_to_prs(@sorted_tags, @pull_requests)
       nil
     end
   end

lib/github_changelog_generator/generator/generator_fetcher.rb

@@ -48,6 +48,109 @@ def detect_actual_closed_dates(issues)
       puts "Fetching closed dates for issues: Done!" if options[:verbose]
     end
 
+    # Adds a key "first_occurring_tag" to each PR with a value of the oldest
+    # tag that a PR's merge commit occurs in in the git history. This should
+    # indicate the release of each PR by git's history regardless of dates and
+    # divergent branches.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to discover the tags of.
+    # @return [Nil] No return; PRs are updated in-place.
+    def add_first_occurring_tag_to_prs(tags, prs)
+      total = prs.count
+
+      prs_left = associate_tagged_prs(tags, prs, total)
+      prs_left = associate_release_branch_prs(prs_left, total)
+      associate_rebase_comment_prs(tags, prs_left, total) if prs_left.any?
+      Helper.log.info "Associating PRs with tags: #{total - prs_left.count}/#{total}"
+    end
+
+    # Associate merged PRs by the merge SHA contained in each tag. If the
+    # merge_commit_sha is not found in any tag's history, skip association.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to associate.
+    # @return [Array] PRs without their merge_commit_sha in a tag.
+    def associate_tagged_prs(tags, prs, total)
+      @fetcher.fetch_tag_shas_async(tags)
+
+      i = 0
+      prs.reject do |pr|
+        found = false
+        # XXX Wish I could use merge_commit_sha, but gcg doesn't currently
+        # fetch that. See
+        # https://developer.github.com/v3/pulls/#get-a-single-pull-request vs.
+        # https://developer.github.com/v3/pulls/#list-pull-requests
+        if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" })
+          # Iterate tags.reverse (oldest to newest) to find first tag of each PR.
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(event["commit_id"]) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+        else
+          raise StandardError, "No merge sha found for PR #{pr['number']}"
+        end
+        found
+      end
+    end
+
+    # Associate merged PRs by the HEAD of the release branch. If no
+    # --release-branch was specified, then the github default branch is used.
+    #
+    # @param [Array] prs_left PRs not associated with any tag.
+    # @param [Integer] total The total number of PRs to associate; used for verbose printing.
+    # @return [Array] PRs without their merge_commit_sha in the branch.
+    def associate_release_branch_prs(prs_left, total)
+      if prs_left.any?
+        i = total - prs_left.count
+        prs_left.reject do |pr|
+          found = false
+          if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" }) && sha_in_release_branch(event["commit_id"])
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+          found
+        end
+      else
+        prs_left
+      end
+    end
+
+    # Associate merged PRs by the SHA detected in github comments of the form
+    # "rebased commit: <sha>". For use when the merge_commit_sha is not in the
+    # actual git history due to rebase.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs_left The PRs not yet associated with any tag or branch.
+    # @return [Nil] No return. Any remaining PRs are unresolvable as-is.
+    def associate_rebase_comment_prs(tags, prs_left, total)
+      i = total - prs_left.count
+      # Any remaining PRs were not found in the list of tags by their merge
+      # commit and not found in any specified release branch. Fallback to
+      # rebased commit comment.
+      @fetcher.fetch_comments_async(prs_left)
+      prs_left.each do |pr|
+        if pr["comments"] && (rebased_comment = pr["comments"].reverse.find { |c| c["body"].match(%r{rebased commit: ([0-9a-f]{40})}i) })
+          rebased_sha = rebased_comment["body"].match(%r{rebased commit: ([0-9a-f]{40})}i)[1]
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(rebased_sha) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            i += 1
+          elsif sha_in_release_branch(rebased_sha)
+            i += 1
+          else
+            raise StandardError, "PR #{pr['number']} has a rebased SHA comment but that SHA was not found in the release branch or any tags"
+          end
+          print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+        else
+          puts "Warning: PR #{pr['number']} merge commit was not found in the release branch or tagged git history and no rebased SHA comment was found"
+        end
+      end
+      nil
+    end
+
     # Fill :actual_date parameter of specified issue by closed date of the commit, if it was closed by commit.
     # @param [Hash] issue
     def find_closed_date_by_commit(issue)
@@ -84,5 +187,19 @@ def set_date_from_event(event, issue)
         end
       end
     end
+
+    private
+
+    # Detect if a sha occurs in the --release-branch. Uses the github repo
+    # default branch if not specified.
+    #
+    # @param [String] sha SHA to check.
+    # @return [Boolean] True if SHA is in the branch git history.
+    def sha_in_release_branch(sha)
+      branch = @options[:release_branch] || @fetcher.default_branch
+      commits_in_branch = @fetcher.fetch_compare(@fetcher.oldest_commit["sha"], branch)
+      shas_in_branch = commits_in_branch["commits"].collect { |commit| commit["sha"] }
+      shas_in_branch.include?(sha)
+    end
   end
 end

lib/github_changelog_generator/octo_fetcher.rb

@@ -213,6 +213,29 @@ def fetch_events_async(issues)
       Helper.log.info "Fetching events for issues and PR: #{i}"
     end
 
+    # Fetch comments for PRs and add them to "comments"
+    #
+    # @param [Array] prs The array of PRs.
+    # @return [Void] No return; PRs are updated in-place.
+    def fetch_comments_async(prs)
+      threads = []
+
+      prs.each_slice(MAX_THREAD_NUMBER) do |prs_slice|
+        prs_slice.each do |pr|
+          threads << Thread.new do
+            pr["comments"] = []
+            iterate_pages(@client, "issue_comments", pr["number"]) do |new_comment|
+              pr["comments"].concat(new_comment)
+            end
+            pr["comments"] = pr["comments"].map { |comment| stringify_keys_deep(comment.to_hash) }
+          end
+        end
+        threads.each(&:join)
+        threads = []
+      end
+      nil
+    end
+
     # Fetch tag time from repo
     #
     # @param [Hash] tag GitHub data item about a Tag
@@ -277,41 +300,39 @@ def oldest_commit
       commits.last
     end
 
-    # Adds a key "first_occurring_tag" to each PR with a value of the oldest
-    # tag that a PR's merge commit occurs in in the git history. This should
-    # indicate the release of each PR by git's history regardless of dates and
-    # divergent branches.
+    # @return [String] Default branch of the repo
+    def default_branch
+      @default_branch ||= @client.repository(user_project)[:default_branch]
+    end
+
+    # Fetch all SHAs occurring in or before a given tag and add them to
+    # "shas_in_tag"
     #
-    # @param [Array] tags The array of tags sorted by time, newest to oldest.
-    # @param [Array] prs The array of PRs to discover the tags of.
-    # @return [Nil] No return; PRs are updated in-place.
-    def add_first_occurring_tag_to_prs(tags, prs)
-      # Shallow-clone tags and prs to avoid modification of passed arrays.
-      # Iterate tags.reverse (oldest to newest) to find first tag of each PR.
-      tags = tags.dup.reverse
-      prs = prs.dup
-      total = prs.length
-      while tags.any? && prs.any?
-        print_in_same_line("Associating PRs with tags: #{total - prs.length}/#{total}")
-        tag = tags.shift
-        # Use oldest commit because comparing two arbitrary tags may be diverged
-        commits_in_tag = fetch_compare(oldest_commit["sha"], tag["name"])
-        shas_in_tag = commits_in_tag["commits"].collect { |commit| commit["sha"] }
-        prs = prs.reject do |pr|
-          # XXX Wish I could use merge_commit_sha, but gcg doesn't currently
-          # fetch that. See https://developer.github.com/v3/pulls/#get-a-single-pull-request
-          if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" })
-            pr_sha = event["commit_id"]
-            pr["first_occurring_tag"] = tag["name"] if shas_in_tag.include?(pr_sha)
-          else
-            raise StandardError, "No merge sha found for PR #{pr['number']}"
+    # @param [Array] tags The array of tags.
+    # @return [Nil] No return; tags are updated in-place.
+    def fetch_tag_shas_async(tags)
+      i = 0
+      threads = []
+      print_in_same_line("Fetching SHAs for tags: #{i}/#{tags.count}\r") if @options[:verbose]
+
+      tags.each_slice(MAX_THREAD_NUMBER) do |tags_slice|
+        tags_slice.each do |tag|
+          threads << Thread.new do
+            # Use oldest commit because comparing two arbitrary tags may be diverged
+            commits_in_tag = fetch_compare(oldest_commit["sha"], tag["name"])
+            tag["shas_in_tag"] = commits_in_tag["commits"].collect { |commit| commit["sha"] }
+            print_in_same_line("Fetching SHAs for tags: #{i + 1}/#{tags.count}") if @options[:verbose]
+            i += 1
           end
         end
+        threads.each(&:join)
+        threads = []
       end
-      # All tags have been shifted or prs mapped, and as many PRs have been
-      # associated with tags as possible. Any remaining PRs are unreleased.
-      # Any remaining tags have no known PRs.
-      Helper.log.info "Associating PRs with tags: #{total}"
+
+      # to clear line from prev print
+      print_empty_line
+
+      Helper.log.info "Fetching SHAs for tags: #{i}"
       nil
     end
 

man/git-generate-changelog.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "GIT\-GENERATE\-CHANGELOG" "1" "January 2018" "" ""
+.TH "GIT\-GENERATE\-CHANGELOG" "1" "March 2018" "" ""
 .
 .SH "NAME"
 \fBgit\-generate\-changelog\fR \- Generate changelog from GitHub
@@ -244,7 +244,7 @@ Limit pull requests to the release branch, such as master or release
 \-\-http\-cache
 .
 .P
-Use HTTP Cache to cache Github API requests (useful for large repos) Default is true\.
+Use HTTP Cache to cache GitHub API requests (useful for large repos) Default is true\.
 .
 .P
 \-\-[no\-]cache\-file [CACHE\-FILE]
@@ -307,6 +307,9 @@ Print version number
 .P
 Displays Help
 .
+.SH "REBASED COMMITS"
+GitHub pull requests that have been merged whose merge commit SHA has been modified through rebasing, cherry picking, or some other method may be tracked via a special comment on GitHub\. Git commit SHAs found in comments on pull requests matching the regular expression \fB/rebased commit: ([0\-9a\-f]{40})/i\fR will be used in place of the original merge SHA when being added to the changelog\.
+.
 .SH "EXAMPLES"
 .
 .SH "AUTHOR"

man/git-generate-changelog.md

@@ -165,7 +165,7 @@ Automatically generate changelog from your tags, issues, labels and pull request
 
    --http-cache
 
-   Use HTTP Cache to cache Github API requests (useful for large repos) Default is true.
+   Use HTTP Cache to cache GitHub API requests (useful for large repos) Default is true.
 
    --[no-]cache-file [CACHE-FILE]
 
@@ -203,6 +203,9 @@ Automatically generate changelog from your tags, issues, labels and pull request
 
    Displays Help
 
+## REBASED COMMITS
+
+GitHub pull requests that have been merged whose merge commit SHA has been modified through rebasing, cherry picking, or some other method may be tracked via a special comment on GitHub. Git commit SHAs found in comments on pull requests matching the regular expression `/rebased commit: ([0-9a-f]{40})/i` will be used in place of the original merge SHA when being added to the changelog.
 
 ## EXAMPLES