Skip to content

Storing branch commits count in db rather than caching them in memory or redis #33954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Next Next commit
Fix dependency recycle
  • Loading branch information
lunny committed Mar 20, 2025
commit 43472b1fbffc28cf93b51ca06bc2b0a4e2966cc3
10 changes: 10 additions & 0 deletions models/git/branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ type Branch struct {
Repo *repo_model.Repository `xorm:"-"`
Name string `xorm:"UNIQUE(s) NOT NULL"` // git's ref-name is case-sensitive internally, however, in some databases (mssql, mysql, by default), it's case-insensitive at the moment
CommitID string
CommitCount int64 // the number of commits in this branch
CommitMessage string `xorm:"TEXT"` // it only stores the message summary (the first line)
PusherID int64
Pusher *user_model.User `xorm:"-"`
Expand Down Expand Up @@ -251,6 +252,15 @@ func UpdateBranch(ctx context.Context, repoID, pusherID int64, branchName string
})
}

func UpdateBranchCommitCount(ctx context.Context, repoID int64, branchName string, commitCount int64) error {
_, err := db.GetEngine(ctx).Where("repo_id=? AND name=?", repoID, branchName).
Cols("commit_count").
Update(&Branch{
CommitCount: commitCount,
})
return err
}

// AddDeletedBranch adds a deleted branch to the database
func AddDeletedBranch(ctx context.Context, repoID int64, branchName string, deletedByID int64) error {
branch, err := GetBranch(ctx, repoID, branchName)
Expand Down
1 change: 1 addition & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ func prepareMigrationTasks() []*migration {
newMigration(315, "Add Ephemeral to ActionRunner", v1_24.AddEphemeralToActionRunner),
newMigration(316, "Add description for secrets and variables", v1_24.AddDescriptionForSecretsAndVariables),
newMigration(317, "Add new index for action for heatmap", v1_24.AddNewIndexForUserDashboard),
newMigration(318, "Add branch commits count for branch table", v1_24.AddBranchCommitsCount),
}
return preparedMigrations
}
Expand Down
15 changes: 15 additions & 0 deletions models/migrations/v1_24/v318.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_24 //nolint

import (
"xorm.io/xorm"
)

func AddBranchCommitsCount(x *xorm.Engine) error {
type Branch struct {
CommitCount int64 // the number of commits in this branch
}
return x.Sync(new(Branch))
}
11 changes: 0 additions & 11 deletions models/repo/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -376,17 +376,6 @@ func (repo *Repository) APIURL() string {
return setting.AppURL + "api/v1/repos/" + url.PathEscape(repo.OwnerName) + "/" + url.PathEscape(repo.Name)
}

// GetCommitsCountCacheKey returns cache key used for commits count caching.
func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string {
var prefix string
if isRef {
prefix = "ref"
} else {
prefix = "commit"
}
return fmt.Sprintf("commits-count-%d-%s-%s", repo.ID, prefix, contextName)
}

// LoadUnits loads repo units into repo.Units
func (repo *Repository) LoadUnits(ctx context.Context) (err error) {
if repo.Units != nil {
Expand Down
12 changes: 1 addition & 11 deletions modules/git/repo_commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,18 +502,8 @@ func (repo *Repository) IsCommitInBranch(commitID, branch string) (r bool, err e
return len(stdout) > 0, err
}

func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error {
func (repo *Repository) AddLastCommitCache(commitsCount int64, fullName, sha string) error {
if repo.LastCommitCache == nil {
commitsCount, err := cache.GetInt64(cacheKey, func() (int64, error) {
commit, err := repo.GetCommit(sha)
if err != nil {
return 0, err
}
return commit.CommitsCount()
})
if err != nil {
return err
}
repo.LastCommitCache = NewLastCommitCache(commitsCount, fullName, repo, cache.GetCache())
}
return nil
Expand Down
4 changes: 3 additions & 1 deletion routers/api/v1/utils/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/services/context"
repo_service "code.gitea.io/gitea/services/repository"
)

// ResolveRefOrSha resolve ref to sha if exist
Expand All @@ -38,7 +39,8 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string {
sha = MustConvertToSHA1(ctx, ctx.Repo, sha)

if ctx.Repo.GitRepo != nil {
err := ctx.Repo.GitRepo.AddLastCommitCache(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, ref != sha), ctx.Repo.Repository.FullName(), sha)
commitsCount, _ := repo_service.GetRefCommitsCount(ctx, ctx.Repo.Repository.ID, git.RefName(ref))
err := ctx.Repo.GitRepo.AddLastCommitCache(commitsCount, ctx.Repo.Repository.FullName(), sha)
if err != nil {
log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package actions
package common

import (
"net/http"
Expand Down
2 changes: 1 addition & 1 deletion routers/web/repo/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ func FileHistory(ctx *context.Context) {
}

func LoadBranchesAndTags(ctx *context.Context) {
response, err := repo_service.LoadBranchesAndTags(ctx, ctx.Repo, ctx.PathParam("sha"))
response, err := repo_service.LoadBranchesAndTags(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.Repo.RepoLink, ctx.PathParam("sha"))
if err == nil {
ctx.JSON(http.StatusOK, response)
return
Expand Down
13 changes: 5 additions & 8 deletions services/context/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
asymkey_service "code.gitea.io/gitea/services/asymkey"
repo_service "code.gitea.io/gitea/services/repository"

"github.com/editorconfig/editorconfig-core-go/v2"
)
Expand Down Expand Up @@ -164,15 +165,11 @@ func (r *Repository) CanCreateIssueDependencies(ctx context.Context, user *user_
}

// GetCommitsCount returns cached commit count for current view
func (r *Repository) GetCommitsCount() (int64, error) {
func (r *Repository) GetCommitsCount(ctx context.Context) (int64, error) {
if r.Commit == nil {
return 0, nil
}
contextName := r.RefFullName.ShortName()
isRef := r.RefFullName.IsBranch() || r.RefFullName.IsTag()
return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, isRef), func() (int64, error) {
return r.Commit.CommitsCount()
})
return repo_service.GetRefCommitsCount(ctx, r.Repository.ID, r.RefFullName)
}

// GetCommitGraphsCount returns cached commit count for current view
Expand Down Expand Up @@ -782,7 +779,7 @@ func RepoRefByDefaultBranch() func(*Context) {
ctx.Repo.RefFullName = git.RefNameFromBranch(ctx.Repo.Repository.DefaultBranch)
ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch
ctx.Repo.Commit, _ = ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.BranchName)
ctx.Repo.CommitsCount, _ = ctx.Repo.GetCommitsCount()
ctx.Repo.CommitsCount, _ = ctx.Repo.GetCommitsCount(ctx)
ctx.Data["RefFullName"] = ctx.Repo.RefFullName
ctx.Data["BranchName"] = ctx.Repo.BranchName
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
Expand Down Expand Up @@ -931,7 +928,7 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {

ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch() // only used by the branch selector dropdown: AllowCreateNewRef

ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount(ctx)
if err != nil {
ctx.ServerError("GetCommitsCount", err)
return
Expand Down
14 changes: 2 additions & 12 deletions services/mirror/mirror_pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (

repo_model "code.gitea.io/gitea/models/repo"
system_model "code.gitea.io/gitea/models/system"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
giturl "code.gitea.io/gitea/modules/git/url"
"code.gitea.io/gitea/modules/gitrepo"
Expand Down Expand Up @@ -411,17 +410,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo)
}

log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo)
branches, _, err := gitrepo.GetBranchesByPath(ctx, m.Repo, 0, 0)
if err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to GetBranches: %v", m.Repo, err)
return nil, false
}

for _, branch := range branches {
cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true))
}

m.UpdatedUnix = timeutil.TimeStampNow()
return parseRemoteUpdateOutput(output, m.GetRemoteName()), true
}
Expand Down Expand Up @@ -616,6 +604,8 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re
hasDefault = hasDefault || name == defaultBranchName
hasMaster = hasMaster || name == "master"
hasMain = hasMain || name == "main"

// TODO: update branch commits count
}

if len(firstName) > 0 {
Expand Down
4 changes: 0 additions & 4 deletions services/pull/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/globallock"
"code.gitea.io/gitea/modules/httplib"
Expand Down Expand Up @@ -251,9 +250,6 @@ func Merge(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.U
notify_service.MergePullRequest(ctx, doer, pr)
}

// Reset cached commit count
cache.Remove(pr.Issue.Repo.GetCommitsCountCacheKey(pr.BaseBranch, true))

return handleCloseCrossReferences(ctx, pr, doer)
}

Expand Down
23 changes: 22 additions & 1 deletion services/repository/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,32 @@ package repository
import (
"context"

git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
)

func GetRefCommitsCount(ctx context.Context, repoID int64, refFullName git.RefName) (int64, error) {
// Get the commit count of the branch or the tag
switch {
case refFullName.IsBranch():
branch, err := git_model.GetBranch(ctx, repoID, refFullName.BranchName())
if err != nil {
return 0, err
}
return branch.CommitCount, nil
case refFullName.IsTag():
tag, err := repo_model.GetRelease(ctx, repoID, refFullName.TagName())
if err != nil {
return 0, err
}
return tag.NumCommits, nil
default:
return 0, nil
}
}

// CacheRef cachhe last commit information of the branch or the tag
func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, fullRefName git.RefName) error {
commit, err := gitRepo.GetCommit(fullRefName.String())
Expand All @@ -19,7 +40,7 @@ func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep
}

if gitRepo.LastCommitCache == nil {
commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(fullRefName.ShortName(), true), commit.CommitsCount)
commitsCount, err := GetRefCommitsCount(ctx, repo.ID, fullRefName)
if err != nil {
return err
}
Expand Down
15 changes: 8 additions & 7 deletions services/repository/commit.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import (
"context"
"fmt"

repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/util"
gitea_ctx "code.gitea.io/gitea/services/context"
)

type ContainedLinks struct { // TODO: better name?
Expand All @@ -23,32 +24,32 @@ type namedLink struct { // TODO: better name?
}

// LoadBranchesAndTags creates a new repository branch
func LoadBranchesAndTags(ctx context.Context, baseRepo *gitea_ctx.Repository, commitSHA string) (*ContainedLinks, error) {
containedTags, err := baseRepo.GitRepo.ListOccurrences(ctx, "tag", commitSHA)
func LoadBranchesAndTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, repoLink string, commitSHA string) (*ContainedLinks, error) {
containedTags, err := gitRepo.ListOccurrences(ctx, "tag", commitSHA)
if err != nil {
return nil, fmt.Errorf("encountered a problem while querying %s: %w", "tags", err)
}
containedBranches, err := baseRepo.GitRepo.ListOccurrences(ctx, "branch", commitSHA)
containedBranches, err := gitRepo.ListOccurrences(ctx, "branch", commitSHA)
if err != nil {
return nil, fmt.Errorf("encountered a problem while querying %s: %w", "branches", err)
}

result := &ContainedLinks{
DefaultBranch: baseRepo.Repository.DefaultBranch,
DefaultBranch: repo.DefaultBranch,
Branches: make([]*namedLink, 0, len(containedBranches)),
Tags: make([]*namedLink, 0, len(containedTags)),
}
for _, tag := range containedTags {
// TODO: Use a common method to get the link to a branch/tag instead of hard-coding it here
result.Tags = append(result.Tags, &namedLink{
Name: tag,
WebLink: fmt.Sprintf("%s/src/tag/%s", baseRepo.RepoLink, util.PathEscapeSegments(tag)),
WebLink: fmt.Sprintf("%s/src/tag/%s", repoLink, util.PathEscapeSegments(tag)),
})
}
for _, branch := range containedBranches {
result.Branches = append(result.Branches, &namedLink{
Name: branch,
WebLink: fmt.Sprintf("%s/src/branch/%s", baseRepo.RepoLink, util.PathEscapeSegments(branch)),
WebLink: fmt.Sprintf("%s/src/branch/%s", repoLink, util.PathEscapeSegments(branch)),
})
}
return result, nil
Expand Down
23 changes: 14 additions & 9 deletions services/repository/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ import (
"time"

"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
Expand Down Expand Up @@ -296,7 +296,16 @@ func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *use
return l, nil
}

func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) {
func UpdateRepoBranchCommitsCount(ctx context.Context, repo *repo_model.Repository, branch string, newCommit *git.Commit, isForcePush bool) error {
// calculate the number of commits in the branch
commitsCount, err := newCommit.CommitsCount()
if err != nil {
return fmt.Errorf("newCommit.CommitsCount: %w", err)
}
return git_model.UpdateBranchCommitCount(ctx, repo.ID, branch, commitsCount)
}

func pushUpdateBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) {
l, err := newCommit.CommitsBeforeUntil(opts.OldCommitID)
if err != nil {
return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err)
Expand All @@ -320,13 +329,9 @@ func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *us
NewCommitID: opts.NewCommitID,
})

if isForcePush {
log.Trace("Push %s is a force push", opts.NewCommitID)

cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
} else {
// TODO: increment update the commit count cache but not remove
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
// calculate the number of commits in the branch
if err := UpdateRepoBranchCommitsCount(ctx, repo, branch, newCommit, isForcePush); err != nil {
log.Error("UpdateRepoBranchCommitsCount: %v", err)
}

return l, nil
Expand Down