Merge remote-tracking branch 'giteaoffical/main'

* giteaoffical/main:
  remove not needed (go-gitea#19128)
  Add warning to set SENDMAIL_ARGS to --  (go-gitea#19102)
  Do not send activation email if manual confirm is set (go-gitea#19119)
  Update tool dependencies (go-gitea#19120)
  Delete related notifications on issue deletion too (go-gitea#18953)
  nit fix (go-gitea#19116)
  Store the foreign ID of issues during migration  (go-gitea#18446)
  Remove italics for `due_date_not_set` (go-gitea#19113)

Makefile

@@ -28,14 +28,14 @@ XGO_VERSION := go-1.18.x
 MIN_GO_VERSION := 001017000
 MIN_NODE_VERSION := 012017000
 
-AIR_PACKAGE ?= github.com/cosmtrek/air@bedc18201271882c2be66d216d0e1a275b526ec4
-EDITORCONFIG_CHECKER_PACKAGE ?= github.com/editorconfig-checker/editorconfig-checker/cmd/editorconfig-checker@50adf46752da119dfef66e57be3ce2693ea4aa9c
-ERRCHECK_PACKAGE ?= github.com/kisielk/errcheck@8ddee489636a8311a376fc92e27a6a13c6658344
+AIR_PACKAGE ?= github.com/cosmtrek/air@v1.29.0
+EDITORCONFIG_CHECKER_PACKAGE ?= github.com/editorconfig-checker/editorconfig-checker/cmd/editorconfig-checker@2.4.0
+ERRCHECK_PACKAGE ?= github.com/kisielk/errcheck@v1.6.0
 GOFUMPT_PACKAGE ?= mvdan.cc/gofumpt@v0.3.0
 GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/cmd/golangci-lint@v1.44.2
 GXZ_PAGAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.10
 MISSPELL_PACKAGE ?= github.com/client9/misspell/cmd/misspell@v0.3.4
-SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@v0.27.0
+SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@v0.29.0
 XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest
 
 DOCKER_IMAGE ?= gitea/gitea

custom/conf/app.example.ini

@@ -1533,6 +1533,7 @@ PATH =
 ;SENDMAIL_PATH = sendmail
 ;;
 ;; Specify any extra sendmail arguments
+;; WARNING: if your sendmail program interprets options you should set this to "--" or terminate these args with "--"
 ;SENDMAIL_ARGS =
 ;;
 ;; Timeout for Sendmail

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -666,7 +666,7 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
    - Enabling dummy will ignore all settings except `ENABLED`, `SUBJECT_PREFIX` and `FROM`.
 - `SENDMAIL_PATH`: **sendmail**: The location of sendmail on the operating system (can be
    command or full path).
-- `SENDMAIL_ARGS`: **_empty_**: Specify any extra sendmail arguments.
+- `SENDMAIL_ARGS`: **_empty_**: Specify any extra sendmail arguments. (NOTE: you should be aware that email addresses can look like options - if your `sendmail` command takes options you must set the option terminator `--`)
 - `SENDMAIL_TIMEOUT`: **5m**: default timeout for sending email through sendmail
 - `SENDMAIL_CONVERT_CRLF`: **true**: Most versions of sendmail prefer LF line endings rather than CRLF line endings. Set this to false if your version of sendmail requires CRLF line endings.
 - `SEND_BUFFER_LEN`: **100**: Buffer length of mailing queue. **DEPRECATED** use `LENGTH` in `[queue.mailer]`

docs/content/doc/usage/email-setup.en-us.md

@@ -35,6 +35,7 @@ ENABLED       = true
 FROM          = gitea@mydomain.com
 MAILER_TYPE   = sendmail
 SENDMAIL_PATH = /usr/sbin/sendmail
+SENDMAIL_ARGS = "--" ; most "sendmail" programs take options, "--" will prevent an email address being interpreted as an option.
 ```
 
 ## Using SMTP

models/fixtures/foreign_reference.yml

@@ -0,0 +1 @@
+[] # empty

models/foreignreference/error.go

@@ -0,0 +1,43 @@
+// Copyright 2022 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreignreference
+
+import (
+	"fmt"
+)
+
+// ErrLocalIndexNotExist represents a "LocalIndexNotExist" kind of error.
+type ErrLocalIndexNotExist struct {
+	RepoID       int64
+	ForeignIndex int64
+	Type         string
+}
+
+// ErrLocalIndexNotExist checks if an error is a ErrLocalIndexNotExist.
+func IsErrLocalIndexNotExist(err error) bool {
+	_, ok := err.(ErrLocalIndexNotExist)
+	return ok
+}
+
+func (err ErrLocalIndexNotExist) Error() string {
+	return fmt.Sprintf("repository %d has no LocalIndex for ForeignIndex %d of type %s", err.RepoID, err.ForeignIndex, err.Type)
+}
+
+// ErrForeignIndexNotExist represents a "ForeignIndexNotExist" kind of error.
+type ErrForeignIndexNotExist struct {
+	RepoID     int64
+	LocalIndex int64
+	Type       string
+}
+
+// ErrForeignIndexNotExist checks if an error is a ErrForeignIndexNotExist.
+func IsErrForeignIndexNotExist(err error) bool {
+	_, ok := err.(ErrForeignIndexNotExist)
+	return ok
+}
+
+func (err ErrForeignIndexNotExist) Error() string {
+	return fmt.Sprintf("repository %d has no ForeignIndex for LocalIndex %d of type %s", err.RepoID, err.LocalIndex, err.Type)
+}

models/foreignreference/foreignreference.go

@@ -0,0 +1,32 @@
+// Copyright 2022 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreignreference
+
+import (
+	"code.gitea.io/gitea/models/db"
+)
+
+// Type* are valid values for the Type field of ForeignReference
+const (
+	TypeIssue         = "issue"
+	TypePullRequest   = "pull_request"
+	TypeComment       = "comment"
+	TypeReview        = "review"
+	TypeReviewComment = "review_comment"
+	TypeRelease       = "release"
+)
+
+// ForeignReference represents external references
+type ForeignReference struct {
+	// RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type)
+	RepoID       int64  `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" `
+	LocalIndex   int64  `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID.
+	ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"`
+	Type         string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"`
+}
+
+func init() {
+	db.RegisterModel(new(ForeignReference))
+}

models/issue.go

@@ -15,6 +15,7 @@ import (
 
 	admin_model "code.gitea.io/gitea/models/admin"
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/foreignreference"
 	"code.gitea.io/gitea/models/issues"
 	"code.gitea.io/gitea/models/perm"
 	repo_model "code.gitea.io/gitea/models/repo"
@@ -67,11 +68,12 @@ type Issue struct {
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
 	ClosedUnix  timeutil.TimeStamp `xorm:"INDEX"`
 
-	Attachments      []*repo_model.Attachment `xorm:"-"`
-	Comments         []*Comment               `xorm:"-"`
-	Reactions        ReactionList             `xorm:"-"`
-	TotalTrackedTime int64                    `xorm:"-"`
-	Assignees        []*user_model.User       `xorm:"-"`
+	Attachments      []*repo_model.Attachment           `xorm:"-"`
+	Comments         []*Comment                         `xorm:"-"`
+	Reactions        ReactionList                       `xorm:"-"`
+	TotalTrackedTime int64                              `xorm:"-"`
+	Assignees        []*user_model.User                 `xorm:"-"`
+	ForeignReference *foreignreference.ForeignReference `xorm:"-"`
 
 	// IsLocked limits commenting abilities to users on an issue
 	// with write access
@@ -271,6 +273,29 @@ func (issue *Issue) loadReactions(ctx context.Context) (err error) {
 	return nil
 }
 
+func (issue *Issue) loadForeignReference(ctx context.Context) (err error) {
+	if issue.ForeignReference != nil {
+		return nil
+	}
+	reference := &foreignreference.ForeignReference{
+		RepoID:     issue.RepoID,
+		LocalIndex: issue.Index,
+		Type:       foreignreference.TypeIssue,
+	}
+	has, err := db.GetEngine(ctx).Get(reference)
+	if err != nil {
+		return err
+	} else if !has {
+		return foreignreference.ErrForeignIndexNotExist{
+			RepoID:     issue.RepoID,
+			LocalIndex: issue.Index,
+			Type:       foreignreference.TypeIssue,
+		}
+	}
+	issue.ForeignReference = reference
+	return nil
+}
+
 func (issue *Issue) loadMilestone(e db.Engine) (err error) {
 	if (issue.Milestone == nil || issue.Milestone.ID != issue.MilestoneID) && issue.MilestoneID > 0 {
 		issue.Milestone, err = getMilestoneByRepoID(e, issue.RepoID, issue.MilestoneID)
@@ -332,6 +357,10 @@ func (issue *Issue) loadAttributes(ctx context.Context) (err error) {
 		}
 	}
 
+	if err = issue.loadForeignReference(ctx); err != nil && !foreignreference.IsErrForeignIndexNotExist(err) {
+		return err
+	}
+
 	return issue.loadReactions(ctx)
 }
 
@@ -1110,6 +1139,26 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) {
 	return issue, nil
 }
 
+// GetIssueByForeignIndex returns raw issue by foreign ID
+func GetIssueByForeignIndex(ctx context.Context, repoID, foreignIndex int64) (*Issue, error) {
+	reference := &foreignreference.ForeignReference{
+		RepoID:       repoID,
+		ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+		Type:         foreignreference.TypeIssue,
+	}
+	has, err := db.GetEngine(ctx).Get(reference)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, foreignreference.ErrLocalIndexNotExist{
+			RepoID:       repoID,
+			ForeignIndex: foreignIndex,
+			Type:         foreignreference.TypeIssue,
+		}
+	}
+	return GetIssueByIndex(repoID, reference.LocalIndex)
+}
+
 // GetIssueWithAttrsByIndex returns issue by index in a repository.
 func GetIssueWithAttrsByIndex(repoID, index int64) (*Issue, error) {
 	issue, err := GetIssueByIndex(repoID, index)
@@ -2075,6 +2124,7 @@ func deleteIssue(ctx context.Context, issue *Issue) error {
 		&IssueDependency{},
 		&IssueAssignees{},
 		&IssueUser{},
+		&Notification{},
 		&Reaction{},
 		&IssueWatch{},
 		&Stopwatch{},

models/issue_comment.go

@@ -1163,7 +1163,7 @@ func deleteComment(e db.Engine, comment *Comment) error {
 	}
 
 	if comment.Type == CommentTypeComment {
-		if _, err := e.Exec("UPDATE `issue` SET num_comments = num_comments - 1 WHERE id = ?", comment.IssueID); err != nil {
+		if _, err := e.ID(comment.IssueID).Decr("num_comments").Update(new(Issue)); err != nil {
 			return err
 		}
 	}

models/issue_test.go

@@ -8,11 +8,13 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/foreignreference"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -534,3 +536,35 @@ func TestCorrectIssueStats(t *testing.T) {
 	assert.NoError(t, err)
 	assert.EqualValues(t, issueStats.OpenCount, issueAmount)
 }
+
+func TestIssueForeignReference(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 4}).(*Issue)
+	assert.NotEqualValues(t, issue.Index, issue.ID) // make sure they are different to avoid false positive
+
+	// it is fine for an issue to not have a foreign reference
+	err := issue.LoadAttributes()
+	assert.NoError(t, err)
+	assert.Nil(t, issue.ForeignReference)
+
+	var foreignIndex int64 = 12345
+	_, err = GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex)
+	assert.True(t, foreignreference.IsErrLocalIndexNotExist(err))
+
+	_, err = db.GetEngine(db.DefaultContext).Insert(&foreignreference.ForeignReference{
+		LocalIndex:   issue.Index,
+		ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+		RepoID:       issue.RepoID,
+		Type:         foreignreference.TypeIssue,
+	})
+	assert.NoError(t, err)
+
+	err = issue.LoadAttributes()
+	assert.NoError(t, err)
+
+	assert.EqualValues(t, issue.ForeignReference.ForeignIndex, strconv.FormatInt(foreignIndex, 10))
+
+	found, err := GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex)
+	assert.NoError(t, err)
+	assert.EqualValues(t, found.Index, issue.Index)
+}

models/migrate.go

@@ -83,6 +83,13 @@ func insertIssue(ctx context.Context, issue *Issue) error {
 		}
 	}
 
+	if issue.ForeignReference != nil {
+		issue.ForeignReference.LocalIndex = issue.Index
+		if _, err := sess.Insert(issue.ForeignReference); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

models/migrate_test.go

@@ -5,8 +5,10 @@
 package models
 
 import (
+	"strconv"
 	"testing"
 
+	"code.gitea.io/gitea/models/foreignreference"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -45,6 +47,7 @@ func assertCreateIssues(t *testing.T, isPull bool) {
 		UserID: owner.ID,
 	}
 
+	foreignIndex := int64(12345)
 	title := "issuetitle1"
 	is := &Issue{
 		RepoID:      repo.ID,
@@ -58,11 +61,20 @@ func assertCreateIssues(t *testing.T, isPull bool) {
 		IsClosed:    true,
 		Labels:      []*Label{label},
 		Reactions:   []*Reaction{reaction},
+		ForeignReference: &foreignreference.ForeignReference{
+			ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+			RepoID:       repo.ID,
+			Type:         foreignreference.TypeIssue,
+		},
 	}
 	err := InsertIssues(is)
 	assert.NoError(t, err)
 
 	i := unittest.AssertExistsAndLoadBean(t, &Issue{Title: title}).(*Issue)
+	assert.Nil(t, i.ForeignReference)
+	err = i.LoadAttributes()
+	assert.NoError(t, err)
+	assert.EqualValues(t, strconv.FormatInt(foreignIndex, 10), i.ForeignReference.ForeignIndex)
 	unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID})
 }
 

models/migrations/migrations.go

@@ -373,6 +373,11 @@ var migrations = []Migration{
 	NewMigration("Increase WebAuthentication CredentialID size to 410 - NO-OPED", increaseCredentialIDTo410),
 	// v210 -> v211
 	NewMigration("v208 was completely broken - remigrate", remigrateU2FCredentials),
+
+	// Gitea 1.16.2 ends at v211
+
+	// v211 -> v212
+	NewMigration("Create ForeignReference table", createForeignReferenceTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v211.go

@@ -0,0 +1,26 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+func createForeignReferenceTable(x *xorm.Engine) error {
+	type ForeignReference struct {
+		// RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type)
+		RepoID       int64  `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" `
+		LocalIndex   int64  `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID.
+		ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"`
+		Type         string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"`
+	}
+
+	if err := x.Sync2(new(ForeignReference)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

modules/hostmatcher/hostmatcher.go

@@ -8,8 +8,6 @@ import (
 	"net"
 	"path/filepath"
 	"strings"
-
-	"code.gitea.io/gitea/modules/util"
 )
 
 // HostMatchList is used to check if a host or IP is in a list.
@@ -104,11 +102,11 @@ func (hl *HostMatchList) checkIP(ip net.IP) bool {
 	for _, builtin := range hl.builtins {
 		switch builtin {
 		case MatchBuiltinExternal:
-			if ip.IsGlobalUnicast() && !util.IsIPPrivate(ip) {
+			if ip.IsGlobalUnicast() && !ip.IsPrivate() {
 				return true
 			}
 		case MatchBuiltinPrivate:
-			if util.IsIPPrivate(ip) {
+			if ip.IsPrivate() {
 				return true
 			}
 		case MatchBuiltinLoopback: