Add push SSH LFS protocol support

cmd/serv.go

@@ -22,6 +22,7 @@ import (
 	"code.gitea.io/gitea/models/perm"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/lfstransfer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/pprof"
 	"code.gitea.io/gitea/modules/private"
@@ -36,7 +37,11 @@ import (
 )
 
 const (
-	lfsAuthenticateVerb = "git-lfs-authenticate"
+	verbUploadPack      = "git-upload-pack"
+	verbUploadArchive   = "git-upload-archive"
+	verbReceivePack     = "git-receive-pack"
+	verbLfsAuthenticate = "git-lfs-authenticate"
+	verbLfsTransfer     = "git-lfs-transfer"
 )
 
 // CmdServ represents the available serv sub-command.
@@ -73,11 +78,18 @@ func setup(ctx context.Context, debug bool) {
 }
 
 var (
-	allowedCommands = map[string]perm.AccessMode{
-		"git-upload-pack":    perm.AccessModeRead,
-		"git-upload-archive": perm.AccessModeRead,
-		"git-receive-pack":   perm.AccessModeWrite,
-		lfsAuthenticateVerb:  perm.AccessModeNone,
+	// anything not in map will return false (zero value)
+	// keep getAccessMode() in sync
+	allowedCommands = map[string]bool{
+		verbUploadPack:      true,
+		verbUploadArchive:   true,
+		verbReceivePack:     true,
+		verbLfsAuthenticate: true,
+		verbLfsTransfer:     true,
+	}
+	allowedCommandsLfs = map[string]bool{
+		verbLfsAuthenticate: true,
+		verbLfsTransfer:     true,
 	}
 	alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)
 )
@@ -124,6 +136,24 @@ func handleCliResponseExtra(extra private.ResponseExtra) error {
 	return nil
 }
 
+func getAccessMode(verb string, lfsVerb string) perm.AccessMode {
+	switch verb {
+	case verbUploadPack, verbUploadArchive:
+		return perm.AccessModeRead
+	case verbReceivePack:
+		return perm.AccessModeWrite
+	case verbLfsAuthenticate, verbLfsTransfer:
+		switch lfsVerb {
+		case "upload":
+			return perm.AccessModeWrite
+		case "download":
+			return perm.AccessModeRead
+		}
+	}
+	// should be unreachable
+	return perm.AccessModeNone
+}
+
 func runServ(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
@@ -193,17 +223,7 @@ func runServ(c *cli.Context) error {
 	if repoPath[0] == '/' {
 		repoPath = repoPath[1:]
 	}
-
 	var lfsVerb string
-	if verb == lfsAuthenticateVerb {
-		if !setting.LFS.StartServer {
-			return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
-		}
-
-		if len(words) > 2 {
-			lfsVerb = words[2]
-		}
-	}
 
 	rr := strings.SplitN(repoPath, "/", 2)
 	if len(rr) != 2 {
@@ -240,28 +260,33 @@ func runServ(c *cli.Context) error {
 		}()
 	}
 
-	requestedMode, has := allowedCommands[verb]
-	if !has {
+	if allowedCommands[verb] {
+		if allowedCommandsLfs[verb] {
+			if !setting.LFS.StartServer {
+				return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
+			}
+			if len(words) > 2 {
+				lfsVerb = words[2]
+			}
+		}
+	} else {
 		return fail(ctx, "Unknown git command", "Unknown git command %s", verb)
 	}
 
-	if verb == lfsAuthenticateVerb {
-		if lfsVerb == "upload" {
-			requestedMode = perm.AccessModeWrite
-		} else if lfsVerb == "download" {
-			requestedMode = perm.AccessModeRead
-		} else {
-			return fail(ctx, "Unknown LFS verb", "Unknown lfs verb %s", lfsVerb)
-		}
-	}
+	requestedMode := getAccessMode(verb, lfsVerb)
 
 	results, extra := private.ServCommand(ctx, keyID, username, reponame, requestedMode, verb, lfsVerb)
 	if extra.HasError() {
 		return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error)
 	}
 
+	// LFS SSH protocol
+	if verb == verbLfsTransfer {
+		return lfstransfer.Main(ctx, repoPath, lfsVerb)
+	}
+
 	// LFS token authentication
-	if verb == lfsAuthenticateVerb {
+	if verb == verbLfsAuthenticate {
 		url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName))
 
 		now := time.Now()
@@ -296,22 +321,22 @@ func runServ(c *cli.Context) error {
 		return nil
 	}
 
-	var gitcmd *exec.Cmd
 	gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin
 	gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack
+	gitExe := gitBinVerb
+	gitArgs := make([]string, 0, 3) // capacity to accommodate max args
 	if _, err := os.Stat(gitBinVerb); err != nil {
 		// if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git
 		// ps: Windows only has "git.exe" in the bin path, so Windows always uses this way
 		verbFields := strings.SplitN(verb, "-", 2)
 		if len(verbFields) == 2 {
 			// use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ...
-			gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath)
+			gitExe = git.GitExecutable
+			gitArgs = append(gitArgs, verbFields[1])
 		}
 	}
-	if gitcmd == nil {
-		// by default, use the verb (it has been checked above by allowedCommands)
-		gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath)
-	}
+	gitArgs = append(gitArgs, repoPath)
+	gitcmd := exec.CommandContext(ctx, gitExe, gitArgs...)
 
 	process.SetSysProcAttribute(gitcmd)
 	gitcmd.Dir = setting.RepoRootPath

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/ethantkoenig/rupture v1.0.1
 	github.com/felixge/fgprof v0.9.4
 	github.com/fsnotify/fsnotify v1.7.0
+	github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825
 	github.com/gliderlabs/ssh v0.3.7
 	github.com/go-ap/activitypub v0.0.0-20240408091739-ba76b44c2594
 	github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73

go.sum

@@ -276,6 +276,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
 github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825 h1:riQhgheTL7tMF4d5raz9t3+IzoR1i1wqxE1kZC6dY+U=
+github.com/git-lfs/pktline v0.0.0-20210330133718-06e9096e2825/go.mod h1:fenKRzpXDjNpsIBhuhUzvjCKlDjKam0boRAenTE0Q6A=
 github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
 github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=

modules/lfs/content_store.go

@@ -16,6 +16,8 @@ import (
 )
 
 var (
+	// ErrObjectNotInStore occurs if the OID is not in store
+	ErrObjectNotInStore = errors.New("content hash does not match OID")
 	// ErrHashMismatch occurs if the content has does not match OID
 	ErrHashMismatch = errors.New("content hash does not match OID")
 	// ErrSizeMismatch occurs if the content size does not match
@@ -89,6 +91,20 @@ func (s *ContentStore) Exists(pointer Pointer) (bool, error) {
 	return true, nil
 }
 
+// GetMeta takes a pointer with OID and returns a pointer with Size
+func (s *ContentStore) GetMeta(pointer Pointer) (Pointer, error) {
+	p := pointer.RelativePath()
+	fi, err := s.ObjectStorage.Stat(p)
+	if os.IsNotExist(err) {
+		return pointer, ErrObjectNotInStore
+	} else if err != nil {
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err)
+		return pointer, err
+	}
+	pointer.Size = fi.Size()
+	return pointer, nil
+}
+
 // Verify returns true if the object exists in the content store and size is correct.
 func (s *ContentStore) Verify(pointer Pointer) (bool, error) {
 	p := pointer.RelativePath()

modules/lfstransfer/backend/backend.go

@@ -0,0 +1,164 @@
+package backend
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/lfstransfer/transfer"
+)
+
+// Version is the git-lfs-transfer protocol version number.
+const Version = "1"
+
+// Capabilities is a list of Git LFS capabilities supported by this package.
+var Capabilities = []string{
+	"version=" + Version,
+	// "locking", // no support yet in gitea backend
+}
+
+// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
+type GiteaBackend struct {
+	ctx   context.Context
+	repo  *repo_model.Repository
+	store *lfs.ContentStore
+}
+
+var _ transfer.Backend = &GiteaBackend{}
+
+// Batch implements transfer.Backend
+func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, _ transfer.Args) ([]transfer.BatchItem, error) {
+	for i := range pointers {
+		pointers[i].Present = false
+		pointer := lfs.Pointer{Oid: pointers[i].Oid, Size: pointers[i].Size}
+		exists, err := g.store.Verify(pointer)
+		if err != nil || !exists {
+			continue
+		}
+		accessible, err := g.repoHasAccess(pointers[i].Oid)
+		if err != nil || !accessible {
+			continue
+		}
+		pointers[i].Present = true
+	}
+	return pointers, nil
+}
+
+// Download implements transfer.Backend. The returned reader must be closed by the
+// caller.
+func (g *GiteaBackend) Download(oid string, _ transfer.Args) (io.ReadCloser, int64, error) {
+	pointer := lfs.Pointer{Oid: oid}
+	pointer, err := g.store.GetMeta(pointer)
+	if err != nil {
+		return nil, 0, err
+	}
+	obj, err := g.store.Get(pointer)
+	if err != nil {
+		return nil, 0, err
+	}
+	accessible, err := g.repoHasAccess(oid)
+	if err != nil {
+		return nil, 0, err
+	}
+	if !accessible {
+		return nil, 0, fmt.Errorf("LFS Meta Object [%v] not accessible from repo: %v", oid, g.repo.RepoPath())
+	}
+	return obj, pointer.Size, nil
+}
+
+// StartUpload implements transfer.Backend.
+func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, _ transfer.Args) error {
+	if r == nil {
+		return fmt.Errorf("%w: received null data", transfer.ErrMissingData)
+	}
+	pointer := lfs.Pointer{Oid: oid, Size: size}
+	exists, err := g.store.Verify(pointer)
+	if err != nil {
+		return err
+	}
+	if exists {
+		accessible, err := g.repoHasAccess(oid)
+		if err != nil {
+			return err
+		}
+		if accessible {
+			// we already have this object in the store and metadata
+			return nil
+		}
+		// we have this object in the store but not accessible
+		// so verify hash and size, and add it to metadata
+		hash := sha256.New()
+		written, err := io.Copy(hash, r)
+		if err != nil {
+			return fmt.Errorf("error creating hash: %v", err)
+		}
+		if written != size {
+			return fmt.Errorf("uploaded object [%v] has unexpected size: %v expected != %v received", oid, size, written)
+		}
+		recvOid := hex.EncodeToString(hash.Sum(nil)) != oid
+		if recvOid {
+			return fmt.Errorf("uploaded object [%v] has hash mismatch: %v received", oid, recvOid)
+		}
+	} else {
+		err = g.store.Put(pointer, r)
+		if err != nil {
+			return err
+		}
+	}
+	_, err = git_model.NewLFSMetaObject(g.ctx, g.repo.ID, pointer)
+	if err != nil {
+		return fmt.Errorf("could not create LFS Meta Object: %v", err)
+	}
+	return nil
+}
+
+// Verify implements transfer.Backend.
+func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) {
+	pointer := lfs.Pointer{Oid: oid, Size: size}
+	exists, err := g.store.Verify(pointer)
+	if err != nil {
+		return transfer.NewStatus(transfer.StatusNotFound, err.Error()), err
+	}
+	if !exists {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), fmt.Errorf("LFS Meta Object [%v] does not exist", oid)
+	}
+	accessible, err := g.repoHasAccess(oid)
+	if err != nil {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), err
+	}
+	if !accessible {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), fmt.Errorf("LFS Meta Object [%v] not accessible from repo: %v", oid, g.repo.RepoPath())
+	}
+	return transfer.SuccessStatus(), nil
+}
+
+// LockBackend implements transfer.Backend.
+func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend {
+	// Gitea doesn't support the locking API
+	// this should never be called as we don't advertise the capability
+	return (transfer.LockBackend)(nil)
+}
+
+// repoHasAccess checks if the repo already has the object with OID stored
+func (g *GiteaBackend) repoHasAccess(oid string) (bool, error) {
+	// check if OID is in global LFS store
+	exists, err := g.store.Exists(lfs.Pointer{Oid: oid})
+	if err != nil || !exists {
+		return false, err
+	}
+	// check if OID is in repo LFS store
+	metaObj, err := git_model.GetLFSMetaObjectByOid(g.ctx, g.repo.ID, oid)
+	if err != nil || metaObj == nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func New(ctx context.Context, r *repo_model.Repository, s *lfs.ContentStore) transfer.Backend {
+	return &GiteaBackend{ctx: ctx, repo: r, store: s}
+}

modules/lfstransfer/logger.go

@@ -0,0 +1,18 @@
+package lfstransfer
+
+import (
+	"code.gitea.io/gitea/modules/lfstransfer/transfer"
+)
+
+// noop logger for passing into transfer
+type GiteaLogger struct{}
+
+// Log implements transfer.Logger
+func (g *GiteaLogger) Log(msg string, itms ...interface{}) {
+}
+
+var _ transfer.Logger = (*GiteaLogger)(nil)
+
+func newLogger() transfer.Logger {
+	return &GiteaLogger{}
+}