Support downloading from "dumb http" with git-annex.

modules/git/command.go

@@ -292,12 +292,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS
 }
 
 // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests
+// It also re-enables git-credential(1), which is used to test git-annex's HTTP support
 func AllowLFSFiltersArgs() []string {
 	// Now here we should explicitly allow lfs filters to run
 	filteredLFSGlobalArgs := make([]string, len(globalCommandArgs))
 	j := 0
 	for _, arg := range globalCommandArgs {
-		if strings.Contains(arg, "lfs") {
+		if strings.Contains(arg, "lfs") || strings.Contains(arg, "credential") {
 			j--
 		} else {
 			filteredLFSGlobalArgs[j] = arg

routers/web/repo/http.go

@@ -611,3 +611,38 @@ func GetIdxFile(ctx *context.Context) {
 		h.sendFile("application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx")
 	}
 }
+
+// GetAnnexObject implements git-annex dumb HTTP
+func GetAnnexObject(ctx *context.Context) {
+	//if !setting.Annex.Enabled { // TODO
+	if false {
+		ctx.PlainText(http.StatusNotFound, "Not found")
+		return
+	}
+	h := httpBase(ctx)
+	if h != nil {
+		// git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key}
+		// where key is a string containing the size and (usually SHA256) checksum of the file,
+		// and hash1+hash2 are the first few bits of the md5sum of key itself.
+		// ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory)
+		// ref: https://git-annex.branchable.com/internals/hashing/
+
+		// keyDir should = key, but we don't enforce that
+		object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key"))
+
+		// Sanitize the input against directory traversals.
+		//
+		// This works because, if a path starts rooted,
+		// path.Clean() will remove all excess '..'. So
+		// this pretends the path is rooted ("/"), then
+		// path.Join() calls path.Clean() internally,
+		// then this unroots the path it ([1:]) (and 
+		//
+		// The router code also disallows "..", so this should be)
+		// redundant, but it's defensive to have it here.
+		object = path.Join("/", object)[1:]
+
+		h.setHeaderCacheForever()
+		h.sendFile("application/octet-stream", "annex/objects/"+object)
+	}
+}

routers/web/web.go

@@ -282,6 +282,13 @@ func RegisterRoutes(m *web.Route) {
 		}
 	}
 
+	annexEnabled := func(ctx *context.Context) {
+		if !setting.Annex.Enabled {
+			ctx.Error(http.StatusNotFound)
+			return
+		}
+	}
+
 	federationEnabled := func(ctx *context.Context) {
 		if !setting.Federation.Enabled {
 			ctx.Error(http.StatusNotFound)
@@ -1250,6 +1257,12 @@ func RegisterRoutes(m *web.Route) {
 				})
 			}, ignSignInAndCsrf, lfsServerEnabled)
 
+			m.Group("", func() {
+				// for git-annex
+				m.GetOptions("/config", repo.GetTextFile("config"))  // needed by clients reading annex.uuid during `git annex initremote`
+				m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject)
+			}, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb())
+
 			m.Group("", func() {
 				m.PostOptions("/git-upload-pack", repo.ServiceUploadPack)
 				m.PostOptions("/git-receive-pack", repo.ServiceReceivePack)

services/auth/auth.go

@@ -51,6 +51,20 @@ func isGitRawReleaseOrLFSPath(req *http.Request) bool {
 	return false
 }
 
+var (
+	annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`)
+)
+
+func isAnnexPath(req *http.Request) bool {
+	//if setting.Annex.Enabled { // TODO
+	if true {
+		// "/config" is git's config, not specifically git-annex's; but the only current
+		// user of it is when git-annex downloads the annex.uuid during 'git annex init'.
+		return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path)
+	}
+	return false
+}
+
 // handleSignIn clears existing session variables and stores new ones for the specified user object
 func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) {
 	// We need to regenerate the session...

services/auth/basic.go

@@ -42,8 +42,8 @@ func (b *Basic) Name() string {
 // name/token on successful validation.
 // Returns nil if header is empty or validation fails.
 func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) *user_model.User {
-	// Basic authentication should only fire on API, Download or on Git or LFSPaths
-	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) {
+	// Basic authentication should only fire on API, Download or on Git, LFSPaths or Git-Annex paths
+	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) && !isAnnexPath(req) {
 		return nil
 	}