git-annex: make /media/ download annexed content (#20)

Previously, Gitea's LFS support allowed direct-downloads of LFS content,
via http://$HOSTNAME:$PORT/$USER/$REPO/media/branch/$BRANCH/$FILE
Expand that grace to git-annex too. Now /media should provide the
relevant *content* from the .git/annex/objects/ folder.

This adds tests too. And expands the tests to try symlink-based annexing,
since /media implicitly supports both that and pointer-file-based annexing.

routers/web/repo/download.go

@@ -9,6 +9,7 @@ import (
  "time"
 
  git_model "code.gitea.io/gitea/models/git"
+ "code.gitea.io/gitea/modules/annex"
  "code.gitea.io/gitea/modules/context"
  "code.gitea.io/gitea/modules/git"
  "code.gitea.io/gitea/modules/httpcache"
@@ -79,6 +80,26 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim
  }
  closed = true
 
+ // check for git-annex files
+ // (this code is weirdly redundant because I'm trying not to delete any lines in order to make merges easier)
+ isAnnexed, err := annex.IsAnnexed(blob)
+ if err != nil {
+ ctx.ServerError("annex.IsAnnexed", err)
+ return err
+ }
+ if isAnnexed {
+ content, err := annex.Content(blob)
+ if err != nil {
+ // XXX are there any other possible failure cases here?
+ // there are, there could be unrelated io errors; those should be ctx.ServerError()s
+ ctx.NotFound("annex.Content", err)
+ return err
+ }
+ defer content.Close()
+ common.ServeContentByReadSeeker(ctx.Base, ctx.Repo.TreePath, lastModified, content)
+ return nil
+ }
+
  return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified)
 }
 

tests/integration/git_annex_test.go

@@ -7,7 +7,9 @@ package integration
 import (
  "errors"
  "fmt"
+ "io"
  "math/rand"
+ "net/http"
  "net/url"
  "os"
  "path"
@@ -56,6 +58,63 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext,
  return nil
 }
 
+func TestGitAnnexMedia(t *testing.T) {
+ if !setting.Annex.Enabled {
+ t.Skip("Skipping since annex support is disabled.")
+ }
+
+ onGiteaRun(t, func(t *testing.T, u *url.URL) {
+ ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository)
+
+ // create a public repo
+ require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false))
+
+ // the filenames here correspond to specific cases defined in doInitAnnexRepository()
+ t.Run("AnnexSymlink", func(t *testing.T) {
+ defer tests.PrintCurrentTest(t)()
+ doAnnexMediaTest(t, ctx, "annexed.tiff")
+ })
+ t.Run("AnnexPointer", func(t *testing.T) {
+ defer tests.PrintCurrentTest(t)()
+ doAnnexMediaTest(t, ctx, "annexed.bin")
+ })
+ })
+}
+
+func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) {
+ // Make sure that downloading via /media on the website recognizes it should give the annexed content
+
+ // TODO:
+ // - [ ] roll this into TestGitAnnexPermissions to ensure that permission enforcement works correctly even on /media?
+
+ session := loginUser(t, ctx.Username) // logs in to the http:// site/API, storing a cookie;
+ // this is a different auth method than the git+ssh:// or git+http:// protocols TestGitAnnexPermissions uses!
+
+ // compute server-side path of the annexed file
+ remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath())
+ remoteObjectPath, err := contentLocation(remoteRepoPath, file)
+ require.NoError(t, err)
+
+ // download annexed file
+ localObjectPath := path.Join(t.TempDir(), file)
+ fd, err := os.OpenFile(localObjectPath, os.O_CREATE|os.O_WRONLY, 0o777)
+ defer fd.Close()
+ require.NoError(t, err)
+
+ mediaLink := path.Join("/", ctx.Username, ctx.Reponame, "/media/branch/master", file)
+ req := NewRequest(t, "GET", mediaLink)
+ resp := session.MakeRequest(t, req, http.StatusOK)
+
+ _, err = io.Copy(fd, resp.Body)
+ require.NoError(t, err)
+ fd.Close()
+
+ // verify the download
+ match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0)
+ require.NoError(t, err)
+ require.True(t, match, "Annexed files should be the same")
+}
+
 /*
 Test that permissions are enforced on git-annex-shell commands.
 
@@ -763,16 +822,16 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) {
  }
 
  // - method 1: 'git annex whereis'.
- // Demonstrates that git-annex understands the annexed file can be found in the remote annex.
- annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath})
+ // Demonstrates that git-annex understands annexed files can be found in the remote annex.
+ annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "annexed.bin").RunStdString(&git.RunOpts{Dir: repoPath})
  if err != nil {
- return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err)
+ return fmt.Errorf("Couldn't `git annex whereis`: %w", err)
  }
  // Note: this regex is unanchored because 'whereis' outputs multiple lines containing
  // headers and 1+ remotes and we just want to find one of them.
  match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis)
  if !match {
- return errors.New("'git annex whereis' should report large.bin is known to be in [origin]")
+ return errors.New("'git annex whereis' should report files are known to be in [origin]")
  }
 
  return nil
@@ -788,27 +847,56 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) {
  return err
  }
 
- // verify the file was downloaded
- localObjectPath, err := contentLocation(repoPath, "large.bin")
- if err != nil {
- return err
+ // verify the files downloaded
+
+ cmp := func(filename string) error {
+ localObjectPath, err := contentLocation(repoPath, filename)
+ if err != nil {
+ return err
+ }
+ // localObjectPath := path.Join(repoPath, filename) // or, just compare against the checked-out file
+
+ remoteObjectPath, err := contentLocation(remoteRepoPath, filename)
+ if err != nil {
+ return err
+ }
+
+ match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0)
+ if err != nil {
+ return err
+ }
+ if !match {
+ return errors.New("Annexed files should be the same")
+ }
+
+ return nil
  }
- // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file
 
- remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin")
+ // this is the annex-symlink file
+ stat, err := os.Lstat(path.Join(repoPath, "annexed.tiff"))
  if err != nil {
+ return fmt.Errorf("Lstat: %w", err)
+ }
+ if !((stat.Mode() & os.ModeSymlink) != 0) {
+ // this line is really just double-checking that the text fixture is set up correctly
+ return errors.New("*.tiff should be a symlink")
+ }
+ if err = cmp("annexed.tiff"); err != nil {
  return err
  }
 
- match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0)
+ // this is the annex-pointer file
+ stat, err = os.Lstat(path.Join(repoPath, "annexed.bin"))
  if err != nil {
- return err
+ return fmt.Errorf("Lstat: %w", err)
  }
- if !match {
- return errors.New("Annexed files should be the same")
+ if !((stat.Mode() & os.ModeSymlink) == 0) {
+ // this line is really just double-checking that the text fixture is set up correctly
+ return errors.New("*.bin should not be a symlink")
  }
+ err = cmp("annexed.bin")
 
- return nil
+ return err
 }
 
 func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) {
@@ -953,16 +1041,36 @@ func doInitAnnexRepository(repoPath string) error {
  return err
  }
 
- // add a file to the annex
- err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin"))
+ // add files to the annex, stored via annex symlinks
+ // // a binary file
+ err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.tiff"))
+ if err != nil {
+ return err
+ }
+
+ err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath})
+ if err != nil {
+ return err
+ }
+
+ // add files to the annex, stored via git-annex-smudge
+ // // a binary file
+ err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.bin"))
+ if err != nil {
+ return err
+ }
+
  if err != nil {
  return err
  }
+
  err = git.AddChanges(repoPath, false, ".")
  if err != nil {
  return err
  }
- err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})
+
+ // save everything
+ err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"})
  if err != nil {
  return err
  }