cornerstone + git-annex

.github/workflows/pull-db-tests.yml

@@ -46,10 +46,11 @@ jobs:
  - name: Add hosts to /etc/hosts
  run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts'
  - run: make deps-backend
+ - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
  - run: make backend
  env:
  TAGS: bindata
- - run: make test-pgsql-migration test-pgsql
+ - run: make test-pgsql-migration test-pgsql#TestGitAnnex
  timeout-minutes: 50
  env:
  TAGS: bindata gogit
@@ -69,10 +70,11 @@ jobs:
  go-version-file: go.mod
  check-latest: true
  - run: make deps-backend
+ - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
  - run: make backend
  env:
  TAGS: bindata gogit sqlite sqlite_unlock_notify
- - run: make test-sqlite-migration test-sqlite
+ - run: make test-sqlite-migration test-sqlite#TestGitAnnex
  timeout-minutes: 50
  env:
  TAGS: bindata gogit sqlite sqlite_unlock_notify
@@ -172,11 +174,12 @@ jobs:
  - name: Add hosts to /etc/hosts
  run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts'
  - run: make deps-backend
+ - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
  - run: make backend
  env:
  TAGS: bindata
  - name: run tests
- run: make test-mysql-migration integration-test-coverage
+ run: make test-mysql-migration test-mysql#TestGitAnnex
  env:
  TAGS: bindata
  RACE_ENABLED: true
@@ -205,10 +208,11 @@ jobs:
  - name: Add hosts to /etc/hosts
  run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts'
  - run: make deps-backend
+ - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
  - run: make backend
  env:
  TAGS: bindata
- - run: make test-mssql-migration test-mssql
+ - run: make test-mssql-migration test-mssql#TestGitAnnex
  timeout-minutes: 50
  env:
  TAGS: bindata

Makefile

@@ -8,7 +8,7 @@ self := $(location)
  @tmpdir=`mktemp --tmpdir -d` ; \
  echo Using temporary directory $$tmpdir for test repositories ; \
  USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \
- STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS
+ STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS
 
 else
 
@@ -114,8 +114,9 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G
 LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
 
 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
-GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
-
+# Only test code modified in the git-annex feature branch; upstream can handle testing the full suite.
+# This list was generated by `git diff --stat --name-only main.. -- '*.go' | xargs dirname | sort | uniq`
+GO_TEST_PACKAGES ?= code.gitea.io/gitea/modules/annex code.gitea.io/gitea/modules/base code.gitea.io/gitea/modules/git code.gitea.io/gitea/modules/private code.gitea.io/gitea/modules/setting code.gitea.io/gitea/modules/util code.gitea.io/gitea/routers/private code.gitea.io/gitea/routers/web code.gitea.io/gitea/services/auth
 FOMANTIC_WORK_DIR := web_src/fomantic
 
 WEBPACK_SOURCES := $(shell find web_src/js web_src/css -type f)

cmd/serv.go

@@ -37,6 +37,7 @@ import (
 
 const (
  lfsAuthenticateVerb = "git-lfs-authenticate"
+ gitAnnexShellVerb = "git-annex-shell"
 )
 
 // CmdServ represents the available serv sub-command.
@@ -89,6 +90,7 @@ var (
  "git-upload-archive": perm.AccessModeRead,
  "git-receive-pack": perm.AccessModeWrite,
  lfsAuthenticateVerb: perm.AccessModeNone,
+ gitAnnexShellVerb: perm.AccessModeNone, // annex permissions are enforced by GIT_ANNEX_SHELL_READONLY, rather than the Gitea API
  }
  alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)
 )
@@ -201,6 +203,7 @@ func runServ(c *cli.Context) error {
 
  verb := words[0]
  repoPath := words[1]
+
  if repoPath[0] == '/' {
  repoPath = repoPath[1:]
  }
@@ -216,9 +219,43 @@ func runServ(c *cli.Context) error {
  }
  }
 
+ if verb == gitAnnexShellVerb {
+ if !setting.Annex.Enabled {
+ return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled")
+ }
+
+ if len(words) < 3 {
+ return fail(ctx, "Too few arguments", "Too few arguments in cmd: %s", cmd)
+ }
+
+ // git-annex always puts the repo in words[2], unlike most other
+ // git subcommands; and it sometimes names repos like /~/, as if
+ // $HOME should get expanded while also being rooted. e.g.:
+ // git-annex-shell 'configlist' '/~/user/repo'
+ // git-annex-shell 'sendkey' '/user/repo 'key'
+ repoPath = words[2]
+ repoPath = strings.TrimPrefix(repoPath, "/")
+ repoPath = strings.TrimPrefix(repoPath, "~/")
+ }
+
  // LowerCase and trim the repoPath as that's how they are stored.
  repoPath = strings.ToLower(strings.TrimSpace(repoPath))
 
+ // prevent directory traversal attacks
+ repoPath = filepath.Clean("/" + repoPath)[1:]
+
+ // put the sanitized repoPath back into the argument list for later
+ if verb == gitAnnexShellVerb {
+ // git-annex-shell demands an absolute path
+ absRepoPath, err := filepath.Abs(filepath.Join(setting.RepoRootPath, repoPath))
+ if err != nil {
+ return fail(ctx, "Error locating repoPath", "%v", err)
+ }
+ words[2] = absRepoPath
+ } else {
+ words[1] = repoPath
+ }
+
  rr := strings.SplitN(repoPath, "/", 2)
  if len(rr) != 2 {
  return fail(ctx, "Invalid repository path", "Invalid repository path: %v", repoPath)
@@ -305,21 +342,46 @@ func runServ(c *cli.Context) error {
  return nil
  }
 
- var gitcmd *exec.Cmd
  gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin
  gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack
  if _, err := os.Stat(gitBinVerb); err != nil {
  // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git
  // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way
+ // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath,
+ // but '{gitBinPath}/git annex-shell' should be able to find them on $PATH.
  verbFields := strings.SplitN(verb, "-", 2)
  if len(verbFields) == 2 {
  // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ...
- gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath)
+ gitBinVerb = git.GitExecutable
+ words = append([]string{verbFields[1]}, words...)
  }
  }
- if gitcmd == nil {
- // by default, use the verb (it has been checked above by allowedCommands)
- gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath)
+
+ // by default, use the verb (it has been checked above by allowedCommands)
+ gitcmd := exec.CommandContext(ctx, gitBinVerb, words[1:]...)
+
+ if verb == gitAnnexShellVerb {
+ // This doesn't get its own isolated section like LFS does, because LFS
+ // is handled by internal Gitea routines, but git-annex has to be shelled out
+ // to like other git subcommands, so we need to build up gitcmd.
+
+ // TODO: does this work on Windows?
+ gitcmd.Env = append(gitcmd.Env,
+ // "If set, disallows running git-shell to handle unknown commands."
+ // - git-annex-shell(1)
+ "GIT_ANNEX_SHELL_LIMITED=True",
+ // "If set, git-annex-shell will refuse to run commands
+ // that do not operate on the specified directory."
+ // - git-annex-shell(1)
+ fmt.Sprintf("GIT_ANNEX_SHELL_DIRECTORY=%s", words[2]),
+ )
+ if results.UserMode < perm.AccessModeWrite {
+ // "If set, disallows any action that could modify the git-annex repository."
+ // - git-annex-shell(1)
+ // We set this when the backend API has told us that we don't have write permission to this repo.
+ log.Debug("Setting GIT_ANNEX_SHELL_READONLY=True")
+ gitcmd.Env = append(gitcmd.Env, "GIT_ANNEX_SHELL_READONLY=True")
+ }
  }
 
  process.SetSysProcAttribute(gitcmd)

cmd/web.go

@@ -311,6 +311,10 @@ func listen(m http.Handler, handleRedirector bool) error {
  log.Info("LFS server enabled")
  }
 
+ if setting.Annex.Enabled {
+ log.Info("git-annex enabled")
+ }
+
  var err error
  switch setting.Protocol {
  case setting.HTTP:

custom/conf/app.example.ini

@@ -2517,6 +2517,15 @@ LEVEL = Info
 ;; override the minio base path if storage type is minio
 ;MINIO_BASE_PATH = lfs/
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;[annex]
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; Whether git-annex is enabled; defaults to false
+;ENABLED = false
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; settings for packages, will override storage setting

modules/annex/annex.go

@@ -0,0 +1,154 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Unlike modules/lfs, which operates mainly on git.Blobs, this operates on git.TreeEntrys.
+// The motivation for this is that TreeEntrys have an easy pointer to the on-disk repo path,
+// while blobs do not (in fact, if building with TAGS=gogit, blobs might exist only in a mock
+// filesystem, living only in process RAM). We must have the on-disk path to do anything
+// useful with git-annex because all of its interesting data is on-disk under .git/annex/.
+
+package annex
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "strings"
+
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/util"
+)
+
+const (
+ // > The maximum size of a pointer file is 32 kb.
+ // - https://git-annex.branchable.com/internals/pointer_file/
+ // It's unclear if that's kilobytes or kibibytes; assuming kibibytes:
+ blobSizeCutoff = 32 * 1024
+)
+
+// ErrInvalidPointer occurs if the pointer's value doesn't parse
+var ErrInvalidPointer = errors.New("Not a git-annex pointer")
+
+// Gets the content of the blob as raw text, up to n bytes.
+// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit)
+func getBlobContent(b *git.Blob, n int) (string, error) {
+ dataRc, err := b.DataAsync()
+ if err != nil {
+ return "", err
+ }
+ defer dataRc.Close()
+ buf := make([]byte, n)
+ n, _ = util.ReadAtMost(dataRc, buf)
+ buf = buf[:n]
+ return string(buf), nil
+}
+
+func Pointer(blob *git.Blob) (string, error) {
+ // git-annex doesn't seem fully spec what its pointer are, but
+ // the fullest description is here:
+ // https://git-annex.branchable.com/internals/pointer_file/
+
+ // a pointer can be:
+ // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY'
+ // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY'
+ //
+ // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because
+ // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one.
+
+ if blob.Size() > blobSizeCutoff {
+ // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file.
+ // https://git-annex.branchable.com/internals/pointer_file/
+
+ // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too.
+ return "", ErrInvalidPointer
+ }
+
+ pointer, err := getBlobContent(blob, blobSizeCutoff)
+ if err != nil {
+ return "", fmt.Errorf("error reading %s: %w", blob.Name(), err)
+ }
+
+ // the spec says a pointer file can contain multiple lines each with a pointer in them
+ // but that makes no sense to me, so I'm just ignoring all but the first
+ lines := strings.Split(pointer, "\n")
+ if len(lines) < 1 {
+ return "", ErrInvalidPointer
+ }
+ pointer = lines[0]
+
+ // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it
+ if !strings.Contains(pointer, "/annex/") {
+ return "", ErrInvalidPointer
+ }
+
+ // extract $KEY
+ pointer = path.Base(strings.TrimSpace(pointer))
+
+ // ask git-annex's opinion on $KEY
+ // XXX: this is probably a bit slow, especially if this operation gets run often
+ // and examinekey is not that strict:
+ // - it doesn't enforce that the "BACKEND" tag is one it knows,
+ // - it doesn't enforce that the fields and their format fit the "BACKEND" tag
+ // so maybe this is a wasteful step
+ _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path})
+ if err != nil {
+ // TODO: make ErrInvalidPointer into a type capable of wrapping err
+ if strings.TrimSpace(examineStderr) == "git-annex: bad key" {
+ return "", ErrInvalidPointer
+ }
+ return "", err
+ }
+
+ return pointer, nil
+}
+
+// return the absolute path of the content pointed to by the annex pointer stored in the git object
+// errors if the content is not found in this repo
+func ContentLocation(blob *git.Blob) (string, error) {
+ pointer, err := Pointer(blob)
+ if err != nil {
+ return "", err
+ }
+
+ contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path})
+ if err != nil {
+ return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err)
+ }
+ contentLocation = strings.TrimSpace(contentLocation)
+ contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals
+ contentLocation = path.Join(blob.Repo().Path, contentLocation)
+
+ return contentLocation, nil
+}
+
+// returns a stream open to the annex content
+func Content(blob *git.Blob) (*os.File, error) {
+ contentLocation, err := ContentLocation(blob)
+ if err != nil {
+ return nil, err
+ }
+
+ return os.Open(contentLocation)
+}
+
+// whether the object appears to be a valid annex pointer
+// does *not* verify if the content is actually in this repo;
+// for that, use ContentLocation()
+func IsAnnexed(blob *git.Blob) (bool, error) {
+ if !setting.Annex.Enabled {
+ return false, nil
+ }
+
+ // Pointer() is written to only return well-formed pointers
+ // so the test is just to see if it errors
+ _, err := Pointer(blob)
+ if err != nil {
+ if errors.Is(err, ErrInvalidPointer) {
+ return false, nil
+ }
+ return false, err
+ }
+ return true, nil
+}

modules/base/tool.go

@@ -19,6 +19,7 @@ import (
  "unicode"
  "unicode/utf8"
 
+ "code.gitea.io/gitea/modules/annex"
  "code.gitea.io/gitea/modules/git"
  "code.gitea.io/gitea/modules/log"
  "code.gitea.io/gitea/modules/setting"
@@ -204,6 +205,12 @@ func IsLetter(ch rune) bool {
 func EntryIcon(entry *git.TreeEntry) string {
  switch {
  case entry.IsLink():
+ isAnnexed, _ := annex.IsAnnexed(entry.Blob())
+ if isAnnexed {
+ // git-annex files are sometimes stored as symlinks;
+ // short-circuit that so like LFS they are displayed as regular files
+ return "file"
+ }
  te, err := entry.FollowLink()
  if err != nil {
  log.Debug(err.Error())