diff --git a/.drone.env b/.drone.env index d4411a352a..e7a8c87f3e 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=37491c15674ab02347cbf6c90124002e75afa339 +APITESTS_COMMITID=5120b86d92f69c04f999697f7b584379327d62cb APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/changelog/unreleased/tusd-checksums.md b/changelog/unreleased/tusd-checksums.md new file mode 100644 index 0000000000..0e64d3cdc7 --- /dev/null +++ b/changelog/unreleased/tusd-checksums.md @@ -0,0 +1,5 @@ +Enhancement: Tusd PATCH checksums + +Check checksums also on chunked uploads during PATCH requests + +https://github.com/cs3org/reva/pull/4807 diff --git a/internal/http/services/owncloud/ocdav/options.go b/internal/http/services/owncloud/ocdav/options.go index 0cfb0b9e1f..59b4995ca0 100644 --- a/internal/http/services/owncloud/ocdav/options.go +++ b/internal/http/services/owncloud/ocdav/options.go @@ -42,7 +42,7 @@ func (s *svc) handleOptions(w http.ResponseWriter, r *http.Request) { w.Header().Set(net.HeaderTusResumable, "1.0.0") // TODO(jfd): only for dirs? w.Header().Set(net.HeaderTusVersion, "1.0.0") w.Header().Set(net.HeaderTusExtension, "creation,creation-with-upload,checksum,expiration") - w.Header().Set(net.HeaderTusChecksumAlgorithm, "md5,sha1,crc32") + w.Header().Set(net.HeaderTusChecksumAlgorithm, "md5,sha1,adler32") } w.WriteHeader(http.StatusNoContent) } diff --git a/pkg/ctx/checksumctx.go b/pkg/ctx/checksumctx.go new file mode 100644 index 0000000000..bdb4b5b95e --- /dev/null +++ b/pkg/ctx/checksumctx.go @@ -0,0 +1,23 @@ +package ctx + +import "context" + +// ContextGetChecksum returns the checksum if set in the given context. +func ContextGetChecksum(ctx context.Context) (string, bool) { + u, ok := ctx.Value(checksumKey).(string) + return u, ok +} + +// ContextWithChecksum returns the checksum if set in the given context. Otherwise it panics. +func ContextMustGetChecksum(ctx context.Context) string { + u, ok := ctx.Value(checksumKey).(string) + if !ok { + panic("checksum not set in context") + } + return u +} + +// ContextSetChecksum returns a new context with the given checksum. +func ContextSetChecksum(ctx context.Context, checksum string) context.Context { + return context.WithValue(ctx, checksumKey, checksum) +} diff --git a/pkg/ctx/userctx.go b/pkg/ctx/userctx.go index bb0c807675..878de7364b 100644 --- a/pkg/ctx/userctx.go +++ b/pkg/ctx/userctx.go @@ -33,6 +33,7 @@ const ( lockIDKey scopeKey initiatorKey + checksumKey ) // ContextGetUser returns the user if set in the given context. diff --git a/pkg/rhttp/datatx/manager/tus/tus.go b/pkg/rhttp/datatx/manager/tus/tus.go index 829440ecaf..fc60e6f713 100644 --- a/pkg/rhttp/datatx/manager/tus/tus.go +++ b/pkg/rhttp/datatx/manager/tus/tus.go @@ -31,6 +31,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/cs3org/reva/v2/pkg/appctx" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rhttp/datatx" @@ -174,6 +175,10 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { }() // set etag, mtime and file id setHeaders(fs, w, r) + // set checksum + if v := r.Header.Get("Upload-Checksum"); v != "" { + r = r.WithContext(ctxpkg.ContextSetChecksum(r.Context(), v)) + } handler.PatchFile(w, r) case "DELETE": handler.DelFile(w, r) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 42a507e303..b67a21e74b 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -1354,10 +1354,6 @@ func enoughDiskSpace(path string, fileSize uint64) bool { // CalculateChecksums calculates the sha1, md5 and adler32 checksums of a file func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - _, subspan := tracer.Start(ctx, "os.Open") f, err := os.Open(path) subspan.End() @@ -1366,11 +1362,20 @@ func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, } defer f.Close() - r1 := io.TeeReader(f, sha1h) + return CalculateChecksumsFromReader(ctx, f) +} + +// CalculateChecksumsFromReader calculates the sha1, md5 and adler32 checksums of a io.Reader +func CalculateChecksumsFromReader(ctx context.Context, r io.Reader) (hash.Hash, hash.Hash, hash.Hash32, error) { + sha1h := sha1.New() + md5h := md5.New() + adler32h := adler32.New() + + r1 := io.TeeReader(r, sha1h) r2 := io.TeeReader(r1, md5h) - _, subspan = tracer.Start(ctx, "io.Copy") - _, err = io.Copy(adler32h, r2) + _, subspan := tracer.Start(ctx, "io.Copy") + _, err := io.Copy(adler32h, r2) subspan.End() if err != nil { return nil, nil, nil, err diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index 4a3bfc61c2..a35ea0563d 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -19,7 +19,9 @@ package upload import ( + "bytes" "context" + "encoding/base64" "encoding/hex" "fmt" "hash" @@ -57,6 +59,22 @@ var defaultFilePerm = os.FileMode(0664) func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { ctx, span := tracer.Start(session.Context(ctx), "WriteChunk") defer span.End() + + // calculate checksum here + if checksum, ok := ctxpkg.ContextGetChecksum(ctx); ok { + // we need to copy the contents into memory so we can write it to disk later + b := bytes.NewBuffer(nil) + sha1, md5, adler32, err := node.CalculateChecksumsFromReader(ctx, io.TeeReader(src, b)) + if err != nil { + return 0, err + } + + if err := verifyChecksum(checksum, sha1, md5, adler32); err != nil { + return 0, err + } + src = b + } + _, subspan := tracer.Start(ctx, "os.OpenFile") file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) subspan.End() @@ -65,10 +83,6 @@ func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io } defer file.Close() - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... _, subspan = tracer.Start(ctx, "io.Copy") n, err := io.Copy(file, src) subspan.End() @@ -116,21 +130,7 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error { // compare if they match the sent checksum // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads if session.info.MetaData["checksum"] != "" { - var err error - parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = checkHash(parts[1], sha1h) - case "md5": - err = checkHash(parts[1], md5h) - case "adler32": - err = checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } + err := verifyChecksum(session.info.MetaData["checksum"], sha1h, md5h, adler32h) if err != nil { session.store.Cleanup(ctx, session, true, false, false) return err @@ -266,8 +266,12 @@ func (session *OcisSession) Finalize() (err error) { func checkHash(expected string, h hash.Hash) error { hash := hex.EncodeToString(h.Sum(nil)) - if expected != hash { - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, hash)) + raw, err := base64.StdEncoding.DecodeString(expected) + if err != nil { + return err + } + if string(raw) != hash { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %s", raw, hash)) } return nil } @@ -376,3 +380,20 @@ func joinurl(paths ...string) string { return s.String() } + +func verifyChecksum(checksum string, sha1h hash.Hash, md5h hash.Hash, adler32h hash.Hash32) error { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch strings.ToLower(parts[0]) { + case "sha1": + return checkHash(parts[1], sha1h) + case "md5": + return checkHash(parts[1], md5h) + case "adler32": + return checkHash(parts[1], adler32h) + default: + return errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } +}