From 21ca0b4d84a45a7d25c1e114dec8c0aada2cfb51 Mon Sep 17 00:00:00 2001 From: Elias Bouassaba Date: Tue, 9 Jul 2024 13:55:16 +0200 Subject: [PATCH] wip(webdav): low memory usage, streaming, but slow upload --- webdav-go/client/api_client.go | 72 +++++++++++++++++++++------------ webdav-go/handler/method_put.go | 61 +++++++--------------------- 2 files changed, 62 insertions(+), 71 deletions(-) diff --git a/webdav-go/client/api_client.go b/webdav-go/client/api_client.go index 12e5ea154..7cf58fc6c 100644 --- a/webdav-go/client/api_client.go +++ b/webdav-go/client/api_client.go @@ -73,11 +73,11 @@ type Thumbnail struct { } type FileCreateOptions struct { - Type string `json:"type"` - WorkspaceID string `json:"workspaceId"` - ParentID string `json:"parentId,omitempty"` - Blob []byte `json:"blob,omitempty"` - Name string `json:"name,omitempty"` + Type string + WorkspaceID string + ParentID string + Reader io.Reader + Name string } func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) { @@ -90,8 +90,8 @@ func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) { if opts.Name != "" { params.Set("name", opts.Name) } - if opts.Type == FileTypeFile && opts.Blob != nil { - return cl.upload(fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), "POST", opts.Blob, opts.Name) + if opts.Type == FileTypeFile && opts.Reader != nil { + return cl.upload(fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), "POST", opts.Reader, opts.Name) } else if opts.Type == FileTypeFolder { req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), nil) if err != nil { @@ -123,29 +123,51 @@ func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) { } type FilePatchOptions struct { - ID string `json:"id"` - Blob []byte `json:"blob"` - Name string `json:"name"` + ID string + Reader io.Reader + Name string } func (cl *APIClient) PatchFile(opts FilePatchOptions) (*File, error) { - return cl.upload(fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, opts.ID), "PATCH", opts.Blob, opts.Name) + return cl.upload(fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, opts.ID), "PATCH", opts.Reader, opts.Name) } -func (cl *APIClient) upload(url, method string, blob []byte, name string) (*File, error) { - body := new(bytes.Buffer) - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("file", name) - if err != nil { - return nil, err - } - if _, err := part.Write(blob); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - req, err := http.NewRequest(method, url, body) +func (cl *APIClient) upload(url, method string, reader io.Reader, name string) (*File, error) { + pr, pw := io.Pipe() + writer := multipart.NewWriter(pw) + go func() { + defer func(pw *io.PipeWriter) { + if err := pw.Close(); err != nil { + infra.GetLogger().Error(err.Error()) + } + }(pw) + part, err := writer.CreateFormFile("file", name) + if err != nil { + if err := pw.CloseWithError(err); err != nil { + infra.GetLogger().Error(err.Error()) + return + } + infra.GetLogger().Error(err.Error()) + return + } + if _, err := io.Copy(part, reader); err != nil { + if err := pw.CloseWithError(err); err != nil { + infra.GetLogger().Error(err.Error()) + return + } + infra.GetLogger().Error(err.Error()) + return + } + if err := writer.Close(); err != nil { + if err := pw.CloseWithError(err); err != nil { + infra.GetLogger().Error(err.Error()) + return + } + infra.GetLogger().Error(err.Error()) + return + } + }() + req, err := http.NewRequest(method, url, pr) if err != nil { return nil, err } diff --git a/webdav-go/handler/method_put.go b/webdav-go/handler/method_put.go index 4dc76775a..7f9cdd7d4 100644 --- a/webdav-go/handler/method_put.go +++ b/webdav-go/handler/method_put.go @@ -2,12 +2,8 @@ package handler import ( "fmt" - "github.com/google/uuid" - "io" "net/http" - "os" "path" - "path/filepath" "voltaserve/client" "voltaserve/helper" "voltaserve/infra" @@ -37,35 +33,7 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { return } apiClient := client.NewAPIClient(token) - directoryPath := helper.DecodeURIComponent(helper.Dirname(r.URL.Path)) - directory, err := apiClient.GetFileByPath(directoryPath) - if err != nil { - infra.HandleError(err, w) - return - } - outputPath := filepath.Join(os.TempDir(), uuid.New().String()) - ws, err := os.Create(outputPath) - if err != nil { - infra.HandleError(err, w) - return - } - defer func(ws *os.File) { - err := ws.Close() - if err != nil { - infra.HandleError(err, w) - } - }(ws) - _, err = io.Copy(ws, r.Body) - if err != nil { - infra.HandleError(err, w) - return - } - err = ws.Close() - if err != nil { - infra.HandleError(err, w) - return - } - blob, err := os.ReadFile(outputPath) + directory, err := apiClient.GetFileByPath(helper.DecodeURIComponent(helper.Dirname(r.URL.Path))) if err != nil { infra.HandleError(err, w) return @@ -73,25 +41,26 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { existingFile, err := apiClient.GetFileByPath(r.URL.Path) if err == nil { if _, err = apiClient.PatchFile(client.FilePatchOptions{ - ID: existingFile.ID, - Blob: blob, - Name: name, + ID: existingFile.ID, + Reader: r.Body, + Name: name, }); err != nil { infra.HandleError(err, w) return } w.WriteHeader(http.StatusCreated) return - } - if _, err = apiClient.CreateFile(client.FileCreateOptions{ - Type: client.FileTypeFile, - WorkspaceID: directory.WorkspaceID, - ParentID: directory.ID, - Blob: blob, - Name: name, - }); err != nil { - infra.HandleError(err, w) - return + } else { + if _, err = apiClient.CreateFile(client.FileCreateOptions{ + Type: client.FileTypeFile, + WorkspaceID: directory.WorkspaceID, + ParentID: directory.ID, + Reader: r.Body, + Name: name, + }); err != nil { + infra.HandleError(err, w) + return + } } w.WriteHeader(http.StatusCreated) }