wip(webdav): low memory usage, streaming, but slow upload

kouprlabs · Jul 9, 2024 · 21ca0b4 · 21ca0b4
1 parent a1223fa
commit 21ca0b4
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 71 deletions.
diff --git a/webdav-go/client/api_client.go b/webdav-go/client/api_client.go
@@ -73,11 +73,11 @@ type Thumbnail struct {
 }
 
 type FileCreateOptions struct {
-	Type        string `json:"type"`
-	WorkspaceID string `json:"workspaceId"`
-	ParentID    string `json:"parentId,omitempty"`
-	Blob        []byte `json:"blob,omitempty"`
-	Name        string `json:"name,omitempty"`
+	Type        string
+	WorkspaceID string
+	ParentID    string
+	Reader      io.Reader
+	Name        string
 }
 
 func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) {
@@ -90,8 +90,8 @@ func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) {
 	if opts.Name != "" {
 		params.Set("name", opts.Name)
 	}
-	if opts.Type == FileTypeFile && opts.Blob != nil {
-		return cl.upload(fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), "POST", opts.Blob, opts.Name)
+	if opts.Type == FileTypeFile && opts.Reader != nil {
+		return cl.upload(fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), "POST", opts.Reader, opts.Name)
 	} else if opts.Type == FileTypeFolder {
 		req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), nil)
 		if err != nil {
@@ -123,29 +123,51 @@ func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) {
 }
 
 type FilePatchOptions struct {
-	ID   string `json:"id"`
-	Blob []byte `json:"blob"`
-	Name string `json:"name"`
+	ID     string
+	Reader io.Reader
+	Name   string
 }
 
 func (cl *APIClient) PatchFile(opts FilePatchOptions) (*File, error) {
-	return cl.upload(fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, opts.ID), "PATCH", opts.Blob, opts.Name)
+	return cl.upload(fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, opts.ID), "PATCH", opts.Reader, opts.Name)
 }
 
-func (cl *APIClient) upload(url, method string, blob []byte, name string) (*File, error) {
-	body := new(bytes.Buffer)
-	writer := multipart.NewWriter(body)
-	part, err := writer.CreateFormFile("file", name)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := part.Write(blob); err != nil {
-		return nil, err
-	}
-	if err := writer.Close(); err != nil {
-		return nil, err
-	}
-	req, err := http.NewRequest(method, url, body)
+func (cl *APIClient) upload(url, method string, reader io.Reader, name string) (*File, error) {
+	pr, pw := io.Pipe()
+	writer := multipart.NewWriter(pw)
+	go func() {
+		defer func(pw *io.PipeWriter) {
+			if err := pw.Close(); err != nil {
+				infra.GetLogger().Error(err.Error())
+			}
+		}(pw)
+		part, err := writer.CreateFormFile("file", name)
+		if err != nil {
+			if err := pw.CloseWithError(err); err != nil {
+				infra.GetLogger().Error(err.Error())
+				return
+			}
+			infra.GetLogger().Error(err.Error())
+			return
+		}
+		if _, err := io.Copy(part, reader); err != nil {
+			if err := pw.CloseWithError(err); err != nil {
+				infra.GetLogger().Error(err.Error())
+				return
+			}
+			infra.GetLogger().Error(err.Error())
+			return
+		}
+		if err := writer.Close(); err != nil {
+			if err := pw.CloseWithError(err); err != nil {
+				infra.GetLogger().Error(err.Error())
+				return
+			}
+			infra.GetLogger().Error(err.Error())
+			return
+		}
+	}()
+	req, err := http.NewRequest(method, url, pr)
 	if err != nil {
 		return nil, err
 	}

diff --git a/webdav-go/handler/method_put.go b/webdav-go/handler/method_put.go
@@ -2,12 +2,8 @@ package handler
 
 import (
 	"fmt"
-	"github.com/google/uuid"
-	"io"
 	"net/http"
-	"os"
 	"path"
-	"path/filepath"
 	"voltaserve/client"
 	"voltaserve/helper"
 	"voltaserve/infra"
@@ -37,61 +33,34 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	apiClient := client.NewAPIClient(token)
-	directoryPath := helper.DecodeURIComponent(helper.Dirname(r.URL.Path))
-	directory, err := apiClient.GetFileByPath(directoryPath)
-	if err != nil {
-		infra.HandleError(err, w)
-		return
-	}
-	outputPath := filepath.Join(os.TempDir(), uuid.New().String())
-	ws, err := os.Create(outputPath)
-	if err != nil {
-		infra.HandleError(err, w)
-		return
-	}
-	defer func(ws *os.File) {
-		err := ws.Close()
-		if err != nil {
-			infra.HandleError(err, w)
-		}
-	}(ws)
-	_, err = io.Copy(ws, r.Body)
-	if err != nil {
-		infra.HandleError(err, w)
-		return
-	}
-	err = ws.Close()
-	if err != nil {
-		infra.HandleError(err, w)
-		return
-	}
-	blob, err := os.ReadFile(outputPath)
+	directory, err := apiClient.GetFileByPath(helper.DecodeURIComponent(helper.Dirname(r.URL.Path)))
 	if err != nil {
 		infra.HandleError(err, w)
 		return
 	}
 	existingFile, err := apiClient.GetFileByPath(r.URL.Path)
 	if err == nil {
 		if _, err = apiClient.PatchFile(client.FilePatchOptions{
-			ID:   existingFile.ID,
-			Blob: blob,
-			Name: name,
+			ID:     existingFile.ID,
+			Reader: r.Body,
+			Name:   name,
 		}); err != nil {
 			infra.HandleError(err, w)
 			return
 		}
 		w.WriteHeader(http.StatusCreated)
 		return
-	}
-	if _, err = apiClient.CreateFile(client.FileCreateOptions{
-		Type:        client.FileTypeFile,
-		WorkspaceID: directory.WorkspaceID,
-		ParentID:    directory.ID,
-		Blob:        blob,
-		Name:        name,
-	}); err != nil {
-		infra.HandleError(err, w)
-		return
+	} else {
+		if _, err = apiClient.CreateFile(client.FileCreateOptions{
+			Type:        client.FileTypeFile,
+			WorkspaceID: directory.WorkspaceID,
+			ParentID:    directory.ID,
+			Reader:      r.Body,
+			Name:        name,
+		}); err != nil {
+			infra.HandleError(err, w)
+			return
+		}
 	}
 	w.WriteHeader(http.StatusCreated)
 }