diff --git a/api/model/snapshot_model.go b/api/model/snapshot_model.go index 4d318d4aa..6de1afd17 100644 --- a/api/model/snapshot_model.go +++ b/api/model/snapshot_model.go @@ -67,3 +67,11 @@ type ImageProps struct { Width int `json:"width"` Height int `json:"height"` } + +type S3Reference struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Size int64 `json:"size"` + SnapshotID string `json:"snapshotId"` + ContentType string `json:"contentType"` +} diff --git a/api/router/file_router.go b/api/router/file_router.go index 7cebb6dc2..7fb3b1b99 100644 --- a/api/router/file_router.go +++ b/api/router/file_router.go @@ -84,6 +84,8 @@ func (r *FileRouter) AppendNonJWTRoutes(g fiber.Router) { g.Get("/:id/original:ext", r.DownloadOriginal) g.Get("/:id/preview:ext", r.DownloadPreview) g.Get("/:id/thumbnail:ext", r.DownloadThumbnail) + g.Post("/create_from_s3", r.CreateFromS3) + g.Patch("/:id/patch_from_s3", r.PatchFromS3) } // Create godoc @@ -158,7 +160,7 @@ func (r *FileRouter) Create(c *fiber.Ctx) error { } } }(tmpPath) - file, err = r.fileSvc.Store(file.ID, tmpPath, userID) + file, err = r.fileSvc.Store(file.ID, service.StoreOptions{Path: &tmpPath}, userID) if err != nil { return err } @@ -225,7 +227,7 @@ func (r *FileRouter) Patch(c *fiber.Ctx) error { } } }(tmpPath) - file, err = r.fileSvc.Store(file.ID, tmpPath, userID) + file, err = r.fileSvc.Store(file.ID, service.StoreOptions{Path: &tmpPath}, userID) if err != nil { return err } @@ -376,11 +378,11 @@ func (r *FileRouter) List(c *fiber.Ctx) error { SortOrder: sortOrder, } if query != "" { - bytes, err := base64.StdEncoding.DecodeString(query + strings.Repeat("=", (4-len(query)%4)%4)) + b, err := base64.StdEncoding.DecodeString(query + strings.Repeat("=", (4-len(query)%4)%4)) if err != nil { return errorpkg.NewInvalidQueryParamError("query") } - if err := json.Unmarshal(bytes, &opts.Query); err != nil { + if err := json.Unmarshal(b, &opts.Query); err != nil { return errorpkg.NewInvalidQueryParamError("query") } res, err = r.fileSvc.Search(id, opts, userID) @@ -916,6 +918,200 @@ func (r *FileRouter) DownloadThumbnail(c *fiber.Ctx) error { return c.Send(b) } +// CreateFromS3 godoc +// +// @Summary Create from S3 +// @Description Create from S3 +// @Tags Files +// @Id files_create_from_s3 +// @Accept x-www-form-urlencoded +// @Produce json +// @Param api_key query string true "API Key" +// @Param access_token query string true "Access Token" +// @Param workspace_id query string true "Workspace ID" +// @Param parent_id query string false "Parent ID" +// @Param name query string false "Name" +// @Param s3_key query string true "S3 Key" +// @Param s3_bucket query string true "S3 Bucket" +// @Param size query string true "Size" +// @Success 200 {object} service.File +// @Failure 404 {object} errorpkg.ErrorResponse +// @Failure 400 {object} errorpkg.ErrorResponse +// @Failure 500 {object} errorpkg.ErrorResponse +// @Router /files/create_from_s3 [post] +func (r *FileRouter) CreateFromS3(c *fiber.Ctx) error { + apiKey := c.Query("api_key") + if apiKey == "" { + return errorpkg.NewMissingQueryParamError("api_key") + } + if apiKey != r.config.Security.APIKey { + return errorpkg.NewInvalidAPIKeyError() + } + accessToken := c.Query("access_token") + if accessToken == "" { + return errorpkg.NewMissingQueryParamError("access_token") + } + userID, err := r.getUserIDFromAccessToken(accessToken) + if err != nil { + return c.SendStatus(http.StatusNotFound) + } + workspaceID := c.Query("workspace_id") + if workspaceID == "" { + return errorpkg.NewMissingQueryParamError("workspace_id") + } + parentID := c.Query("parent_id") + if parentID == "" { + workspace, err := r.workspaceSvc.Find(workspaceID, userID) + if err != nil { + return err + } + parentID = workspace.RootID + } + name := c.Query("name") + if name == "" { + return errorpkg.NewMissingQueryParamError("name") + } + s3Key := c.Query("s3_key") + if s3Key == "" { + return errorpkg.NewMissingQueryParamError("s3_key") + } + s3Bucket := c.Query("s3_bucket") + if s3Bucket == "" { + return errorpkg.NewMissingQueryParamError("s3_bucket") + } + snapshotID := c.Query("snapshot_id") + if snapshotID == "" { + return errorpkg.NewMissingQueryParamError("snapshot_id") + } + contentType := c.Query("content_type") + if contentType == "" { + return errorpkg.NewMissingQueryParamError("content_type") + } + var size int64 + if c.Query("size") == "" { + return errorpkg.NewMissingQueryParamError("size") + } + size, err = strconv.ParseInt(c.Query("size"), 10, 64) + if err != nil { + return err + } + ok, err := r.workspaceSvc.HasEnoughSpaceForByteSize(workspaceID, size) + if err != nil { + return err + } + if !*ok { + return errorpkg.NewStorageLimitExceededError() + } + file, err := r.fileSvc.Create(service.FileCreateOptions{ + Name: name, + Type: model.FileTypeFile, + ParentID: &parentID, + WorkspaceID: workspaceID, + }, userID) + if err != nil { + return err + } + file, err = r.fileSvc.Store(file.ID, service.StoreOptions{ + S3Reference: &model.S3Reference{ + Key: s3Key, + Bucket: s3Bucket, + SnapshotID: snapshotID, + Size: size, + ContentType: contentType, + }, + }, userID) + if err != nil { + return err + } + return c.Status(http.StatusCreated).JSON(file) +} + +// PatchFromS3 godoc +// +// @Summary Patch from S3 +// @Description Patch from S3 +// @Tags Files +// @Id files_patch_from_s3 +// @Accept x-www-form-urlencoded +// @Produce json +// @Param api_key query string true "API Key" +// @Param access_token query string true "Access Token" +// @Param s3_key query string true "S3 Key" +// @Param s3_bucket query string true "S3 Bucket" +// @Param size query string true "Size" +// @Param id path string true "ID" +// @Success 200 {object} service.File +// @Failure 404 {object} errorpkg.ErrorResponse +// @Failure 400 {object} errorpkg.ErrorResponse +// @Failure 500 {object} errorpkg.ErrorResponse +// @Router /files/{id}/patch_from_s3 [patch] +func (r *FileRouter) PatchFromS3(c *fiber.Ctx) error { + apiKey := c.Query("api_key") + if apiKey == "" { + return errorpkg.NewMissingQueryParamError("api_key") + } + if apiKey != r.config.Security.APIKey { + return errorpkg.NewInvalidAPIKeyError() + } + accessToken := c.Query("access_token") + if accessToken == "" { + return errorpkg.NewMissingQueryParamError("access_token") + } + userID, err := r.getUserIDFromAccessToken(accessToken) + if err != nil { + return c.SendStatus(http.StatusNotFound) + } + files, err := r.fileSvc.Find([]string{c.Params("id")}, userID) + if err != nil { + return err + } + file := files[0] + s3Key := c.Query("s3_key") + if s3Key == "" { + return errorpkg.NewMissingQueryParamError("s3_key") + } + s3Bucket := c.Query("s3_bucket") + if s3Bucket == "" { + return errorpkg.NewMissingQueryParamError("s3_bucket") + } + var size int64 + if c.Query("size") == "" { + return errorpkg.NewMissingQueryParamError("size") + } + size, err = strconv.ParseInt(c.Query("size"), 10, 64) + if err != nil { + return err + } + snapshotID := c.Query("snapshot_id") + if snapshotID == "" { + return errorpkg.NewMissingQueryParamError("snapshot_id") + } + contentType := c.Query("content_type") + if contentType == "" { + return errorpkg.NewMissingQueryParamError("content_type") + } + ok, err := r.workspaceSvc.HasEnoughSpaceForByteSize(file.WorkspaceID, size) + if err != nil { + return err + } + if !*ok { + return errorpkg.NewStorageLimitExceededError() + } + file, err = r.fileSvc.Store(file.ID, service.StoreOptions{ + S3Reference: &model.S3Reference{ + Key: s3Key, + Bucket: s3Bucket, + SnapshotID: snapshotID, + Size: size, + ContentType: contentType, + }, + }, userID) + if err != nil { + return err + } + return c.JSON(file) +} + func (r *FileRouter) getUserIDFromAccessToken(accessToken string) (string, error) { token, err := jwt.Parse(accessToken, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { diff --git a/api/service/file_service.go b/api/service/file_service.go index 3e2cce0c1..cd561249f 100644 --- a/api/service/file_service.go +++ b/api/service/file_service.go @@ -200,7 +200,12 @@ func (svc *FileService) validateParent(id string, userID string) error { return nil } -func (svc *FileService) Store(id string, path string, userID string) (*File, error) { +type StoreOptions struct { + S3Reference *model.S3Reference + Path *string +} + +func (svc *FileService) Store(id string, opts StoreOptions, userID string) (*File, error) { file, err := svc.fileRepo.Find(id) if err != nil { return nil, err @@ -213,7 +218,39 @@ func (svc *FileService) Store(id string, path string, userID string) (*File, err if err != nil { return nil, err } - snapshotID := helper.NewID() + var snapshotID string + var size int64 + var path string + var original model.S3Object + var bucket string + var contentType string + if opts.S3Reference == nil { + snapshotID = helper.NewID() + path = *opts.Path + stat, err := os.Stat(*opts.Path) + if err != nil { + return nil, err + } + size = stat.Size() + original = model.S3Object{ + Bucket: workspace.GetBucket(), + Key: snapshotID + "/original" + strings.ToLower(filepath.Ext(path)), + Size: helper.ToPtr(size), + } + bucket = workspace.GetBucket() + contentType = infra.DetectMimeFromPath(path) + } else { + snapshotID = opts.S3Reference.SnapshotID + path = opts.S3Reference.Key + size = opts.S3Reference.Size + original = model.S3Object{ + Bucket: opts.S3Reference.Bucket, + Key: opts.S3Reference.Key, + Size: helper.ToPtr(size), + } + bucket = opts.S3Reference.Bucket + contentType = opts.S3Reference.ContentType + } snapshot := repo.NewSnapshot() snapshot.SetID(snapshotID) snapshot.SetVersion(latestVersion + 1) @@ -227,18 +264,11 @@ func (svc *FileService) Store(id string, path string, userID string) (*File, err if err = svc.snapshotRepo.MapWithFile(snapshotID, id); err != nil { return nil, err } - stat, err := os.Stat(path) - if err != nil { - return nil, err - } - exceedsProcessingLimit := stat.Size() > helper.MegabyteToByte(svc.fileIdent.GetProcessingLimitMB(path)) - original := model.S3Object{ - Bucket: workspace.GetBucket(), - Key: snapshotID + "/original" + strings.ToLower(filepath.Ext(path)), - Size: helper.ToPtr(stat.Size()), - } - if err = svc.s3.PutFile(original.Key, path, infra.DetectMimeFromPath(path), workspace.GetBucket(), minio.PutObjectOptions{}); err != nil { - return nil, err + exceedsProcessingLimit := size > helper.MegabyteToByte(svc.fileIdent.GetProcessingLimitMB(path)) + if opts.S3Reference == nil { + if err = svc.s3.PutFile(original.Key, path, contentType, bucket, minio.PutObjectOptions{}); err != nil { + return nil, err + } } snapshot.SetOriginal(&original) if exceedsProcessingLimit { diff --git a/api/voltaserve-api b/api/voltaserve-api new file mode 100755 index 000000000..1e901c078 Binary files /dev/null and b/api/voltaserve-api differ diff --git a/webdav-go/.env b/webdav-go/.env index c5e676e29..d19a51ad0 100644 --- a/webdav-go/.env +++ b/webdav-go/.env @@ -1,3 +1,15 @@ PORT=8082 + +# Security +SECURITY_API_KEY="7znl9Zd8F!4lRZA43lEQb757mCy" + +# URLs IDP_URL="http://127.0.0.1:8081" -API_URL="http://127.0.0.1:8080" \ No newline at end of file +API_URL="http://127.0.0.1:8080" + +# S3 +S3_URL="127.0.0.1:9000" +S3_ACCESS_KEY="voltaserve" +S3_SECRET_KEY="voltaserve" +S3_REGION="us-east-1" +S3_SECURE=false \ No newline at end of file diff --git a/webdav-go/cache/workspace_cache.go b/webdav-go/cache/workspace_cache.go new file mode 100644 index 000000000..8023ca330 --- /dev/null +++ b/webdav-go/cache/workspace_cache.go @@ -0,0 +1,51 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package cache + +import ( + "encoding/json" + "voltaserve/infra" +) + +type WorkspaceCache struct { + redis *infra.RedisManager + keyPrefix string +} + +func NewWorkspaceCache() *WorkspaceCache { + return &WorkspaceCache{ + redis: infra.NewRedisManager(), + keyPrefix: "workspace:", + } +} + +type Workspace struct { + ID string `json:"id," gorm:"column:id;size:36"` + Name string `json:"name" gorm:"column:name;size:255"` + StorageCapacity int64 `json:"storageCapacity" gorm:"column:storage_capacity"` + RootID string `json:"rootId" gorm:"column:root_id;size:36"` + OrganizationID string `json:"organizationId" gorm:"column:organization_id;size:36"` + Bucket string `json:"bucket" gorm:"column:bucket;size:255"` + CreateTime string `json:"createTime" gorm:"column:create_time"` + UpdateTime *string `json:"updateTime,omitempty" gorm:"column:update_time"` +} + +func (c *WorkspaceCache) Get(id string) (*Workspace, error) { + value, err := c.redis.Get(c.keyPrefix + id) + if err != nil { + return nil, err + } + var res Workspace + if err = json.Unmarshal([]byte(value), &res); err != nil { + return nil, err + } + return &res, nil +} diff --git a/webdav-go/client/api_client.go b/webdav-go/client/api_client.go index 1f938e22e..86bf180a7 100644 --- a/webdav-go/client/api_client.go +++ b/webdav-go/client/api_client.go @@ -1,13 +1,11 @@ package client import ( - "bufio" "bytes" "encoding/json" "errors" "fmt" "io" - "mime/multipart" "net/http" "net/url" "os" @@ -73,126 +71,137 @@ type Thumbnail struct { Height int `json:"height"` } -type FileCreateOptions struct { +type FileCreateFolderOptions struct { Type string WorkspaceID string ParentID string - Reader io.Reader Name string } -func (cl *APIClient) CreateFile(opts FileCreateOptions) (*File, error) { +func (cl *APIClient) CreateFolder(opts FileCreateFolderOptions) (*File, error) { params := url.Values{} params.Set("type", opts.Type) params.Set("workspace_id", opts.WorkspaceID) if opts.ParentID != "" { params.Set("parent_id", opts.ParentID) } - if opts.Name != "" { - params.Set("name", opts.Name) + params.Set("name", opts.Name) + req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), nil) + if err != nil { + return nil, err } - if opts.Type == FileTypeFile && opts.Reader != nil { - return cl.upload(fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), "POST", opts.Reader, opts.Name) - } else if opts.Type == FileTypeFolder { - req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files?%s", cl.config.APIURL, params.Encode()), nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+cl.token.AccessToken) - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - infra.GetLogger().Error(err.Error()) - } - }(resp.Body) - body, err := cl.jsonResponseOrThrow(resp) + req.Header.Set("Authorization", "Bearer "+cl.token.AccessToken) + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func(Body io.ReadCloser) { + err := Body.Close() if err != nil { - return nil, err - } - var file File - if err = json.Unmarshal(body, &file); err != nil { - return nil, err + infra.GetLogger().Error(err.Error()) } - return &file, nil + }(resp.Body) + body, err := cl.jsonResponseOrThrow(resp) + if err != nil { + return nil, err + } + var file File + if err = json.Unmarshal(body, &file); err != nil { + return nil, err } - return nil, errors.New("invalid file type or missing blob") + return &file, nil } -type FilePatchOptions struct { - ID string - Reader io.Reader - Name string +type S3Reference struct { + Bucket string + Key string + SnapshotID string + Size int64 + ContentType string } -func (cl *APIClient) PatchFile(opts FilePatchOptions) (*File, error) { - return cl.upload(fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, opts.ID), "PATCH", opts.Reader, opts.Name) +type FileCreateFromS3Options struct { + Type string + WorkspaceID string + ParentID string + Name string + S3Reference S3Reference } -func (cl *APIClient) upload(url, method string, reader io.Reader, name string) (*File, error) { - pr, pw := io.Pipe() - writer := multipart.NewWriter(pw) - go func() { - defer func(pw *io.PipeWriter) { - if err := pw.Close(); err != nil { - infra.GetLogger().Error(err.Error()) - } - }(pw) - part, err := writer.CreateFormFile("file", name) - if err != nil { - if err := pw.CloseWithError(err); err != nil { - infra.GetLogger().Error(err.Error()) - return - } - infra.GetLogger().Error(err.Error()) - return - } - br := bufio.NewReaderSize(reader, 100*1024*1024) // 100 MB buffer - bw := bufio.NewWriterSize(part, 100*1024*1024) // 100 MB buffer - if _, err := io.Copy(bw, br); err != nil { - if err := pw.CloseWithError(err); err != nil { - infra.GetLogger().Error(err.Error()) - return - } - infra.GetLogger().Error(err.Error()) - return - } - if err := writer.Close(); err != nil { - if err := pw.CloseWithError(err); err != nil { - infra.GetLogger().Error(err.Error()) - return - } - infra.GetLogger().Error(err.Error()) - return - } - }() - req, err := http.NewRequest(method, url, pr) +func (cl *APIClient) CreateFileFromS3(opts FileCreateFromS3Options) (*File, error) { + body, err := json.Marshal(opts) if err != nil { return nil, err } - req.Header.Set("Authorization", "Bearer "+cl.token.AccessToken) - req.Header.Set("Content-Type", writer.FormDataContentType()) + req, err := http.NewRequest("POST", + fmt.Sprintf("%s/v2/files/create_from_s3?api_key=%s&access_token=%s&workspace_id=%s&name=%s&s3_key=%s&s3_bucket=%s&snapshot_id=%s&content_type=%s&size=%d", + cl.config.APIURL, + cl.config.Security.APIKey, + cl.token.AccessToken, + opts.WorkspaceID, + opts.Name, + opts.S3Reference.Key, + opts.S3Reference.Bucket, + opts.S3Reference.SnapshotID, + opts.S3Reference.ContentType, + opts.S3Reference.Size, + ), + bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") client := &http.Client{} - resp, err := client.Do(req) + res, err := client.Do(req) if err != nil { return nil, err } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - infra.GetLogger().Error(err.Error()) - } - }(resp.Body) - respBody, err := cl.jsonResponseOrThrow(resp) + defer res.Body.Close() + var file File + if err = json.Unmarshal(body, &file); err != nil { + return nil, err + } + return &file, nil +} + +type FilePatchFromS3Options struct { + ID string + Name string + S3Reference S3Reference +} + +func (cl *APIClient) PatchFileFromS3(opts FilePatchFromS3Options) (*File, error) { + body, err := json.Marshal(opts) if err != nil { return nil, err } + req, err := http.NewRequest("PATCH", + fmt.Sprintf("%s/v2/files/%s/patch_from_s3?api_key=%s&access_token=%s&name=%s&s3_key=%s&s3_bucket=%s&snapshot_id=%s&content_type=%s&size=%d", + cl.config.APIURL, + cl.config.Security.APIKey, + cl.token.AccessToken, + opts.ID, + opts.Name, + opts.S3Reference.Key, + opts.S3Reference.Bucket, + opts.S3Reference.SnapshotID, + opts.S3Reference.ContentType, + opts.S3Reference.Size, + ), + bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() var file File - if err = json.Unmarshal(respBody, &file); err != nil { + if err = json.Unmarshal(body, &file); err != nil { return nil, err } return &file, nil diff --git a/webdav-go/config/config.go b/webdav-go/config/config.go index 2594e1a01..966bc9b06 100644 --- a/webdav-go/config/config.go +++ b/webdav-go/config/config.go @@ -16,9 +16,30 @@ import ( ) type Config struct { - Port int - APIURL string - IdPURL string + Port int + APIURL string + IdPURL string + S3 S3Config + Redis RedisConfig + Security SecurityConfig +} + +type S3Config struct { + URL string + AccessKey string + SecretKey string + Region string + Secure bool +} + +type RedisConfig struct { + Address string + Password string + DB int +} + +type SecurityConfig struct { + APIKey string `json:"api_key"` } var config *Config @@ -33,6 +54,9 @@ func GetConfig() *Config { Port: port, } readURLs(config) + readS3(config) + readRedis(config) + readSecurity(config) } return config } @@ -41,3 +65,33 @@ func readURLs(config *Config) { config.APIURL = os.Getenv("API_URL") config.IdPURL = os.Getenv("IDP_URL") } + +func readS3(config *Config) { + config.S3.URL = os.Getenv("S3_URL") + config.S3.AccessKey = os.Getenv("S3_ACCESS_KEY") + config.S3.SecretKey = os.Getenv("S3_SECRET_KEY") + config.S3.Region = os.Getenv("S3_REGION") + if len(os.Getenv("S3_SECURE")) > 0 { + v, err := strconv.ParseBool(os.Getenv("S3_SECURE")) + if err != nil { + panic(err) + } + config.S3.Secure = v + } +} + +func readRedis(config *Config) { + config.Redis.Address = os.Getenv("REDIS_ADDRESS") + config.Redis.Password = os.Getenv("REDIS_PASSWORD") + if len(os.Getenv("REDIS_DB")) > 0 { + v, err := strconv.ParseInt(os.Getenv("REDIS_DB"), 10, 32) + if err != nil { + panic(err) + } + config.Redis.DB = int(v) + } +} + +func readSecurity(config *Config) { + config.Security.APIKey = os.Getenv("SECURITY_API_KEY") +} diff --git a/webdav-go/go.mod b/webdav-go/go.mod index e14d72975..e60039c5b 100644 --- a/webdav-go/go.mod +++ b/webdav-go/go.mod @@ -12,6 +12,22 @@ require ( ) require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.73 // indirect + github.com/redis/go-redis/v9 v9.5.3 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/stretchr/testify v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect ) diff --git a/webdav-go/go.sum b/webdav-go/go.sum index 52dadb21e..511dbfef7 100644 --- a/webdav-go/go.sum +++ b/webdav-go/go.sum @@ -1,11 +1,36 @@ +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= +github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= +github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU= +github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/speps/go-hashids/v2 v2.0.1 h1:ViWOEqWES/pdOSq+C1SLVa8/Tnsd52XC34RY7lt7m4g= github.com/speps/go-hashids/v2 v2.0.1/go.mod h1:47LKunwvDZki/uRVD6NImtyk712yFzIs3UF3KlHohGw= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -16,5 +41,14 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/webdav-go/handler/handler.go b/webdav-go/handler/handler.go index 8ab1db6f3..a3d354400 100644 --- a/webdav-go/handler/handler.go +++ b/webdav-go/handler/handler.go @@ -2,13 +2,21 @@ package handler import ( "net/http" + "voltaserve/cache" "voltaserve/client" + "voltaserve/infra" ) -type Handler struct{} +type Handler struct { + s3 *infra.S3Manager + workspaceCache *cache.WorkspaceCache +} func NewHandler() *Handler { - return &Handler{} + return &Handler{ + s3: infra.NewS3Manager(), + workspaceCache: cache.NewWorkspaceCache(), + } } func (h *Handler) Dispatch(w http.ResponseWriter, r *http.Request) { diff --git a/webdav-go/handler/method_mkcol.go b/webdav-go/handler/method_mkcol.go index 894c2427e..338dfd6c9 100644 --- a/webdav-go/handler/method_mkcol.go +++ b/webdav-go/handler/method_mkcol.go @@ -32,7 +32,7 @@ func (h *Handler) methodMkcol(w http.ResponseWriter, r *http.Request) { infra.HandleError(err, w) return } - if _, err = apiClient.CreateFile(client.FileCreateOptions{ + if _, err = apiClient.CreateFolder(client.FileCreateFolderOptions{ Type: client.FileTypeFolder, WorkspaceID: directory.WorkspaceID, ParentID: directory.ID, diff --git a/webdav-go/handler/method_put.go b/webdav-go/handler/method_put.go index 7f9cdd7d4..66435085c 100644 --- a/webdav-go/handler/method_put.go +++ b/webdav-go/handler/method_put.go @@ -2,8 +2,14 @@ package handler import ( "fmt" + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "io" "net/http" + "os" "path" + "path/filepath" + "strings" "voltaserve/client" "voltaserve/helper" "voltaserve/infra" @@ -38,12 +44,58 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { infra.HandleError(err, w) return } + outputPath := filepath.Join(os.TempDir(), uuid.New().String()) + ws, err := os.Create(outputPath) + if err != nil { + infra.HandleError(err, w) + return + } + defer func(ws *os.File) { + err := ws.Close() + if err != nil { + infra.HandleError(err, w) + } + }(ws) + _, err = io.Copy(ws, r.Body) + if err != nil { + infra.HandleError(err, w) + return + } + err = ws.Close() + if err != nil { + infra.HandleError(err, w) + return + } + workspaceID := helper.ExtractWorkspaceIDFromPath(r.URL.Path) + workspace, err := h.workspaceCache.Get(workspaceID) + if err != nil { + infra.HandleError(err, w) + return + } + snapshotID := helper.NewID() + key := snapshotID + "/original" + strings.ToLower(filepath.Ext(name)) + if err = h.s3.PutFile(key, outputPath, infra.DetectMimeFromPath(outputPath), workspace.Bucket, minio.PutObjectOptions{}); err != nil { + infra.HandleError(err, w) + return + } + stat, err := os.Stat(outputPath) + if err != nil { + infra.HandleError(err, w) + return + } + s3Reference := client.S3Reference{ + Bucket: workspace.Bucket, + Key: key, + SnapshotID: snapshotID, + Size: stat.Size(), + ContentType: infra.DetectMimeFromPath(outputPath), + } existingFile, err := apiClient.GetFileByPath(r.URL.Path) if err == nil { - if _, err = apiClient.PatchFile(client.FilePatchOptions{ - ID: existingFile.ID, - Reader: r.Body, - Name: name, + if _, err = apiClient.PatchFileFromS3(client.FilePatchFromS3Options{ + ID: existingFile.ID, + Name: name, + S3Reference: s3Reference, }); err != nil { infra.HandleError(err, w) return @@ -51,12 +103,12 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) return } else { - if _, err = apiClient.CreateFile(client.FileCreateOptions{ + if _, err = apiClient.CreateFileFromS3(client.FileCreateFromS3Options{ Type: client.FileTypeFile, WorkspaceID: directory.WorkspaceID, ParentID: directory.ID, - Reader: r.Body, Name: name, + S3Reference: s3Reference, }); err != nil { infra.HandleError(err, w) return diff --git a/webdav-go/helper/workspace.go b/webdav-go/helper/workspace.go new file mode 100644 index 000000000..8409dbbea --- /dev/null +++ b/webdav-go/helper/workspace.go @@ -0,0 +1,12 @@ +package helper + +import "strings" + +func ExtractWorkspaceIDFromPath(path string) string { + slashParts := strings.Split(strings.TrimPrefix(path, "/"), "/") + dashParts := strings.Split(slashParts[0], "-") + if len(dashParts) > 1 { + return dashParts[len(dashParts)-1] + } + return "" +} diff --git a/webdav-go/infra/mime.go b/webdav-go/infra/mime.go new file mode 100644 index 000000000..e58a2256c --- /dev/null +++ b/webdav-go/infra/mime.go @@ -0,0 +1,21 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package infra + +import "github.com/gabriel-vasile/mimetype" + +func DetectMimeFromPath(path string) string { + mime, err := mimetype.DetectFile(path) + if err != nil { + return "application/octet-stream" + } + return mime.String() +} diff --git a/webdav-go/infra/redis.go b/webdav-go/infra/redis.go new file mode 100644 index 000000000..e60469d16 --- /dev/null +++ b/webdav-go/infra/redis.go @@ -0,0 +1,116 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package infra + +import ( + "context" + "github.com/redis/go-redis/v9" + "strings" + "voltaserve/config" +) + +type RedisManager struct { + config config.RedisConfig + client *redis.Client + clusterClient *redis.ClusterClient +} + +func NewRedisManager() *RedisManager { + return &RedisManager{ + config: config.GetConfig().Redis, + } +} + +func (mgr *RedisManager) Set(key string, value interface{}) error { + if err := mgr.Connect(); err != nil { + return err + } + if mgr.clusterClient != nil { + if _, err := mgr.clusterClient.Set(context.Background(), key, value, 0).Result(); err != nil { + return err + } + } else { + if _, err := mgr.client.Set(context.Background(), key, value, 0).Result(); err != nil { + return err + } + } + return nil +} + +func (mgr *RedisManager) Get(key string) (string, error) { + if err := mgr.Connect(); err != nil { + return "", err + } + if mgr.clusterClient != nil { + value, err := mgr.clusterClient.Get(context.Background(), key).Result() + if err != nil { + return "", err + } + return value, nil + } else { + value, err := mgr.client.Get(context.Background(), key).Result() + if err != nil { + return "", err + } + return value, nil + } +} + +func (mgr *RedisManager) Delete(key string) error { + if err := mgr.Connect(); err != nil { + return err + } + if mgr.clusterClient != nil { + if _, err := mgr.clusterClient.Del(context.Background(), key).Result(); err != nil { + return err + } + } else { + if _, err := mgr.client.Del(context.Background(), key).Result(); err != nil { + return err + } + } + return nil +} + +func (mgr *RedisManager) Close() error { + if mgr.client != nil { + if err := mgr.client.Close(); err != nil { + return err + } + } + return nil +} + +func (mgr *RedisManager) Connect() error { + if mgr.client != nil || mgr.clusterClient != nil { + return nil + } + addresses := strings.Split(mgr.config.Address, ";") + if len(addresses) > 1 { + mgr.clusterClient = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: addresses, + Password: mgr.config.Password, + }) + if err := mgr.clusterClient.Ping(context.Background()).Err(); err != nil { + return err + } + } else { + mgr.client = redis.NewClient(&redis.Options{ + Addr: mgr.config.Address, + Password: mgr.config.Password, + DB: mgr.config.DB, + }) + if err := mgr.client.Ping(context.Background()).Err(); err != nil { + return err + } + } + return nil +} diff --git a/webdav-go/infra/s3.go b/webdav-go/infra/s3.go new file mode 100644 index 000000000..e22f3792f --- /dev/null +++ b/webdav-go/infra/s3.go @@ -0,0 +1,195 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package infra + +import ( + "bytes" + "context" + "errors" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "io" + "strings" + "voltaserve/config" +) + +type S3Manager struct { + config config.S3Config + client *minio.Client +} + +func NewS3Manager() *S3Manager { + mgr := new(S3Manager) + mgr.config = config.GetConfig().S3 + return mgr +} + +func (mgr *S3Manager) StatObject(objectName string, bucketName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + if mgr.client == nil { + if err := mgr.Connect(); err != nil { + return minio.ObjectInfo{}, err + } + } + return mgr.client.StatObject(context.Background(), bucketName, objectName, opts) +} + +func (mgr *S3Manager) GetFile(objectName string, filePath string, bucketName string, opts minio.GetObjectOptions) error { + if mgr.client == nil { + if err := mgr.Connect(); err != nil { + return err + } + } + if err := mgr.client.FGetObject(context.Background(), bucketName, objectName, filePath, opts); err != nil { + return err + } + return nil +} + +func (mgr *S3Manager) PutFile(objectName string, filePath string, contentType string, bucketName string, opts minio.PutObjectOptions) error { + if err := mgr.Connect(); err != nil { + return err + } + if contentType == "" { + contentType = "application/octet-stream" + } + opts.ContentType = contentType + if _, err := mgr.client.FPutObject(context.Background(), bucketName, objectName, filePath, opts); err != nil { + return err + } + return nil +} + +func (mgr *S3Manager) PutText(objectName string, text string, contentType string, bucketName string, opts minio.PutObjectOptions) error { + if contentType != "" && contentType != "text/plain" && contentType != "application/json" { + return errors.New("invalid content type '" + contentType + "'") + } + if contentType == "" { + contentType = "text/plain" + } + if err := mgr.Connect(); err != nil { + return err + } + opts.ContentType = contentType + if _, err := mgr.client.PutObject(context.Background(), bucketName, objectName, strings.NewReader(text), int64(len(text)), opts); err != nil { + return err + } + return nil +} + +func (mgr *S3Manager) GetObject(objectName string, bucketName string, opts minio.GetObjectOptions) (*bytes.Buffer, *int64, error) { + if err := mgr.Connect(); err != nil { + return nil, nil, err + } + reader, err := mgr.client.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + return nil, nil, err + } + var buf bytes.Buffer + written, err := io.Copy(io.Writer(&buf), reader) + if err != nil { + return nil, nil, nil + } + return &buf, &written, nil +} + +func (mgr *S3Manager) GetObjectWithBuffer(objectName string, bucketName string, buf *bytes.Buffer, opts minio.GetObjectOptions) (*int64, error) { + if err := mgr.Connect(); err != nil { + return nil, err + } + reader, err := mgr.client.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + return nil, err + } + written, err := io.Copy(io.Writer(buf), reader) + if err != nil { + return nil, nil + } + return &written, nil +} + +func (mgr *S3Manager) GetText(objectName string, bucketName string, opts minio.GetObjectOptions) (string, error) { + if err := mgr.Connect(); err != nil { + return "", err + } + reader, err := mgr.client.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + return "", err + } + buf := new(strings.Builder) + _, err = io.Copy(buf, reader) + if err != nil { + return "", nil + } + return buf.String(), nil +} + +func (mgr *S3Manager) RemoveObject(objectName string, bucketName string, opts minio.RemoveObjectOptions) error { + if err := mgr.Connect(); err != nil { + return err + } + err := mgr.client.RemoveObject(context.Background(), bucketName, objectName, opts) + if err != nil { + return err + } + return nil +} + +func (mgr *S3Manager) CreateBucket(bucketName string) error { + if err := mgr.Connect(); err != nil { + return err + } + found, err := mgr.client.BucketExists(context.Background(), bucketName) + if err != nil { + return err + } + if !found { + if err = mgr.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{ + Region: mgr.config.Region, + }); err != nil { + return err + } + } + return nil +} + +func (mgr *S3Manager) RemoveBucket(bucketName string) error { + if err := mgr.Connect(); err != nil { + return err + } + found, err := mgr.client.BucketExists(context.Background(), bucketName) + if err != nil { + return err + } + if !found { + return nil + } + objectCh := mgr.client.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{ + Prefix: "", + Recursive: true, + }) + mgr.client.RemoveObjects(context.Background(), bucketName, objectCh, minio.RemoveObjectsOptions{}) + if err = mgr.client.RemoveBucket(context.Background(), bucketName); err != nil { + return err + } + return nil +} + +func (mgr *S3Manager) Connect() error { + client, err := minio.New(mgr.config.URL, &minio.Options{ + Creds: credentials.NewStaticV4(mgr.config.AccessKey, mgr.config.SecretKey, ""), + Secure: mgr.config.Secure, + }) + if err != nil { + return err + } + mgr.client = client + return nil +} diff --git a/webdav-go/voltaserve-webdav b/webdav-go/voltaserve-webdav new file mode 100755 index 000000000..e1ba9faea Binary files /dev/null and b/webdav-go/voltaserve-webdav differ