From 123a7f542d3ed7b967fdaf7b56fe6bc5e5570308 Mon Sep 17 00:00:00 2001 From: Anass Bouassaba Date: Tue, 23 Jul 2024 21:04:02 +0200 Subject: [PATCH] refactor: clients, preparing the ground for deduplication (#212) * refactor: client, preparing for deduplication * chore(api): format code --- conversion/client/api_client.go | 213 ------------------ conversion/client/api_client/health_client.go | 52 +++++ .../client/api_client/snapshot_client.go | 110 +++++++++ conversion/client/api_client/task_client.go | 130 +++++++++++ .../{ => language_client}/language_client.go | 2 +- .../{ => mosaic_client}/mosaic_client.go | 2 +- conversion/identifier/pipeline_identifier.go | 4 +- conversion/model/types.go | 8 +- conversion/pipeline/audio_video_pipeline.go | 56 ++--- conversion/pipeline/dispatcher.go | 41 ++-- conversion/pipeline/glb_pipeline.go | 44 ++-- conversion/pipeline/image_pipeline.go | 76 ++++--- conversion/pipeline/insights_pipeline.go | 53 +++-- conversion/pipeline/mosaic_pipeline.go | 47 ++-- conversion/pipeline/office_pipeline.go | 44 ++-- conversion/pipeline/pdf_pipeline.go | 72 +++--- conversion/pipeline/zip_pipeline.go | 48 ++-- conversion/processor/image_processor.go | 8 +- conversion/router/health_router.go | 6 +- conversion/router/pipeline_router.go | 8 +- conversion/runtime/scheduler.go | 12 +- ui/src/client/api/task.ts | 2 +- ui/src/components/task/task-item.tsx | 15 +- .../file_client.go} | 82 ++----- webdav/client/api_client/health_client.go | 48 ++++ webdav/client/idp_client/health_client.go | 48 ++++ .../token_client.go} | 40 +--- webdav/handler/handler.go | 11 +- webdav/handler/method_copy.go | 14 +- webdav/handler/method_delete.go | 10 +- webdav/handler/method_get.go | 10 +- webdav/handler/method_head.go | 12 +- webdav/handler/method_mkcol.go | 32 +-- webdav/handler/method_move.go | 14 +- webdav/handler/method_propfind.go | 20 +- webdav/handler/method_put.go | 16 +- webdav/main.go | 16 +- 37 files changed, 774 insertions(+), 652 deletions(-) delete mode 100644 conversion/client/api_client.go create mode 100644 conversion/client/api_client/health_client.go create mode 100644 conversion/client/api_client/snapshot_client.go create mode 100644 conversion/client/api_client/task_client.go rename conversion/client/{ => language_client}/language_client.go (98%) rename conversion/client/{ => mosaic_client}/mosaic_client.go (99%) rename webdav/client/{api_client.go => api_client/file_client.go} (84%) create mode 100644 webdav/client/api_client/health_client.go create mode 100644 webdav/client/idp_client/health_client.go rename webdav/client/{idp_client.go => idp_client/token_client.go} (75%) diff --git a/conversion/client/api_client.go b/conversion/client/api_client.go deleted file mode 100644 index 747cc2619..000000000 --- a/conversion/client/api_client.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2023 Anass Bouassaba. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the GNU Affero General Public License v3.0 only, included in the file -// licenses/AGPL.txt. - -package client - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/kouprlabs/voltaserve/conversion/config" -) - -type APIClient struct { - config *config.Config -} - -func NewAPIClient() *APIClient { - return &APIClient{ - config: config.GetConfig(), - } -} - -func (cl *APIClient) GetHealth() (string, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/v2/health", cl.config.APIURL), nil) - if err != nil { - return "", err - } - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(body), nil -} - -type PipelineRunOptions struct { - PipelineID *string `json:"pipelineId,omitempty"` - TaskID string `json:"taskId"` - SnapshotID string `json:"snapshotId"` - Bucket string `json:"bucket"` - Key string `json:"key"` - Payload map[string]string `json:"payload,omitempty"` -} - -type SnapshotPatchOptions struct { - Options PipelineRunOptions `json:"options"` - Fields []string `json:"fields"` - Original *S3Object `json:"original"` - Preview *S3Object `json:"preview"` - Text *S3Object `json:"text"` - OCR *S3Object `json:"ocr"` - Entities *S3Object `json:"entities"` - Mosaic *S3Object `json:"mosaic"` - Thumbnail *S3Object `json:"thumbnail"` - Status *string `json:"status"` - TaskID *string `json:"taskId"` -} - -const ( - SnapshotStatusWaiting = "waiting" - SnapshotStatusProcessing = "processing" - SnapshotStatusReady = "ready" - SnapshotStatusError = "error" -) - -const ( - SnapshotFieldOriginal = "original" - SnapshotFieldPreview = "preview" - SnapshotFieldText = "text" - SnapshotFieldOCR = "ocr" - SnapshotFieldEntities = "entities" - SnapshotFieldMosaic = "mosaic" - SnapshotFieldThumbnail = "thumbnail" - SnapshotFieldStatus = "status" - SnapshotFieldLanguage = "language" - SnapshotFieldTaskID = "taskId" -) - -type S3Object struct { - Bucket string `json:"bucket"` - Key string `json:"key"` - Size *int64 `json:"size,omitempty"` - Image *ImageProps `json:"image,omitempty"` -} - -type ImageProps struct { - Width int `json:"width"` - Height int `json:"height"` -} - -func (cl *APIClient) PatchSnapshot(opts SnapshotPatchOptions) error { - body, err := json.Marshal(opts) - if err != nil { - return err - } - req, err := http.NewRequest("PATCH", fmt.Sprintf("%s/v2/snapshots/%s?api_key=%s", cl.config.APIURL, opts.Options.SnapshotID, cl.config.Security.APIKey), bytes.NewBuffer(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - return nil -} - -type TaskCreateOptions struct { - Name string `json:"name"` - Error *string `json:"error,omitempty"` - Percentage *int `json:"percentage,omitempty"` - IsIndeterminate bool `json:"isIndeterminate"` - UserID string `json:"userId"` - Status string `json:"status"` -} - -const ( - TaskStatusWaiting = "waiting" - TaskStatusRunning = "running" - TaskStatusSuccess = "success" - TaskStatusError = "error" -) - -func (cl *APIClient) CreateTask(opts TaskCreateOptions) error { - body, err := json.Marshal(opts) - if err != nil { - return err - } - req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/tasks?api_key=%s", cl.config.APIURL, cl.config.Security.APIKey), bytes.NewBuffer(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - return nil -} - -type TaskPatchOptions struct { - Fields []string `json:"fields"` - Name *string `json:"name"` - Error *string `json:"error"` - Percentage *int `json:"percentage"` - IsIndeterminate *bool `json:"isIndeterminate"` - UserID *string `json:"userId"` - Status *string `json:"status"` - Payload map[string]string `json:"payload"` -} - -const ( - TaskFieldName = "name" - TaskFieldError = "error" - TaskFieldPercentage = "percentage" - TaskFieldIsIndeterminate = "isIndeterminate" - TaskFieldUserID = "userId" - TaskFieldStatus = "status" - TaskFieldPayload = "payload" -) - -func (cl *APIClient) PatchTask(id string, opts TaskPatchOptions) error { - body, err := json.Marshal(opts) - if err != nil { - return err - } - req, err := http.NewRequest("PATCH", fmt.Sprintf("%s/v2/tasks/%s?api_key=%s", cl.config.APIURL, id, cl.config.Security.APIKey), bytes.NewBuffer(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - return nil -} - -func (cl *APIClient) DeletTask(id string) error { - req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/v2/tasks/%s?api_key=%s", cl.config.APIURL, id, cl.config.Security.APIKey), nil) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - return nil -} diff --git a/conversion/client/api_client/health_client.go b/conversion/client/api_client/health_client.go new file mode 100644 index 000000000..e3e7abd7d --- /dev/null +++ b/conversion/client/api_client/health_client.go @@ -0,0 +1,52 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package api_client + +import ( + "fmt" + "io" + "net/http" + + "github.com/kouprlabs/voltaserve/conversion/config" + "github.com/kouprlabs/voltaserve/conversion/infra" +) + +type HealthClient struct { + config *config.Config +} + +func NewHealthClient() *HealthClient { + return &HealthClient{ + config: config.GetConfig(), + } +} + +func (cl *HealthClient) Get() (string, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/v2/health", cl.config.APIURL), nil) + if err != nil { + return "", err + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer func(Body io.ReadCloser) { + if err := Body.Close(); err != nil { + infra.GetLogger().Error(err) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} diff --git a/conversion/client/api_client/snapshot_client.go b/conversion/client/api_client/snapshot_client.go new file mode 100644 index 000000000..d3351223b --- /dev/null +++ b/conversion/client/api_client/snapshot_client.go @@ -0,0 +1,110 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package api_client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/kouprlabs/voltaserve/conversion/config" + "github.com/kouprlabs/voltaserve/conversion/infra" +) + +type SnapshotClient struct { + config *config.Config +} + +func NewSnapshotClient() *SnapshotClient { + return &SnapshotClient{ + config: config.GetConfig(), + } +} + +type SnapshotPatchOptions struct { + Options PipelineRunOptions `json:"options"` + Fields []string `json:"fields"` + Original *S3Object `json:"original"` + Preview *S3Object `json:"preview"` + Text *S3Object `json:"text"` + OCR *S3Object `json:"ocr"` + Entities *S3Object `json:"entities"` + Mosaic *S3Object `json:"mosaic"` + Thumbnail *S3Object `json:"thumbnail"` + Status *string `json:"status"` + TaskID *string `json:"taskId"` +} + +const ( + SnapshotStatusWaiting = "waiting" + SnapshotStatusProcessing = "processing" + SnapshotStatusReady = "ready" + SnapshotStatusError = "error" +) + +const ( + SnapshotFieldOriginal = "original" + SnapshotFieldPreview = "preview" + SnapshotFieldText = "text" + SnapshotFieldOCR = "ocr" + SnapshotFieldEntities = "entities" + SnapshotFieldMosaic = "mosaic" + SnapshotFieldThumbnail = "thumbnail" + SnapshotFieldStatus = "status" + SnapshotFieldLanguage = "language" + SnapshotFieldTaskID = "taskId" +) + +type PipelineRunOptions struct { + PipelineID *string `json:"pipelineId,omitempty"` + TaskID string `json:"taskId"` + SnapshotID string `json:"snapshotId"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Payload map[string]string `json:"payload,omitempty"` +} + +type S3Object struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Size *int64 `json:"size,omitempty"` + Image *ImageProps `json:"image,omitempty"` +} + +type ImageProps struct { + Width int `json:"width"` + Height int `json:"height"` +} + +func (cl *SnapshotClient) Patch(opts SnapshotPatchOptions) error { + body, err := json.Marshal(opts) + if err != nil { + return err + } + req, err := http.NewRequest("PATCH", fmt.Sprintf("%s/v2/snapshots/%s?api_key=%s", cl.config.APIURL, opts.Options.SnapshotID, cl.config.Security.APIKey), bytes.NewBuffer(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + if err := Body.Close(); err != nil { + infra.GetLogger().Error(err) + } + }(resp.Body) + return nil +} diff --git a/conversion/client/api_client/task_client.go b/conversion/client/api_client/task_client.go new file mode 100644 index 000000000..c107003fd --- /dev/null +++ b/conversion/client/api_client/task_client.go @@ -0,0 +1,130 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package api_client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/kouprlabs/voltaserve/conversion/config" + "github.com/kouprlabs/voltaserve/conversion/infra" +) + +type TaskClient struct { + config *config.Config +} + +func NewTaskClient() *TaskClient { + return &TaskClient{ + config: config.GetConfig(), + } +} + +type TaskCreateOptions struct { + Name string `json:"name"` + Error *string `json:"error,omitempty"` + Percentage *int `json:"percentage,omitempty"` + IsIndeterminate bool `json:"isIndeterminate"` + UserID string `json:"userId"` + Status string `json:"status"` +} + +const ( + TaskStatusWaiting = "waiting" + TaskStatusRunning = "running" + TaskStatusSuccess = "success" + TaskStatusError = "error" +) + +func (cl *TaskClient) Create(opts TaskCreateOptions) error { + body, err := json.Marshal(opts) + if err != nil { + return err + } + req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/tasks?api_key=%s", cl.config.APIURL, cl.config.Security.APIKey), bytes.NewBuffer(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + return nil +} + +type TaskPatchOptions struct { + Fields []string `json:"fields"` + Name *string `json:"name"` + Error *string `json:"error"` + Percentage *int `json:"percentage"` + IsIndeterminate *bool `json:"isIndeterminate"` + UserID *string `json:"userId"` + Status *string `json:"status"` + Payload map[string]string `json:"payload"` +} + +const ( + TaskFieldName = "name" + TaskFieldError = "error" + TaskFieldPercentage = "percentage" + TaskFieldIsIndeterminate = "isIndeterminate" + TaskFieldUserID = "userId" + TaskFieldStatus = "status" + TaskFieldPayload = "payload" +) + +func (cl *TaskClient) Patch(id string, opts TaskPatchOptions) error { + body, err := json.Marshal(opts) + if err != nil { + return err + } + req, err := http.NewRequest("PATCH", fmt.Sprintf("%s/v2/tasks/%s?api_key=%s", cl.config.APIURL, id, cl.config.Security.APIKey), bytes.NewBuffer(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + if err := Body.Close(); err != nil { + infra.GetLogger().Error(err) + } + }(resp.Body) + return nil +} + +func (cl *TaskClient) Delete(id string) error { + req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/v2/tasks/%s?api_key=%s", cl.config.APIURL, id, cl.config.Security.APIKey), nil) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + if err := Body.Close(); err != nil { + infra.GetLogger().Error(err) + } + }(resp.Body) + return nil +} diff --git a/conversion/client/language_client.go b/conversion/client/language_client/language_client.go similarity index 98% rename from conversion/client/language_client.go rename to conversion/client/language_client/language_client.go index 4ef5e8425..3219962ca 100644 --- a/conversion/client/language_client.go +++ b/conversion/client/language_client/language_client.go @@ -8,7 +8,7 @@ // by the GNU Affero General Public License v3.0 only, included in the file // licenses/AGPL.txt. -package client +package language_client import ( "bytes" diff --git a/conversion/client/mosaic_client.go b/conversion/client/mosaic_client/mosaic_client.go similarity index 99% rename from conversion/client/mosaic_client.go rename to conversion/client/mosaic_client/mosaic_client.go index 3ded36be1..4a0dbed7a 100644 --- a/conversion/client/mosaic_client.go +++ b/conversion/client/mosaic_client/mosaic_client.go @@ -8,7 +8,7 @@ // by the GNU Affero General Public License v3.0 only, included in the file // licenses/AGPL.txt. -package client +package mosaic_client import ( "bytes" diff --git a/conversion/identifier/pipeline_identifier.go b/conversion/identifier/pipeline_identifier.go index 591346fd1..b51124a9b 100644 --- a/conversion/identifier/pipeline_identifier.go +++ b/conversion/identifier/pipeline_identifier.go @@ -11,7 +11,7 @@ package identifier import ( - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/model" ) @@ -25,7 +25,7 @@ func NewPipelineIdentifier() *PipelineIdentifier { } } -func (pi *PipelineIdentifier) Identify(opts client.PipelineRunOptions) string { +func (pi *PipelineIdentifier) Identify(opts api_client.PipelineRunOptions) string { if opts.PipelineID != nil { return *opts.PipelineID } else { diff --git a/conversion/model/types.go b/conversion/model/types.go index 9e918d50a..aa64a9ab6 100644 --- a/conversion/model/types.go +++ b/conversion/model/types.go @@ -10,12 +10,14 @@ package model -import "github.com/kouprlabs/voltaserve/conversion/client" +import ( + "github.com/kouprlabs/voltaserve/conversion/client/api_client" +) type Pipeline interface { - Run(client.PipelineRunOptions) error + Run(api_client.PipelineRunOptions) error } type Builder interface { - Build(client.PipelineRunOptions) error + Build(api_client.PipelineRunOptions) error } diff --git a/conversion/pipeline/audio_video_pipeline.go b/conversion/pipeline/audio_video_pipeline.go index 660cc41ad..51600a6a1 100644 --- a/conversion/pipeline/audio_video_pipeline.go +++ b/conversion/pipeline/audio_video_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -26,24 +26,26 @@ import ( ) type audioVideoPipeline struct { - videoProc *processor.VideoProcessor - imageProc *processor.ImageProcessor - s3 *infra.S3Manager - apiClient *client.APIClient - config *config.Config + videoProc *processor.VideoProcessor + imageProc *processor.ImageProcessor + s3 *infra.S3Manager + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + config *config.Config } func NewAudioVideoPipeline() model.Pipeline { return &audioVideoPipeline{ - videoProc: processor.NewVideoProcessor(), - imageProc: processor.NewImageProcessor(), - s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - config: config.GetConfig(), + videoProc: processor.NewVideoProcessor(), + imageProc: processor.NewImageProcessor(), + s3: infra.NewS3Manager(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + config: config.GetConfig(), } } -func (p *audioVideoPipeline) Run(opts client.PipelineRunOptions) error { +func (p *audioVideoPipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -55,16 +57,16 @@ func (p *audioVideoPipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Creating thumbnail."), }); err != nil { return err } // Here we intentionally ignore the error, as the media file may contain just audio _ = p.createThumbnail(inputPath, opts) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Saving preview."), }); err != nil { return err @@ -72,17 +74,17 @@ func (p *audioVideoPipeline) Run(opts client.PipelineRunOptions) error { if err := p.saveOriginalAsPreview(inputPath, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName, client.TaskFieldStatus}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName, api_client.TaskFieldStatus}, Name: helper.ToPtr("Done."), - Status: helper.ToPtr(client.TaskStatusSuccess), + Status: helper.ToPtr(api_client.TaskStatusSuccess), }); err != nil { return err } return nil } -func (p *audioVideoPipeline) createThumbnail(inputPath string, opts client.PipelineRunOptions) error { +func (p *audioVideoPipeline) createThumbnail(inputPath string, opts api_client.PipelineRunOptions) error { tmpPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png") defer func(path string) { _, err := os.Stat(path) @@ -103,7 +105,7 @@ func (p *audioVideoPipeline) createThumbnail(inputPath string, opts client.Pipel if err != nil { return err } - s3Object := &client.S3Object{ + s3Object := &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/thumbnail" + filepath.Ext(tmpPath), Image: props, @@ -112,9 +114,9 @@ func (p *audioVideoPipeline) createThumbnail(inputPath string, opts client.Pipel if err := p.s3.PutFile(s3Object.Key, tmpPath, helper.DetectMimeFromFile(tmpPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldThumbnail}, + Fields: []string{api_client.SnapshotFieldThumbnail}, Thumbnail: s3Object, }); err != nil { return err @@ -122,15 +124,15 @@ func (p *audioVideoPipeline) createThumbnail(inputPath string, opts client.Pipel return nil } -func (p *audioVideoPipeline) saveOriginalAsPreview(inputPath string, opts client.PipelineRunOptions) error { +func (p *audioVideoPipeline) saveOriginalAsPreview(inputPath string, opts api_client.PipelineRunOptions) error { stat, err := os.Stat(inputPath) if err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.Key, Size: helper.ToPtr(stat.Size()), diff --git a/conversion/pipeline/dispatcher.go b/conversion/pipeline/dispatcher.go index 25469c224..a117e9518 100644 --- a/conversion/pipeline/dispatcher.go +++ b/conversion/pipeline/dispatcher.go @@ -11,7 +11,7 @@ package pipeline import ( - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/errorpkg" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" @@ -28,7 +28,8 @@ type Dispatcher struct { mosaicPipeline model.Pipeline glbPipeline model.Pipeline zipPipeline model.Pipeline - apiClient *client.APIClient + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient } func NewDispatcher() *Dispatcher { @@ -42,22 +43,22 @@ func NewDispatcher() *Dispatcher { mosaicPipeline: NewMosaicPipeline(), glbPipeline: NewGLBPipeline(), zipPipeline: NewZIPPipeline(), - apiClient: client.NewAPIClient(), + taskClient: api_client.NewTaskClient(), } } -func (d *Dispatcher) Dispatch(opts client.PipelineRunOptions) error { - if err := d.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ +func (d *Dispatcher) Dispatch(opts api_client.PipelineRunOptions) error { + if err := d.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldStatus}, - Status: helper.ToPtr(client.SnapshotStatusProcessing), + Fields: []string{api_client.SnapshotFieldStatus}, + Status: helper.ToPtr(api_client.SnapshotStatusProcessing), }); err != nil { return err } - if err := d.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ + if err := d.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ Name: helper.ToPtr("Processing."), - Fields: []string{client.TaskFieldStatus}, - Status: helper.ToPtr(client.TaskStatusRunning), + Fields: []string{api_client.TaskFieldStatus}, + Status: helper.ToPtr(api_client.TaskStatusRunning), }); err != nil { return err } @@ -81,30 +82,30 @@ func (d *Dispatcher) Dispatch(opts client.PipelineRunOptions) error { err = d.zipPipeline.Run(opts) } if err != nil { - if err := d.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := d.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldStatus}, - Status: helper.ToPtr(client.SnapshotStatusError), + Fields: []string{api_client.SnapshotFieldStatus}, + Status: helper.ToPtr(api_client.SnapshotStatusError), }); err != nil { return err } - if err := d.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldStatus, client.TaskFieldError}, - Status: helper.ToPtr(client.TaskStatusError), + if err := d.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldStatus, api_client.TaskFieldError}, + Status: helper.ToPtr(api_client.TaskStatusError), Error: helper.ToPtr(errorpkg.GetUserFriendlyMessage(err.Error(), errorpkg.FallbackMessage)), }); err != nil { return err } return err } else { - if err := d.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := d.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldStatus, client.SnapshotFieldTaskID}, - Status: helper.ToPtr(client.SnapshotStatusReady), + Fields: []string{api_client.SnapshotFieldStatus, api_client.SnapshotFieldTaskID}, + Status: helper.ToPtr(api_client.SnapshotStatusReady), }); err != nil { return err } - if err := d.apiClient.DeletTask(opts.TaskID); err != nil { + if err := d.taskClient.Delete(opts.TaskID); err != nil { return err } return nil diff --git a/conversion/pipeline/glb_pipeline.go b/conversion/pipeline/glb_pipeline.go index 6c3c56830..8617a2dcf 100644 --- a/conversion/pipeline/glb_pipeline.go +++ b/conversion/pipeline/glb_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -26,24 +26,26 @@ import ( ) type glbPipeline struct { - glbProc *processor.GLBProcessor - imageProc *processor.ImageProcessor - s3 *infra.S3Manager - apiClient *client.APIClient - config *config.Config + glbProc *processor.GLBProcessor + imageProc *processor.ImageProcessor + s3 *infra.S3Manager + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + config *config.Config } func NewGLBPipeline() model.Pipeline { return &glbPipeline{ - glbProc: processor.NewGLBProcessor(), - imageProc: processor.NewImageProcessor(), - s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - config: config.GetConfig(), + glbProc: processor.NewGLBProcessor(), + imageProc: processor.NewImageProcessor(), + s3: infra.NewS3Manager(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + config: config.GetConfig(), } } -func (p *glbPipeline) Run(opts client.PipelineRunOptions) error { +func (p *glbPipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -55,8 +57,8 @@ func (p *glbPipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Creating thumbnail."), }); err != nil { return err @@ -68,10 +70,10 @@ func (p *glbPipeline) Run(opts client.PipelineRunOptions) error { if err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.Key, Size: helper.ToPtr(stat.Size()), @@ -82,7 +84,7 @@ func (p *glbPipeline) Run(opts client.PipelineRunOptions) error { return nil } -func (p *glbPipeline) createThumbnail(inputPath string, opts client.PipelineRunOptions) error { +func (p *glbPipeline) createThumbnail(inputPath string, opts api_client.PipelineRunOptions) error { tmpPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".jpeg") defer func(path string) { if err := os.Remove(path); errors.Is(err, os.ErrNotExist) { @@ -101,7 +103,7 @@ func (p *glbPipeline) createThumbnail(inputPath string, opts client.PipelineRunO if err != nil { return err } - s3Object := &client.S3Object{ + s3Object := &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/thumbnail" + filepath.Ext(tmpPath), Image: props, @@ -110,9 +112,9 @@ func (p *glbPipeline) createThumbnail(inputPath string, opts client.PipelineRunO if err := p.s3.PutFile(s3Object.Key, tmpPath, helper.DetectMimeFromFile(tmpPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldThumbnail}, + Fields: []string{api_client.SnapshotFieldThumbnail}, Thumbnail: s3Object, }); err != nil { return err diff --git a/conversion/pipeline/image_pipeline.go b/conversion/pipeline/image_pipeline.go index 16addbafe..186c2e223 100644 --- a/conversion/pipeline/image_pipeline.go +++ b/conversion/pipeline/image_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" @@ -27,24 +27,26 @@ import ( ) type imagePipeline struct { - imageProc *processor.ImageProcessor - s3 *infra.S3Manager - apiClient *client.APIClient - fileIdent *identifier.FileIdentifier - config *config.Config + imageProc *processor.ImageProcessor + s3 *infra.S3Manager + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + fileIdent *identifier.FileIdentifier + config *config.Config } func NewImagePipeline() model.Pipeline { return &imagePipeline{ - imageProc: processor.NewImageProcessor(), - s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - fileIdent: identifier.NewFileIdentifier(), - config: config.GetConfig(), + imageProc: processor.NewImageProcessor(), + s3: infra.NewS3Manager(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + fileIdent: identifier.NewFileIdentifier(), + config: config.GetConfig(), } } -func (p *imagePipeline) Run(opts client.PipelineRunOptions) error { +func (p *imagePipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -56,8 +58,8 @@ func (p *imagePipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Measuring image dimensions."), }); err != nil { return err @@ -68,8 +70,8 @@ func (p *imagePipeline) Run(opts client.PipelineRunOptions) error { } var imagePath string if filepath.Ext(inputPath) == ".tiff" { - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Converting TIFF image to JPEG format."), }); err != nil { return err @@ -92,25 +94,25 @@ func (p *imagePipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(imagePath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Creating thumbnail."), }); err != nil { return err } // We don't consider failing the creation of the thumbnail as an error _ = p.createThumbnail(imagePath, opts) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName, client.TaskFieldStatus}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName, api_client.TaskFieldStatus}, Name: helper.ToPtr("Done."), - Status: helper.ToPtr(client.TaskStatusSuccess), + Status: helper.ToPtr(api_client.TaskStatusSuccess), }); err != nil { return err } return nil } -func (p *imagePipeline) measureImageDimensions(inputPath string, opts client.PipelineRunOptions) (*client.ImageProps, error) { +func (p *imagePipeline) measureImageDimensions(inputPath string, opts api_client.PipelineRunOptions) (*api_client.ImageProps, error) { imageProps, err := p.imageProc.MeasureImage(inputPath) if err != nil { return nil, err @@ -119,10 +121,10 @@ func (p *imagePipeline) measureImageDimensions(inputPath string, opts client.Pip if err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldOriginal}, - Original: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldOriginal}, + Original: &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.Key, Size: helper.ToPtr(stat.Size()), @@ -134,7 +136,7 @@ func (p *imagePipeline) measureImageDimensions(inputPath string, opts client.Pip return imageProps, nil } -func (p *imagePipeline) createThumbnail(inputPath string, opts client.PipelineRunOptions) error { +func (p *imagePipeline) createThumbnail(inputPath string, opts api_client.PipelineRunOptions) error { tmpPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png") isAvailable, err := p.imageProc.Thumbnail(inputPath, tmpPath) if err != nil { @@ -159,7 +161,7 @@ func (p *imagePipeline) createThumbnail(inputPath string, opts client.PipelineRu if err != nil { return err } - s3Object := &client.S3Object{ + s3Object := &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/thumbnail" + filepath.Ext(tmpPath), Image: props, @@ -168,9 +170,9 @@ func (p *imagePipeline) createThumbnail(inputPath string, opts client.PipelineRu if err := p.s3.PutFile(s3Object.Key, tmpPath, helper.DetectMimeFromFile(tmpPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldThumbnail}, + Fields: []string{api_client.SnapshotFieldThumbnail}, Thumbnail: s3Object, }); err != nil { return err @@ -178,7 +180,7 @@ func (p *imagePipeline) createThumbnail(inputPath string, opts client.PipelineRu return nil } -func (p *imagePipeline) convertTIFFToJPEG(inputPath string, imageProps client.ImageProps, opts client.PipelineRunOptions) (*string, error) { +func (p *imagePipeline) convertTIFFToJPEG(inputPath string, imageProps api_client.ImageProps, opts api_client.PipelineRunOptions) (*string, error) { jpegPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".jpg") if err := p.imageProc.ConvertImage(inputPath, jpegPath); err != nil { return nil, err @@ -187,7 +189,7 @@ func (p *imagePipeline) convertTIFFToJPEG(inputPath string, imageProps client.Im if err != nil { return nil, err } - s3Object := &client.S3Object{ + s3Object := &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/preview.jpg", Size: helper.ToPtr(stat.Size()), @@ -196,9 +198,9 @@ func (p *imagePipeline) convertTIFFToJPEG(inputPath string, imageProps client.Im if err := p.s3.PutFile(s3Object.Key, jpegPath, helper.DetectMimeFromFile(jpegPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, + Fields: []string{api_client.SnapshotFieldPreview}, Preview: s3Object, }); err != nil { return nil, err @@ -206,15 +208,15 @@ func (p *imagePipeline) convertTIFFToJPEG(inputPath string, imageProps client.Im return &jpegPath, nil } -func (p *imagePipeline) saveOriginalAsPreview(inputPath string, opts client.PipelineRunOptions) error { +func (p *imagePipeline) saveOriginalAsPreview(inputPath string, opts api_client.PipelineRunOptions) error { stat, err := os.Stat(inputPath) if err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.Key, Size: helper.ToPtr(stat.Size()), diff --git a/conversion/pipeline/insights_pipeline.go b/conversion/pipeline/insights_pipeline.go index 7ab0d1b2b..af78b8816 100644 --- a/conversion/pipeline/insights_pipeline.go +++ b/conversion/pipeline/insights_pipeline.go @@ -18,7 +18,8 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" + "github.com/kouprlabs/voltaserve/conversion/client/language_client" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -32,8 +33,9 @@ type insightsPipeline struct { ocrProc *processor.OCRProcessor fileIdent *identifier.FileIdentifier s3 *infra.S3Manager - apiClient *client.APIClient - languageClient *client.LanguageClient + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + languageClient *language_client.LanguageClient } func NewInsightsPipeline() model.Pipeline { @@ -43,12 +45,13 @@ func NewInsightsPipeline() model.Pipeline { ocrProc: processor.NewOCRProcessor(), fileIdent: identifier.NewFileIdentifier(), s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - languageClient: client.NewLanguageClient(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + languageClient: language_client.NewLanguageClient(), } } -func (p *insightsPipeline) Run(opts client.PipelineRunOptions) error { +func (p *insightsPipeline) Run(opts api_client.PipelineRunOptions) error { if opts.Payload == nil || opts.Payload["language"] == "" { return errors.New("language is undefined") } @@ -63,8 +66,8 @@ func (p *insightsPipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Extracting text."), }); err != nil { return err @@ -73,8 +76,8 @@ func (p *insightsPipeline) Run(opts client.PipelineRunOptions) error { if err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Collecting entities."), }); err != nil { return err @@ -82,17 +85,17 @@ func (p *insightsPipeline) Run(opts client.PipelineRunOptions) error { if err := p.createEntities(*text, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName, client.TaskFieldStatus}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName, api_client.TaskFieldStatus}, Name: helper.ToPtr("Done."), - Status: helper.ToPtr(client.TaskStatusSuccess), + Status: helper.ToPtr(api_client.TaskStatusSuccess), }); err != nil { return err } return nil } -func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunOptions) (*string, error) { +func (p *insightsPipeline) createText(inputPath string, opts api_client.PipelineRunOptions) (*string, error) { /* Generate PDF/A */ var pdfPath string if p.fileIdent.IsImage(opts.Key) { @@ -130,7 +133,7 @@ func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunO if err != nil { return nil, err } - s3Object := client.S3Object{ + s3Object := api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/ocr.pdf", Size: helper.ToPtr(stat.Size()), @@ -138,9 +141,9 @@ func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunO if err := p.s3.PutFile(s3Object.Key, pdfPath, helper.DetectMimeFromFile(pdfPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldOCR}, + Fields: []string{api_client.SnapshotFieldOCR}, OCR: &s3Object, }); err != nil { return nil, err @@ -156,7 +159,7 @@ func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunO return nil, err } /* Set text S3 object */ - s3Object := client.S3Object{ + s3Object := api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/text.txt", Size: helper.ToPtr(int64(len(*text))), @@ -164,9 +167,9 @@ func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunO if err := p.s3.PutText(s3Object.Key, *text, "text/plain", s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldText}, + Fields: []string{api_client.SnapshotFieldText}, Text: &s3Object, }); err != nil { return nil, err @@ -174,14 +177,14 @@ func (p *insightsPipeline) createText(inputPath string, opts client.PipelineRunO return text, nil } -func (p *insightsPipeline) createEntities(text string, opts client.PipelineRunOptions) error { +func (p *insightsPipeline) createEntities(text string, opts api_client.PipelineRunOptions) error { if len(text) == 0 { return errors.New("text is empty") } if len(text) > 1000000 { return errors.New("text exceeds supported limit of 1000000 characters") } - res, err := p.languageClient.GetEntities(client.GetEntitiesOptions{ + res, err := p.languageClient.GetEntities(language_client.GetEntitiesOptions{ Text: text, Language: opts.Payload["language"], }) @@ -193,7 +196,7 @@ func (p *insightsPipeline) createEntities(text string, opts client.PipelineRunOp return err } content := string(b) - s3Object := client.S3Object{ + s3Object := api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/entities.json", Size: helper.ToPtr(int64(len(content))), @@ -201,9 +204,9 @@ func (p *insightsPipeline) createEntities(text string, opts client.PipelineRunOp if err := p.s3.PutText(s3Object.Key, content, "application/json", s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldEntities}, + Fields: []string{api_client.SnapshotFieldEntities}, Entities: &s3Object, }); err != nil { return err diff --git a/conversion/pipeline/mosaic_pipeline.go b/conversion/pipeline/mosaic_pipeline.go index 3d4a2fa3d..72033a792 100644 --- a/conversion/pipeline/mosaic_pipeline.go +++ b/conversion/pipeline/mosaic_pipeline.go @@ -17,7 +17,8 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" + "github.com/kouprlabs/voltaserve/conversion/client/mosaic_client" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -26,24 +27,26 @@ import ( ) type mosaicPipeline struct { - videoProc *processor.VideoProcessor - fileIdent *identifier.FileIdentifier - s3 *infra.S3Manager - apiClient *client.APIClient - mosaicClient *client.MosaicClient + videoProc *processor.VideoProcessor + fileIdent *identifier.FileIdentifier + s3 *infra.S3Manager + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + mosaicClient *mosaic_client.MosaicClient } func NewMosaicPipeline() model.Pipeline { return &mosaicPipeline{ - videoProc: processor.NewVideoProcessor(), - fileIdent: identifier.NewFileIdentifier(), - s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - mosaicClient: client.NewMosaicClient(), + videoProc: processor.NewVideoProcessor(), + fileIdent: identifier.NewFileIdentifier(), + s3: infra.NewS3Manager(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + mosaicClient: mosaic_client.NewMosaicClient(), } } -func (p *mosaicPipeline) Run(opts client.PipelineRunOptions) error { +func (p *mosaicPipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -55,8 +58,8 @@ func (p *mosaicPipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Creating mosaic."), }); err != nil { return err @@ -64,29 +67,29 @@ func (p *mosaicPipeline) Run(opts client.PipelineRunOptions) error { if err := p.create(inputPath, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName, client.TaskFieldStatus}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName, api_client.TaskFieldStatus}, Name: helper.ToPtr("Done."), - Status: helper.ToPtr(client.TaskStatusSuccess), + Status: helper.ToPtr(api_client.TaskStatusSuccess), }); err != nil { return err } return nil } -func (p *mosaicPipeline) create(inputPath string, opts client.PipelineRunOptions) error { +func (p *mosaicPipeline) create(inputPath string, opts api_client.PipelineRunOptions) error { if p.fileIdent.IsImage(opts.Key) { - if _, err := p.mosaicClient.Create(client.MosaicCreateOptions{ + if _, err := p.mosaicClient.Create(mosaic_client.MosaicCreateOptions{ Path: inputPath, S3Key: filepath.FromSlash(opts.SnapshotID), S3Bucket: opts.Bucket, }); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldMosaic}, - Mosaic: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldMosaic}, + Mosaic: &api_client.S3Object{ Key: filepath.FromSlash(opts.SnapshotID + "/mosaic.json"), Bucket: opts.Bucket, }, diff --git a/conversion/pipeline/office_pipeline.go b/conversion/pipeline/office_pipeline.go index 5bcc9a588..22e08fdca 100644 --- a/conversion/pipeline/office_pipeline.go +++ b/conversion/pipeline/office_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -26,26 +26,28 @@ import ( ) type officePipeline struct { - pdfPipeline model.Pipeline - officeProc *processor.OfficeProcessor - pdfProc *processor.PDFProcessor - s3 *infra.S3Manager - config *config.Config - apiClient *client.APIClient + pdfPipeline model.Pipeline + officeProc *processor.OfficeProcessor + pdfProc *processor.PDFProcessor + s3 *infra.S3Manager + config *config.Config + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient } func NewOfficePipeline() model.Pipeline { return &officePipeline{ - pdfPipeline: NewPDFPipeline(), - officeProc: processor.NewOfficeProcessor(), - pdfProc: processor.NewPDFProcessor(), - s3: infra.NewS3Manager(), - config: config.GetConfig(), - apiClient: client.NewAPIClient(), + pdfPipeline: NewPDFPipeline(), + officeProc: processor.NewOfficeProcessor(), + pdfProc: processor.NewPDFProcessor(), + s3: infra.NewS3Manager(), + config: config.GetConfig(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), } } -func (p *officePipeline) Run(opts client.PipelineRunOptions) error { +func (p *officePipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -58,8 +60,8 @@ func (p *officePipeline) Run(opts client.PipelineRunOptions) error { } } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Converting to PDF."), }); err != nil { return err @@ -68,7 +70,7 @@ func (p *officePipeline) Run(opts client.PipelineRunOptions) error { if err != nil { return err } - if err := p.pdfPipeline.Run(client.PipelineRunOptions{ + if err := p.pdfPipeline.Run(api_client.PipelineRunOptions{ Bucket: opts.Bucket, Key: *pdfKey, SnapshotID: opts.SnapshotID, @@ -78,7 +80,7 @@ func (p *officePipeline) Run(opts client.PipelineRunOptions) error { return nil } -func (p *officePipeline) convertToPDF(inputPath string, opts client.PipelineRunOptions) (*string, error) { +func (p *officePipeline) convertToPDF(inputPath string, opts api_client.PipelineRunOptions) (*string, error) { outputDir := filepath.FromSlash(os.TempDir() + "/" + helper.NewID()) outputPath, err := p.officeProc.PDF(inputPath, outputDir) if err != nil { @@ -104,10 +106,10 @@ func (p *officePipeline) convertToPDF(inputPath string, opts client.PipelineRunO if err := p.s3.PutFile(pdfKey, *outputPath, helper.DetectMimeFromFile(*outputPath), opts.Bucket, minio.PutObjectOptions{}); err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: pdfKey, Size: helper.ToPtr(stat.Size()), diff --git a/conversion/pipeline/pdf_pipeline.go b/conversion/pipeline/pdf_pipeline.go index 7d886d761..4484aaddb 100644 --- a/conversion/pipeline/pdf_pipeline.go +++ b/conversion/pipeline/pdf_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" @@ -27,26 +27,28 @@ import ( ) type pdfPipeline struct { - pdfProc *processor.PDFProcessor - imageProc *processor.ImageProcessor - s3 *infra.S3Manager - apiClient *client.APIClient - fileIdent *identifier.FileIdentifier - config *config.Config + pdfProc *processor.PDFProcessor + imageProc *processor.ImageProcessor + s3 *infra.S3Manager + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient + fileIdent *identifier.FileIdentifier + config *config.Config } func NewPDFPipeline() model.Pipeline { return &pdfPipeline{ - pdfProc: processor.NewPDFProcessor(), - imageProc: processor.NewImageProcessor(), - s3: infra.NewS3Manager(), - apiClient: client.NewAPIClient(), - fileIdent: identifier.NewFileIdentifier(), - config: config.GetConfig(), + pdfProc: processor.NewPDFProcessor(), + imageProc: processor.NewImageProcessor(), + s3: infra.NewS3Manager(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), + fileIdent: identifier.NewFileIdentifier(), + config: config.GetConfig(), } } -func (p *pdfPipeline) Run(opts client.PipelineRunOptions) error { +func (p *pdfPipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -58,8 +60,8 @@ func (p *pdfPipeline) Run(opts client.PipelineRunOptions) error { infra.GetLogger().Error(err) } }(inputPath) - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Creating thumbnail."), }); err != nil { return err @@ -67,8 +69,8 @@ func (p *pdfPipeline) Run(opts client.PipelineRunOptions) error { if err := p.createThumbnail(inputPath, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Saving preview."), }); err != nil { return err @@ -76,8 +78,8 @@ func (p *pdfPipeline) Run(opts client.PipelineRunOptions) error { if err := p.saveOriginalAsPreview(inputPath, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Extracting text."), }); err != nil { return err @@ -85,17 +87,17 @@ func (p *pdfPipeline) Run(opts client.PipelineRunOptions) error { if err := p.extractText(inputPath, opts); err != nil { return err } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName, client.TaskFieldStatus}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName, api_client.TaskFieldStatus}, Name: helper.ToPtr("Done."), - Status: helper.ToPtr(client.TaskStatusSuccess), + Status: helper.ToPtr(api_client.TaskStatusSuccess), }); err != nil { return err } return nil } -func (p *pdfPipeline) createThumbnail(inputPath string, opts client.PipelineRunOptions) error { +func (p *pdfPipeline) createThumbnail(inputPath string, opts api_client.PipelineRunOptions) error { tmpPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png") // We don't consider failing the creation of the thumbnail as an error _ = p.pdfProc.Thumbnail(inputPath, 0, p.config.Limits.ImagePreviewMaxHeight, tmpPath) @@ -114,7 +116,7 @@ func (p *pdfPipeline) createThumbnail(inputPath string, opts client.PipelineRunO if err != nil { return err } - s3Object := &client.S3Object{ + s3Object := &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.SnapshotID + "/thumbnail" + filepath.Ext(tmpPath), Image: props, @@ -123,9 +125,9 @@ func (p *pdfPipeline) createThumbnail(inputPath string, opts client.PipelineRunO if err := p.s3.PutFile(s3Object.Key, tmpPath, helper.DetectMimeFromFile(tmpPath), s3Object.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldThumbnail}, + Fields: []string{api_client.SnapshotFieldThumbnail}, Thumbnail: s3Object, }); err != nil { return err @@ -133,7 +135,7 @@ func (p *pdfPipeline) createThumbnail(inputPath string, opts client.PipelineRunO return nil } -func (p *pdfPipeline) extractText(inputPath string, opts client.PipelineRunOptions) error { +func (p *pdfPipeline) extractText(inputPath string, opts api_client.PipelineRunOptions) error { text, err := p.pdfProc.TextFromPDF(inputPath) if err != nil { infra.GetLogger().Named(infra.StrPipeline).Errorw(err.Error()) @@ -145,10 +147,10 @@ func (p *pdfPipeline) extractText(inputPath string, opts client.PipelineRunOptio if err := p.s3.PutText(key, *text, "text/plain", opts.Bucket, minio.PutObjectOptions{}); err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldText}, - Text: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldText}, + Text: &api_client.S3Object{ Bucket: opts.Bucket, Key: key, Size: helper.ToPtr(int64(len(*text))), @@ -159,15 +161,15 @@ func (p *pdfPipeline) extractText(inputPath string, opts client.PipelineRunOptio return nil } -func (p *pdfPipeline) saveOriginalAsPreview(inputPath string, opts client.PipelineRunOptions) error { +func (p *pdfPipeline) saveOriginalAsPreview(inputPath string, opts api_client.PipelineRunOptions) error { stat, err := os.Stat(inputPath) if err != nil { return err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: opts.Key, Size: helper.ToPtr(stat.Size()), diff --git a/conversion/pipeline/zip_pipeline.go b/conversion/pipeline/zip_pipeline.go index 305121a28..8b9786303 100644 --- a/conversion/pipeline/zip_pipeline.go +++ b/conversion/pipeline/zip_pipeline.go @@ -17,7 +17,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" "github.com/kouprlabs/voltaserve/conversion/infra" @@ -26,26 +26,28 @@ import ( ) type zipPipeline struct { - glbPipeline model.Pipeline - zipProc *processor.ZIPProcessor - gltfProc *processor.GLTFProcessor - s3 *infra.S3Manager - fi *identifier.FileIdentifier - apiClient *client.APIClient + glbPipeline model.Pipeline + zipProc *processor.ZIPProcessor + gltfProc *processor.GLTFProcessor + s3 *infra.S3Manager + fi *identifier.FileIdentifier + taskClient *api_client.TaskClient + snapshotClient *api_client.SnapshotClient } func NewZIPPipeline() model.Pipeline { return &zipPipeline{ - glbPipeline: NewGLBPipeline(), - zipProc: processor.NewZIPProcessor(), - gltfProc: processor.NewGLTFProcessor(), - s3: infra.NewS3Manager(), - fi: identifier.NewFileIdentifier(), - apiClient: client.NewAPIClient(), + glbPipeline: NewGLBPipeline(), + zipProc: processor.NewZIPProcessor(), + gltfProc: processor.NewGLTFProcessor(), + s3: infra.NewS3Manager(), + fi: identifier.NewFileIdentifier(), + taskClient: api_client.NewTaskClient(), + snapshotClient: api_client.NewSnapshotClient(), } } -func (p *zipPipeline) Run(opts client.PipelineRunOptions) error { +func (p *zipPipeline) Run(opts api_client.PipelineRunOptions) error { inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key)) if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket, minio.GetObjectOptions{}); err != nil { return err @@ -62,8 +64,8 @@ func (p *zipPipeline) Run(opts client.PipelineRunOptions) error { return err } if isGLTF { - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Extracting ZIP."), }); err != nil { return err @@ -85,8 +87,8 @@ func (p *zipPipeline) Run(opts client.PipelineRunOptions) error { // Do nothing, treat it as a ZIP file return nil } - if err := p.apiClient.PatchTask(opts.TaskID, client.TaskPatchOptions{ - Fields: []string{client.TaskFieldName}, + if err := p.taskClient.Patch(opts.TaskID, api_client.TaskPatchOptions{ + Fields: []string{api_client.TaskFieldName}, Name: helper.ToPtr("Converting to GLB."), }); err != nil { return err @@ -95,7 +97,7 @@ func (p *zipPipeline) Run(opts client.PipelineRunOptions) error { if err != nil { return err } - if err := p.glbPipeline.Run(client.PipelineRunOptions{ + if err := p.glbPipeline.Run(api_client.PipelineRunOptions{ Bucket: opts.Bucket, Key: *glbKey, SnapshotID: opts.SnapshotID, @@ -107,7 +109,7 @@ func (p *zipPipeline) Run(opts client.PipelineRunOptions) error { return nil } -func (p *zipPipeline) convertToGLB(inputPath string, opts client.PipelineRunOptions) (*string, error) { +func (p *zipPipeline) convertToGLB(inputPath string, opts api_client.PipelineRunOptions) (*string, error) { outputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".glb") if err := p.gltfProc.ToGLB(inputPath, outputPath); err != nil { return nil, err @@ -127,10 +129,10 @@ func (p *zipPipeline) convertToGLB(inputPath string, opts client.PipelineRunOpti if err := p.s3.PutFile(glbKey, outputPath, helper.DetectMimeFromFile(outputPath), opts.Bucket, minio.PutObjectOptions{}); err != nil { return nil, err } - if err := p.apiClient.PatchSnapshot(client.SnapshotPatchOptions{ + if err := p.snapshotClient.Patch(api_client.SnapshotPatchOptions{ Options: opts, - Fields: []string{client.SnapshotFieldPreview}, - Preview: &client.S3Object{ + Fields: []string{api_client.SnapshotFieldPreview}, + Preview: &api_client.S3Object{ Bucket: opts.Bucket, Key: glbKey, Size: helper.ToPtr(stat.Size()), diff --git a/conversion/processor/image_processor.go b/conversion/processor/image_processor.go index ae0396f3c..9375cb023 100644 --- a/conversion/processor/image_processor.go +++ b/conversion/processor/image_processor.go @@ -14,7 +14,7 @@ import ( "strconv" "strings" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/helper" "github.com/kouprlabs/voltaserve/conversion/identifier" @@ -22,14 +22,12 @@ import ( ) type ImageProcessor struct { - apiClient *client.APIClient fileIdent *identifier.FileIdentifier config *config.Config } func NewImageProcessor() *ImageProcessor { return &ImageProcessor{ - apiClient: client.NewAPIClient(), fileIdent: identifier.NewFileIdentifier(), config: config.GetConfig(), } @@ -55,7 +53,7 @@ func (p *ImageProcessor) Thumbnail(inputPath string, outputPath string) (*bool, return helper.ToPtr(false), nil } -func (p *ImageProcessor) MeasureImage(inputPath string) (*client.ImageProps, error) { +func (p *ImageProcessor) MeasureImage(inputPath string) (*api_client.ImageProps, error) { size, err := infra.NewCommand().ReadOutput("identify", "-format", "%w,%h", inputPath) if err != nil { return nil, err @@ -69,7 +67,7 @@ func (p *ImageProcessor) MeasureImage(inputPath string) (*client.ImageProps, err if err != nil { return nil, err } - return &client.ImageProps{ + return &api_client.ImageProps{ Width: width, Height: height, }, nil diff --git a/conversion/router/health_router.go b/conversion/router/health_router.go index 2412c43d1..9cc0ff9c6 100644 --- a/conversion/router/health_router.go +++ b/conversion/router/health_router.go @@ -15,7 +15,7 @@ import ( "github.com/gofiber/fiber/v2" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/infra" "github.com/kouprlabs/voltaserve/conversion/runtime" ) @@ -38,7 +38,7 @@ func (r *HealthRouter) AppendRoutes(g fiber.Router) { g.Get("health", r.GetHealth) } -// Healdth godoc +// GetHealth godoc // // @Summary Get Health // @Description Get Health @@ -55,7 +55,7 @@ func (r *HealthRouter) GetHealth(c *fiber.Ctx) error { if err := infra.NewS3Manager().Connect(); err != nil { return c.SendStatus(http.StatusServiceUnavailable) } - if ok, err := client.NewAPIClient().GetHealth(); err != nil || ok != "OK" { + if ok, err := api_client.NewHealthClient().Get(); err != nil || ok != "OK" { return c.SendStatus(http.StatusServiceUnavailable) } return c.SendString("OK") diff --git a/conversion/router/pipeline_router.go b/conversion/router/pipeline_router.go index bad0f7e7a..18c6ca890 100644 --- a/conversion/router/pipeline_router.go +++ b/conversion/router/pipeline_router.go @@ -14,7 +14,7 @@ import ( "github.com/go-playground/validator/v10" "github.com/gofiber/fiber/v2" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/config" "github.com/kouprlabs/voltaserve/conversion/errorpkg" "github.com/kouprlabs/voltaserve/conversion/runtime" @@ -40,7 +40,7 @@ func (r *PipelineRouter) AppendRoutes(g fiber.Router) { g.Post("pipelines/run", r.Run) } -// Create godoc +// Run godoc // // @Summary Run // @Description Run @@ -48,13 +48,13 @@ func (r *PipelineRouter) AppendRoutes(g fiber.Router) { // @Id pipelines_run // @Accept json // @Produce json -// @Param body body client.PipelineRunOptions true "Body" +// @Param body body api_client.PipelineRunOptions true "Body" // @Success 200 // @Failure 400 // @Failure 500 // @Router /pipelines/run [post] func (r *PipelineRouter) Run(c *fiber.Ctx) error { - opts := new(client.PipelineRunOptions) + opts := new(api_client.PipelineRunOptions) if err := c.BodyParser(opts); err != nil { return err } diff --git a/conversion/runtime/scheduler.go b/conversion/runtime/scheduler.go index 3d93ead17..9c77f030c 100644 --- a/conversion/runtime/scheduler.go +++ b/conversion/runtime/scheduler.go @@ -14,17 +14,16 @@ import ( "runtime" "time" - "github.com/kouprlabs/voltaserve/conversion/client" + "github.com/kouprlabs/voltaserve/conversion/client/api_client" "github.com/kouprlabs/voltaserve/conversion/infra" "github.com/kouprlabs/voltaserve/conversion/pipeline" ) type Scheduler struct { - pipelineQueue [][]client.PipelineRunOptions + pipelineQueue [][]api_client.PipelineRunOptions pipelineWorkerCount int activePipelineCount int installer *Installer - apiClient *client.APIClient } type SchedulerOptions struct { @@ -44,10 +43,9 @@ func NewDefaultSchedulerOptions() SchedulerOptions { func NewScheduler(opts SchedulerOptions) *Scheduler { return &Scheduler{ - pipelineQueue: make([][]client.PipelineRunOptions, opts.PipelineWorkerCount), + pipelineQueue: make([][]api_client.PipelineRunOptions, opts.PipelineWorkerCount), pipelineWorkerCount: opts.PipelineWorkerCount, installer: opts.Installer, - apiClient: client.NewAPIClient(), } } @@ -60,7 +58,7 @@ func (s *Scheduler) Start() { go s.pipelineWorkerStatus() } -func (s *Scheduler) SchedulePipeline(opts *client.PipelineRunOptions) { +func (s *Scheduler) SchedulePipeline(opts *api_client.PipelineRunOptions) { index := s.choosePipeline() infra.GetLogger().Named(infra.StrScheduler).Infow("👉 choosing", "pipeline", index) s.pipelineQueue[index] = append(s.pipelineQueue[index], *opts) @@ -81,7 +79,7 @@ func (s *Scheduler) choosePipeline() int { func (s *Scheduler) pipelineWorker(index int) { dispatcher := pipeline.NewDispatcher() - s.pipelineQueue[index] = make([]client.PipelineRunOptions, 0) + s.pipelineQueue[index] = make([]api_client.PipelineRunOptions, 0) infra.GetLogger().Named(infra.StrPipeline).Infow("⚙️ running", "worker", index) for { if len(s.pipelineQueue[index]) > 0 && !s.installer.IsRunning() { diff --git a/ui/src/client/api/task.ts b/ui/src/client/api/task.ts index a5fded926..163b8167e 100644 --- a/ui/src/client/api/task.ts +++ b/ui/src/client/api/task.ts @@ -40,7 +40,7 @@ export enum Status { } export type TaskPayload = { - fileId?: string + object?: string } export type List = { diff --git a/ui/src/components/task/task-item.tsx b/ui/src/components/task/task-item.tsx index 8242e1eeb..369cbb482 100644 --- a/ui/src/components/task/task-item.tsx +++ b/ui/src/components/task/task-item.tsx @@ -7,7 +7,6 @@ // the Business Source License, use of this software will be governed // by the GNU Affero General Public License v3.0 only, included in the file // licenses/AGPL.txt. - import { useCallback, useState } from 'react' import { Accordion, @@ -17,14 +16,12 @@ import { AccordionPanel, Card, CardBody, - Text, CircularProgress, IconButton, + Text, } from '@chakra-ui/react' import cx from 'classnames' -import FileAPI from '@/client/api/file' import TaskAPI, { Status, Task } from '@/client/api/task' -import { swrConfig } from '@/client/options' import { IconCheckCircle, IconClose, @@ -41,12 +38,6 @@ export type TaskDrawerItemProps = { const TaskDrawerItem = ({ task }: TaskDrawerItemProps) => { const [isDismissing, setIsDismissing] = useState(false) const mutateList = useAppSelector((state) => state.ui.tasks.mutateList) - const { data: file } = FileAPI.useGet( - task.payload?.fileId, - swrConfig(), - false, - ) - const fileName = file ? file?.name : 'File deleted.' const handleDismiss = useCallback(async () => { try { @@ -83,9 +74,9 @@ const TaskDrawerItem = ({ task }: TaskDrawerItemProps) => { ) : null}
- {fileName ? ( + {task.payload?.object ? ( - {truncateMiddle(fileName, 40)} + {truncateMiddle(task.payload.object, 40)} ) : null} {task.status !== Status.Error ? ( diff --git a/webdav/client/api_client.go b/webdav/client/api_client/file_client.go similarity index 84% rename from webdav/client/api_client.go rename to webdav/client/api_client/file_client.go index 1dbf544d9..5a3eacc30 100644 --- a/webdav/client/api_client.go +++ b/webdav/client/api_client/file_client.go @@ -8,7 +8,7 @@ // by the GNU Affero General Public License v3.0 only, included in the file // licenses/AGPL.txt. -package client +package api_client import ( "bytes" @@ -31,13 +31,13 @@ const ( FileTypeFolder = "folder" ) -type APIClient struct { +type FileClient struct { config *config.Config token *infra.Token } -func NewAPIClient(token *infra.Token) *APIClient { - return &APIClient{ +func NewFileClient(token *infra.Token) *FileClient { + return &FileClient{ token: token, config: config.GetConfig(), } @@ -89,7 +89,7 @@ type FileCreateFolderOptions struct { Name string } -func (cl *APIClient) CreateFolder(opts FileCreateFolderOptions) (*File, error) { +func (cl *FileClient) CreateFolder(opts FileCreateFolderOptions) (*File, error) { params := url.Values{} params.Set("type", opts.Type) params.Set("workspace_id", opts.WorkspaceID) @@ -140,7 +140,7 @@ type FileCreateFromS3Options struct { S3Reference S3Reference } -func (cl *APIClient) CreateFileFromS3(opts FileCreateFromS3Options) (*File, error) { +func (cl *FileClient) CreateFromS3(opts FileCreateFromS3Options) (*File, error) { body, err := json.Marshal(opts) if err != nil { return nil, err @@ -189,7 +189,7 @@ type FilePatchFromS3Options struct { S3Reference S3Reference } -func (cl *APIClient) PatchFileFromS3(opts FilePatchFromS3Options) (*File, error) { +func (cl *FileClient) PatchFromS3(opts FilePatchFromS3Options) (*File, error) { body, err := json.Marshal(opts) if err != nil { return nil, err @@ -230,7 +230,7 @@ func (cl *APIClient) PatchFileFromS3(opts FilePatchFromS3Options) (*File, error) return &file, nil } -func (cl *APIClient) GetFileByPath(path string) (*File, error) { +func (cl *FileClient) GetByPath(path string) (*File, error) { req, err := http.NewRequest("GET", fmt.Sprintf("%s/v2/files?path=%s", cl.config.APIURL, helper.EncodeURIComponent(path)), nil) if err != nil { return nil, err @@ -259,7 +259,7 @@ func (cl *APIClient) GetFileByPath(path string) (*File, error) { return &file, nil } -func (cl *APIClient) ListFilesByPath(path string) ([]File, error) { +func (cl *FileClient) ListByPath(path string) ([]File, error) { req, err := http.NewRequest("GET", fmt.Sprintf("%s/v2/files/list?path=%s", cl.config.APIURL, helper.EncodeURIComponent(path)), nil) if err != nil { return nil, err @@ -288,7 +288,7 @@ func (cl *APIClient) ListFilesByPath(path string) ([]File, error) { return files, nil } -func (cl *APIClient) CopyOne(id string, targetID string) ([]File, error) { +func (cl *FileClient) CopyOne(id string, targetID string) (*File, error) { req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files/%s/copy/%s", cl.config.APIURL, id, targetID), nil) if err != nil { return nil, err @@ -310,14 +310,14 @@ func (cl *APIClient) CopyOne(id string, targetID string) ([]File, error) { if err != nil { return nil, err } - var files []File - if err = json.Unmarshal(body, &files); err != nil { + var file *File + if err = json.Unmarshal(body, &file); err != nil { return nil, err } - return files, nil + return file, nil } -func (cl *APIClient) MoveOne(id string, targetID string) error { +func (cl *FileClient) MoveOne(id string, targetID string) error { req, err := http.NewRequest("POST", fmt.Sprintf("%s/v2/files/%s/move/%s", cl.config.APIURL, id, targetID), nil) if err != nil { return err @@ -338,11 +338,11 @@ func (cl *APIClient) MoveOne(id string, targetID string) error { return cl.successfulResponseOrThrow(resp) } -type FileRenameOptions struct { +type FilePatchNameOptions struct { Name string `json:"name"` } -func (cl *APIClient) PatchFileName(id string, opts FileRenameOptions) (*File, error) { +func (cl *FileClient) PatchName(id string, opts FilePatchNameOptions) (*File, error) { b, err := json.Marshal(opts) if err != nil { return nil, err @@ -375,17 +375,17 @@ func (cl *APIClient) PatchFileName(id string, opts FileRenameOptions) (*File, er return &file, nil } -func (cl *APIClient) DeleteOne(id string) ([]string, error) { +func (cl *FileClient) DeleteOne(id string) error { req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/v2/files/%s", cl.config.APIURL, id), nil) if err != nil { - return nil, err + return err } req.Header.Set("Authorization", "Bearer "+cl.token.AccessToken) req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { - return nil, err + return err } defer func(Body io.ReadCloser) { err := Body.Close() @@ -393,18 +393,10 @@ func (cl *APIClient) DeleteOne(id string) ([]string, error) { infra.GetLogger().Error(err.Error()) } }(resp.Body) - body, err := cl.jsonResponseOrThrow(resp) - if err != nil { - return nil, err - } - var ids []string - if err = json.Unmarshal(body, &ids); err != nil { - return nil, err - } - return ids, nil + return cl.successfulResponseOrThrow(resp) } -func (cl *APIClient) DownloadOriginal(file *File, outputPath string) error { +func (cl *FileClient) DownloadOriginal(file *File, outputPath string) error { resp, err := http.Get(fmt.Sprintf("%s/v2/files/%s/original%s?access_token=%s", cl.config.APIURL, file.ID, file.Snapshot.Original.Extension, cl.token.AccessToken)) if err != nil { return err @@ -429,7 +421,7 @@ func (cl *APIClient) DownloadOriginal(file *File, outputPath string) error { return err } -func (cl *APIClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { +func (cl *FileClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { if strings.HasPrefix(resp.Header.Get("content-type"), "application/json") { body, err := io.ReadAll(resp.Body) if err != nil { @@ -449,7 +441,7 @@ func (cl *APIClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { } } -func (cl *APIClient) successfulResponseOrThrow(resp *http.Response) error { +func (cl *FileClient) successfulResponseOrThrow(resp *http.Response) error { if resp.StatusCode > 299 { body, err := io.ReadAll(resp.Body) if err != nil { @@ -464,31 +456,3 @@ func (cl *APIClient) successfulResponseOrThrow(resp *http.Response) error { return nil } } - -type HealthAPIClient struct { - config *config.Config -} - -func NewHealthAPIClient() *HealthAPIClient { - return &HealthAPIClient{ - config: config.GetConfig(), - } -} - -func (cl *HealthAPIClient) GetHealth() (string, error) { - resp, err := http.Get(fmt.Sprintf("%s/v2/health", cl.config.IdPURL)) - if err != nil { - return "", err - } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - infra.GetLogger().Error(err.Error()) - } - }(resp.Body) - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - return string(body), nil -} diff --git a/webdav/client/api_client/health_client.go b/webdav/client/api_client/health_client.go new file mode 100644 index 000000000..eda7f4f16 --- /dev/null +++ b/webdav/client/api_client/health_client.go @@ -0,0 +1,48 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package api_client + +import ( + "fmt" + "io" + "net/http" + + "github.com/kouprlabs/voltaserve/webdav/config" + "github.com/kouprlabs/voltaserve/webdav/infra" +) + +type HealthClient struct { + config *config.Config +} + +func NewHealthClient() *HealthClient { + return &HealthClient{ + config: config.GetConfig(), + } +} + +func (cl *HealthClient) Get() (string, error) { + resp, err := http.Get(fmt.Sprintf("%s/v2/health", cl.config.IdPURL)) + if err != nil { + return "", err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + infra.GetLogger().Error(err.Error()) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} diff --git a/webdav/client/idp_client/health_client.go b/webdav/client/idp_client/health_client.go new file mode 100644 index 000000000..cda80ceeb --- /dev/null +++ b/webdav/client/idp_client/health_client.go @@ -0,0 +1,48 @@ +// Copyright 2023 Anass Bouassaba. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the GNU Affero General Public License v3.0 only, included in the file +// licenses/AGPL.txt. + +package idp_client + +import ( + "fmt" + "io" + "net/http" + + "github.com/kouprlabs/voltaserve/webdav/config" + "github.com/kouprlabs/voltaserve/webdav/infra" +) + +type HealthClient struct { + config *config.Config +} + +func NewHealthClient() *HealthClient { + return &HealthClient{ + config: config.GetConfig(), + } +} + +func (cl *HealthClient) Get() (string, error) { + resp, err := http.Get(fmt.Sprintf("%s/v2/health", cl.config.IdPURL)) + if err != nil { + return "", err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + infra.GetLogger().Error(err.Error()) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} diff --git a/webdav/client/idp_client.go b/webdav/client/idp_client/token_client.go similarity index 75% rename from webdav/client/idp_client.go rename to webdav/client/idp_client/token_client.go index 4e47f733e..4210d4450 100644 --- a/webdav/client/idp_client.go +++ b/webdav/client/idp_client/token_client.go @@ -8,7 +8,7 @@ // by the GNU Affero General Public License v3.0 only, included in the file // licenses/AGPL.txt. -package client +package idp_client import ( "bytes" @@ -38,17 +38,17 @@ type TokenExchangeOptions struct { Locale string `json:"locale,omitempty"` } -type IdPClient struct { +type TokenClient struct { config *config.Config } -func NewIdPClient() *IdPClient { - return &IdPClient{ +func NewTokenClient() *TokenClient { + return &TokenClient{ config: config.GetConfig(), } } -func (cl *IdPClient) Exchange(options TokenExchangeOptions) (*infra.Token, error) { +func (cl *TokenClient) Exchange(options TokenExchangeOptions) (*infra.Token, error) { form := url.Values{} form.Set("grant_type", options.GrantType) if options.Username != "" { @@ -87,7 +87,7 @@ func (cl *IdPClient) Exchange(options TokenExchangeOptions) (*infra.Token, error return &token, nil } -func (cl *IdPClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { +func (cl *TokenClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { if strings.HasPrefix(resp.Header.Get("content-type"), "application/json") { body, err := io.ReadAll(resp.Body) if err != nil { @@ -107,31 +107,3 @@ func (cl *IdPClient) jsonResponseOrThrow(resp *http.Response) ([]byte, error) { return nil, errors.New("unexpected response format") } } - -type HealthIdPClient struct { - config *config.Config -} - -func NewHealthIdPClient() *HealthIdPClient { - return &HealthIdPClient{ - config: config.GetConfig(), - } -} - -func (cl *HealthIdPClient) GetHealth() (string, error) { - resp, err := http.Get(fmt.Sprintf("%s/v2/health", cl.config.IdPURL)) - if err != nil { - return "", err - } - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - infra.GetLogger().Error(err.Error()) - } - }(resp.Body) - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - return string(body), nil -} diff --git a/webdav/handler/handler.go b/webdav/handler/handler.go index 57dd7193e..efb46fb38 100644 --- a/webdav/handler/handler.go +++ b/webdav/handler/handler.go @@ -14,7 +14,8 @@ import ( "net/http" "github.com/kouprlabs/voltaserve/webdav/cache" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" + "github.com/kouprlabs/voltaserve/webdav/client/idp_client" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -58,14 +59,14 @@ func (h *Handler) Dispatch(w http.ResponseWriter, r *http.Request) { } func (h *Handler) Health(w http.ResponseWriter, _ *http.Request) { - apiClient := client.NewHealthAPIClient() - apiHealth, err := apiClient.GetHealth() + apiClient := api_client.NewHealthClient() + apiHealth, err := apiClient.Get() if err != nil { w.WriteHeader(http.StatusInternalServerError) return } - idpClient := client.NewHealthIdPClient() - idpHealth, err := idpClient.GetHealth() + idpClient := idp_client.NewHealthClient() + idpHealth, err := idpClient.Get() if err != nil { w.WriteHeader(http.StatusInternalServerError) return diff --git a/webdav/handler/method_copy.go b/webdav/handler/method_copy.go index 00acc5630..009896a31 100644 --- a/webdav/handler/method_copy.go +++ b/webdav/handler/method_copy.go @@ -15,7 +15,7 @@ import ( "net/http" "path" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -36,16 +36,16 @@ func (h *Handler) methodCopy(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) + cl := api_client.NewFileClient(token) sourcePath := helper.DecodeURIComponent(r.URL.Path) targetPath := helper.DecodeURIComponent(helper.GetTargetPath(r)) - sourceFile, err := apiClient.GetFileByPath(sourcePath) + sourceFile, err := cl.GetByPath(sourcePath) if err != nil { infra.HandleError(err, w) return } targetDir := helper.DecodeURIComponent(helper.Dirname(helper.GetTargetPath(r))) - targetFile, err := apiClient.GetFileByPath(targetDir) + targetFile, err := cl.GetByPath(targetDir) if err != nil { infra.HandleError(err, w) return @@ -56,14 +56,12 @@ func (h *Handler) methodCopy(w http.ResponseWriter, r *http.Request) { return } } else { - clones, err := apiClient.CopyOne(targetFile.ID, sourceFile.ID) + clone, err := cl.CopyOne(sourceFile.ID, targetFile.ID) if err != nil { infra.HandleError(err, w) return } - if _, err = apiClient.PatchFileName(clones[0].ID, client.FileRenameOptions{ - Name: path.Base(targetPath), - }); err != nil { + if _, err = cl.PatchName(clone.ID, api_client.FilePatchNameOptions{Name: path.Base(targetPath)}); err != nil { infra.HandleError(err, w) return } diff --git a/webdav/handler/method_delete.go b/webdav/handler/method_delete.go index 77bc07ab1..51a9351e9 100644 --- a/webdav/handler/method_delete.go +++ b/webdav/handler/method_delete.go @@ -14,7 +14,7 @@ import ( "fmt" "net/http" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -25,7 +25,7 @@ This method deletes a resource identified by the URL. Example implementation: - Extract the file path from the URL. -- Use fs.unlink() to delete the file. +- Delete the file. - Set the response status code to 204 if successful or an appropriate error code if the file is not found. - Return the response. */ @@ -35,13 +35,13 @@ func (h *Handler) methodDelete(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) - file, err := apiClient.GetFileByPath(helper.DecodeURIComponent(r.URL.Path)) + cl := api_client.NewFileClient(token) + file, err := cl.GetByPath(helper.DecodeURIComponent(r.URL.Path)) if err != nil { infra.HandleError(err, w) return } - if _, err = apiClient.DeleteOne(file.ID); err != nil { + if err = cl.DeleteOne(file.ID); err != nil { infra.HandleError(err, w) return } diff --git a/webdav/handler/method_get.go b/webdav/handler/method_get.go index ca6eba840..0b3a6a6c4 100644 --- a/webdav/handler/method_get.go +++ b/webdav/handler/method_get.go @@ -21,7 +21,7 @@ import ( "github.com/google/uuid" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -42,15 +42,15 @@ func (h *Handler) methodGet(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) - filePath := helper.DecodeURIComponent(r.URL.Path) - file, err := apiClient.GetFileByPath(filePath) + cl := api_client.NewFileClient(token) + inputPath := helper.DecodeURIComponent(r.URL.Path) + file, err := cl.GetByPath(inputPath) if err != nil { infra.HandleError(err, w) return } outputPath := filepath.Join(os.TempDir(), uuid.New().String()) - err = apiClient.DownloadOriginal(file, outputPath) + err = cl.DownloadOriginal(file, outputPath) if err != nil { infra.HandleError(err, w) return diff --git a/webdav/handler/method_head.go b/webdav/handler/method_head.go index c0c7885ae..bfa5cfc2e 100644 --- a/webdav/handler/method_head.go +++ b/webdav/handler/method_head.go @@ -14,7 +14,7 @@ import ( "fmt" "net/http" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -25,7 +25,7 @@ This method is similar to GET but only retrieves the metadata of a resource, wit Example implementation: - Extract the file path from the URL. -- Retrieve the file metadata using fs.stat(). +- Retrieve the file metadata. - Set the response status code to 200 if successful or an appropriate error code if the file is not found. - Set the Content-Length header with the file size. - Return the response. @@ -36,14 +36,14 @@ func (h *Handler) methodHead(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) - filePath := helper.DecodeURIComponent(r.URL.Path) - file, err := apiClient.GetFileByPath(filePath) + cl := api_client.NewFileClient(token) + inputPath := helper.DecodeURIComponent(r.URL.Path) + file, err := cl.GetByPath(inputPath) if err != nil { infra.HandleError(err, w) return } - if file.Type == client.FileTypeFile { + if file.Type == api_client.FileTypeFile { w.Header().Set("Content-Length", fmt.Sprintf("%d", file.Snapshot.Original.Size)) } w.WriteHeader(http.StatusOK) diff --git a/webdav/handler/method_mkcol.go b/webdav/handler/method_mkcol.go index 23222a26f..248124fde 100644 --- a/webdav/handler/method_mkcol.go +++ b/webdav/handler/method_mkcol.go @@ -15,7 +15,7 @@ import ( "net/http" "path" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -26,7 +26,7 @@ This method creates a new collection (directory) at the specified URL. Example implementation: - Extract the directory path from the URL. -- Use fs.mkdir() to create the directory. +- Create the directory. - Set the response status code to 201 if created or an appropriate error code if the directory already exists or encountered an error. - Return the response. */ @@ -36,21 +36,25 @@ func (h *Handler) methodMkcol(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) - directoryPath := helper.DecodeURIComponent(helper.Dirname(r.URL.Path)) - directory, err := apiClient.GetFileByPath(directoryPath) + cl := api_client.NewFileClient(token) + wantedPath := helper.DecodeURIComponent(helper.Dirname(r.URL.Path)) + directory, err := cl.GetByPath(wantedPath) if err != nil { infra.HandleError(err, w) return } - if _, err = apiClient.CreateFolder(client.FileCreateFolderOptions{ - Type: client.FileTypeFolder, - WorkspaceID: directory.WorkspaceID, - ParentID: directory.ID, - Name: helper.DecodeURIComponent(path.Base(r.URL.Path)), - }); err != nil { - infra.HandleError(err, w) - return + if directory.Name != "/" && directory.WorkspaceID != "" { + if _, err = cl.CreateFolder(api_client.FileCreateFolderOptions{ + Type: api_client.FileTypeFolder, + WorkspaceID: directory.WorkspaceID, + ParentID: directory.ID, + Name: helper.DecodeURIComponent(path.Base(r.URL.Path)), + }); err != nil { + infra.HandleError(err, w) + return + } + w.WriteHeader(http.StatusCreated) + } else { + w.WriteHeader(http.StatusMethodNotAllowed) } - w.WriteHeader(http.StatusCreated) } diff --git a/webdav/handler/method_move.go b/webdav/handler/method_move.go index 6534089ab..c71acc014 100644 --- a/webdav/handler/method_move.go +++ b/webdav/handler/method_move.go @@ -16,7 +16,7 @@ import ( "path" "strings" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -27,7 +27,7 @@ This method moves or renames a resource from a source URL to a destination URL. Example implementation: - Extract the source and destination paths from the headers or request body. -- Use fs.rename() to move or rename the file from the source to the destination. +- Move or rename the file from the source to the destination. - Set the response status code to 204 if successful or an appropriate error code if the source file is not found or encountered an error. - Return the response. */ @@ -37,16 +37,16 @@ func (h *Handler) methodMove(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) + cl := api_client.NewFileClient(token) sourcePath := helper.DecodeURIComponent(r.URL.Path) targetPath := helper.DecodeURIComponent(helper.GetTargetPath(r)) - sourceFile, err := apiClient.GetFileByPath(sourcePath) + sourceFile, err := cl.GetByPath(sourcePath) if err != nil { infra.HandleError(err, w) return } targetDir := helper.DecodeURIComponent(helper.Dirname(helper.GetTargetPath(r))) - targetFile, err := apiClient.GetFileByPath(targetDir) + targetFile, err := cl.GetByPath(targetDir) if err != nil { infra.HandleError(err, w) return @@ -61,14 +61,14 @@ func (h *Handler) methodMove(w http.ResponseWriter, r *http.Request) { sourcePathParts := strings.Split(sourcePath, "/") targetPathParts := strings.Split(targetPath, "/") if len(sourcePathParts) == len(targetPathParts) && helper.Dirname(sourcePath) == helper.Dirname(targetPath) { - if _, err := apiClient.PatchFileName(sourceFile.ID, client.FileRenameOptions{ + if _, err := cl.PatchName(sourceFile.ID, api_client.FilePatchNameOptions{ Name: helper.DecodeURIComponent(path.Base(targetPath)), }); err != nil { infra.HandleError(err, w) return } } else { - if err := apiClient.MoveOne(targetFile.ID, sourceFile.ID); err != nil { + if err := cl.MoveOne(sourceFile.ID, targetFile.ID); err != nil { infra.HandleError(err, w) return } diff --git a/webdav/handler/method_propfind.go b/webdav/handler/method_propfind.go index 60c56f416..184170bdb 100644 --- a/webdav/handler/method_propfind.go +++ b/webdav/handler/method_propfind.go @@ -14,7 +14,7 @@ import ( "fmt" "net/http" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -25,7 +25,7 @@ This method retrieves properties and metadata of a resource. Example implementation: - Extract the file path from the URL. -- Use fs.stat() to retrieve the file metadata. +- Retrieve the file metadata. - Format the response body in the desired XML format with the properties and metadata. - Set the response status code to 207 if successful or an appropriate error code if the file is not found or encountered an error. - Set the Content-Type header to indicate the XML format. @@ -37,13 +37,13 @@ func (h *Handler) methodPropfind(w http.ResponseWriter, r *http.Request) { infra.HandleError(fmt.Errorf("missing token"), w) return } - apiClient := client.NewAPIClient(token) - file, err := apiClient.GetFileByPath(helper.DecodeURIComponent(r.URL.Path)) + cl := api_client.NewFileClient(token) + file, err := cl.GetByPath(helper.DecodeURIComponent(r.URL.Path)) if err != nil { infra.HandleError(err, w) return } - if file.Type == client.FileTypeFile { + if file.Type == api_client.FileTypeFile { responseXml := fmt.Sprintf( ` @@ -61,7 +61,7 @@ func (h *Handler) methodPropfind(w http.ResponseWriter, r *http.Request) { `, helper.EncodeURIComponent(file.Name), func() int { - if file.Type == client.FileTypeFile && file.Snapshot != nil && file.Snapshot.Original != nil { + if file.Type == api_client.FileTypeFile && file.Snapshot != nil && file.Snapshot.Original != nil { return file.Snapshot.Original.Size } return 0 @@ -75,7 +75,7 @@ func (h *Handler) methodPropfind(w http.ResponseWriter, r *http.Request) { infra.HandleError(err, w) return } - } else if file.Type == client.FileTypeFolder { + } else if file.Type == api_client.FileTypeFolder { responseXml := fmt.Sprintf( ` @@ -94,7 +94,7 @@ func (h *Handler) methodPropfind(w http.ResponseWriter, r *http.Request) { helper.ToUTCString(file.UpdateTime), helper.ToUTCString(&file.CreateTime), ) - list, err := apiClient.ListFilesByPath(helper.DecodeURIComponent(r.URL.Path)) + list, err := cl.ListByPath(helper.DecodeURIComponent(r.URL.Path)) if err != nil { infra.HandleError(err, w) return @@ -115,13 +115,13 @@ func (h *Handler) methodPropfind(w http.ResponseWriter, r *http.Request) { `, helper.EncodeURIComponent(r.URL.Path+item.Name), func() string { - if item.Type == client.FileTypeFolder { + if item.Type == api_client.FileTypeFolder { return "" } return "" }(), func() int { - if item.Type == client.FileTypeFile && item.Snapshot != nil && item.Snapshot.Original != nil { + if item.Type == api_client.FileTypeFile && item.Snapshot != nil && item.Snapshot.Original != nil { return item.Snapshot.Original.Size } return 0 diff --git a/webdav/handler/method_put.go b/webdav/handler/method_put.go index 3765bea83..296c4586c 100644 --- a/webdav/handler/method_put.go +++ b/webdav/handler/method_put.go @@ -23,7 +23,7 @@ import ( "github.com/google/uuid" "github.com/minio/minio-go/v7" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/api_client" "github.com/kouprlabs/voltaserve/webdav/helper" "github.com/kouprlabs/voltaserve/webdav/infra" ) @@ -51,8 +51,8 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) return } - apiClient := client.NewAPIClient(token) - directory, err := apiClient.GetFileByPath(helper.DecodeURIComponent(helper.Dirname(r.URL.Path))) + cl := api_client.NewFileClient(token) + directory, err := cl.GetByPath(helper.DecodeURIComponent(helper.Dirname(r.URL.Path))) if err != nil { infra.HandleError(err, w) return @@ -94,16 +94,16 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { infra.HandleError(err, w) return } - s3Reference := client.S3Reference{ + s3Reference := api_client.S3Reference{ Bucket: workspace.Bucket, Key: key, SnapshotID: snapshotID, Size: stat.Size(), ContentType: infra.DetectMimeFromPath(outputPath), } - existingFile, err := apiClient.GetFileByPath(r.URL.Path) + existingFile, err := cl.GetByPath(r.URL.Path) if err == nil { - if _, err = apiClient.PatchFileFromS3(client.FilePatchFromS3Options{ + if _, err = cl.PatchFromS3(api_client.FilePatchFromS3Options{ ID: existingFile.ID, Name: name, S3Reference: s3Reference, @@ -114,8 +114,8 @@ func (h *Handler) methodPut(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) return } else { - if _, err = apiClient.CreateFileFromS3(client.FileCreateFromS3Options{ - Type: client.FileTypeFile, + if _, err = cl.CreateFromS3(api_client.FileCreateFromS3Options{ + Type: api_client.FileTypeFile, WorkspaceID: directory.WorkspaceID, ParentID: directory.ID, Name: name, diff --git a/webdav/main.go b/webdav/main.go index c8ed98302..9ecff8d2f 100644 --- a/webdav/main.go +++ b/webdav/main.go @@ -22,7 +22,7 @@ import ( "github.com/joho/godotenv" - "github.com/kouprlabs/voltaserve/webdav/client" + "github.com/kouprlabs/voltaserve/webdav/client/idp_client" "github.com/kouprlabs/voltaserve/webdav/config" "github.com/kouprlabs/voltaserve/webdav/handler" "github.com/kouprlabs/voltaserve/webdav/helper" @@ -35,7 +35,7 @@ var ( mu sync.RWMutex ) -func startTokenRefresh(idpClient *client.IdPClient) { +func startTokenRefresh(idpClient *idp_client.TokenClient) { ticker := time.NewTicker(5 * time.Second) go func() { for { @@ -44,8 +44,8 @@ func startTokenRefresh(idpClient *client.IdPClient) { for username, token := range tokens { expiry := expiries[username] if time.Now().After(expiry.Add(-1 * time.Minute)) { - newToken, err := idpClient.Exchange(client.TokenExchangeOptions{ - GrantType: client.GrantTypeRefreshToken, + newToken, err := idpClient.Exchange(idp_client.TokenExchangeOptions{ + GrantType: idp_client.GrantTypeRefreshToken, RefreshToken: token.RefreshToken, }) if err == nil { @@ -59,7 +59,7 @@ func startTokenRefresh(idpClient *client.IdPClient) { }() } -func basicAuthMiddleware(next http.Handler, idpClient *client.IdPClient) http.Handler { +func basicAuthMiddleware(next http.Handler, idpClient *idp_client.TokenClient) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { username, password, ok := r.BasicAuth() if !ok { @@ -72,8 +72,8 @@ func basicAuthMiddleware(next http.Handler, idpClient *client.IdPClient) http.Ha token, exists := tokens[username] if !exists { var err error - token, err = idpClient.Exchange(client.TokenExchangeOptions{ - GrantType: client.GrantTypePassword, + token, err = idpClient.Exchange(idp_client.TokenExchangeOptions{ + GrantType: idp_client.GrantTypePassword, Username: username, Password: password, }) @@ -106,7 +106,7 @@ func main() { cfg := config.GetConfig() - idpClient := client.NewIdPClient() + idpClient := idp_client.NewTokenClient() h := handler.NewHandler() mux := http.NewServeMux()