perf: fast delete (#211)

* perf: fast delete * chore(api): generate docs * fix(api): swagger comment * chore(api): generate docs * refactor(api): change order of routes * chore(api): generate docs * fix(api): update api/repo/file_repo.go Co-authored-by: Daniël Sonck <[email protected]> * fix(api): query * chore(api): format code --------- Co-authored-by: Daniël Sonck <[email protected]>
kouprlabs · Jul 23, 2024 · 465d178 · 465d178
1 parent 208fc94
commit 465d178
Show file tree

Hide file tree

Showing 10 changed files with 271 additions and 130 deletions.
diff --git a/api/docs/index.html b/api/docs/index.html
diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml
@@ -98,15 +98,6 @@ definitions:
       size:
         type: integer
     type: object
-  router.FileDeleteOptions:
-    properties:
-      ids:
-        items:
-          type: string
-        type: array
-    required:
-    - ids
-    type: object
   router.FileGrantGroupPermissionOptions:
     properties:
       groupId:
@@ -282,6 +273,15 @@ definitions:
           type: string
         type: array
     type: object
+  service.FileDeleteManyOptions:
+    properties:
+      sourceIds:
+        items:
+          type: string
+        type: array
+    required:
+    - sourceIds
+    type: object
   service.FileList:
     properties:
       data:
@@ -818,15 +818,15 @@ info:
 paths:
   /files:
     delete:
-      description: Delete
-      operationId: files_delete
+      description: Delete Many
+      operationId: files_delete_many
       parameters:
       - description: Body
         in: body
         name: body
         required: true
         schema:
-          $ref: '#/definitions/router.FileDeleteOptions'
+          $ref: '#/definitions/service.FileDeleteManyOptions'
       produces:
       - application/json
       responses:
@@ -840,7 +840,7 @@ paths:
           description: Internal Server Error
           schema:
             $ref: '#/definitions/errorpkg.ErrorResponse'
-      summary: Delete
+      summary: Delete Many
       tags:
       - Files
     get:
@@ -922,6 +922,34 @@ paths:
       tags:
       - Files
   /files/{id}:
+    delete:
+      description: Delete One
+      operationId: files_delete_one
+      parameters:
+      - description: ID
+        in: path
+        name: id
+        required: true
+        type: string
+      - description: Target ID
+        in: path
+        name: targetId
+        required: true
+        type: string
+      produces:
+      - application/json
+      responses:
+        "404":
+          description: Not Found
+          schema:
+            $ref: '#/definitions/errorpkg.ErrorResponse'
+        "500":
+          description: Internal Server Error
+          schema:
+            $ref: '#/definitions/errorpkg.ErrorResponse'
+      summary: Delete One
+      tags:
+      - Files
     get:
       description: Get
       operationId: files_get

diff --git a/api/repo/file_repo.go b/api/repo/file_repo.go
@@ -28,6 +28,7 @@ type FileRepo interface {
 	FindPath(id string) ([]model.File, error)
 	FindTree(id string) ([]model.File, error)
 	FindTreeIDs(id string) ([]string, error)
+	DeleteChunk(ids []string) error
 	Count() (int64, error)
 	GetIDsByWorkspace(workspaceID string) ([]string, error)
 	GetIDsBySnapshot(snapshotID string) ([]string, error)
@@ -339,6 +340,13 @@ func (repo *fileRepo) FindTreeIDs(id string) ([]string, error) {
 	return res, nil
 }
 
+func (repo *fileRepo) DeleteChunk(ids []string) error {
+	if db := repo.db.Delete(&fileEntity{}, ids); db.Error != nil {
+		return db.Error
+	}
+	return nil
+}
+
 func (repo *fileRepo) Count() (int64, error) {
 	type Result struct {
 		Result int64

diff --git a/api/repo/snapshot_repo.go b/api/repo/snapshot_repo.go
@@ -38,6 +38,7 @@ type SnapshotRepo interface {
 	MapWithFile(id string, fileID string) error
 	BulkMapWithFile(entities []*SnapshotFileEntity, chunkSize int) error
 	DeleteMappingsForFile(fileID string) error
+	DeleteMappingsForTree(fileID string) error
 	DeleteAllDangling() error
 	GetLatestVersionForFile(fileID string) (int64, error)
 	CountAssociations(id string) (int, error)
@@ -536,6 +537,19 @@ func (repo *snapshotRepo) findAllForFile(fileID string) ([]*snapshotEntity, erro
 	return res, nil
 }
 
+func (repo *snapshotRepo) DeleteMappingsForTree(fileID string) error {
+	db := repo.db.
+		Exec(`WITH RECURSIVE rec (id, parent_id, create_time) AS
+              (SELECT f.id, f.parent_id, f.create_time FROM file f WHERE f.parent_id = ?
+              UNION SELECT f.id, f.parent_id, f.create_time FROM rec, file f WHERE f.parent_id = rec.id)
+              DELETE FROM snapshot_file WHERE file_id in (SELECT id FROM rec);`,
+			fileID)
+	if db.Error != nil {
+		return db.Error
+	}
+	return nil
+}
+
 func (repo *snapshotRepo) FindAllForFile(fileID string) ([]model.Snapshot, error) {
 	snapshots, err := repo.findAllForFile(fileID)
 	if err != nil {

diff --git a/api/router/file_router.go b/api/router/file_router.go
@@ -65,15 +65,16 @@ func (r *FileRouter) AppendRoutes(g fiber.Router) {
 	g.Post("/move", r.MoveMany)
 	g.Post("/copy", r.CopyMany)
 	g.Get("/", r.GetByPath)
-	g.Delete("/", r.Delete)
+	g.Delete("/", r.DeleteMany)
 	g.Get("/:id", r.Get)
 	g.Patch("/:id", r.Patch)
 	g.Get("/:id/list", r.List)
 	g.Get("/:id/count", r.GetCount)
 	g.Get("/:id/path", r.GetPath)
+	g.Delete("/:id", r.DeleteOne)
 	g.Post("/:id/move/:targetId", r.MoveOne)
-	g.Patch("/:id/name", r.PatchName)
 	g.Post("/:id/copy/:targetId", r.CopyOne)
+	g.Patch("/:id/name", r.PatchName)
 	g.Get("/:id/size", r.GetSize)
 	g.Post("/grant_user_permission", r.GrantUserPermission)
 	g.Post("/revoke_user_permission", r.RevokeUserPermission)
@@ -567,27 +568,47 @@ type FileDeleteOptions struct {
 	IDs []string `json:"ids" validate:"required"`
 }
 
-// Delete godoc
+// DeleteOne godoc
+//
+//	@Summary		Delete One
+//	@Description	Delete One
+//	@Tags			Files
+//	@Id				files_delete_one
+//	@Produce		json
+//	@Param			id			path		string	true	"ID"
+//	@Param			targetId	path		string	true	"Target ID"
+//	@Failure		404			{object}	errorpkg.ErrorResponse
+//	@Failure		500			{object}	errorpkg.ErrorResponse
+//	@Router			/files/{id} [delete]
+func (r *FileRouter) DeleteOne(c *fiber.Ctx) error {
+	userID := GetUserID(c)
+	if err := r.fileSvc.DeleteOne(c.Params("id"), userID); err != nil {
+		return err
+	}
+	return c.JSON(http.StatusNoContent)
+}
+
+// DeleteMany godoc
 //
-//	@Summary		Delete
-//	@Description	Delete
+//	@Summary		Delete Many
+//	@Description	Delete Many
 //	@Tags			Files
-//	@Id				files_delete
+//	@Id				files_delete_many
 //	@Produce		json
-//	@Param			body	body		FileDeleteOptions	true	"Body"
+//	@Param			body	body		service.FileDeleteManyOptions	true	"Body"
 //	@Success		200		{array}		string
 //	@Failure		500		{object}	errorpkg.ErrorResponse
 //	@Router			/files [delete]
-func (r *FileRouter) Delete(c *fiber.Ctx) error {
+func (r *FileRouter) DeleteMany(c *fiber.Ctx) error {
 	userID := GetUserID(c)
-	opts := new(FileDeleteOptions)
+	opts := new(service.FileDeleteManyOptions)
 	if err := c.BodyParser(opts); err != nil {
 		return err
 	}
 	if err := validator.New().Struct(opts); err != nil {
 		return errorpkg.NewRequestBodyValidationError(err)
 	}
-	res, err := r.fileSvc.Delete(opts.IDs, userID)
+	res, err := r.fileSvc.DeleteMany(*opts, userID)
 	if err != nil {
 		return err
 	}