Skip to content

Commit

Permalink
PLT-???? Prepare file upload infrastructure for Data Retention. (matt…
Browse files Browse the repository at this point in the history
…ermost#7266)

* Prepare file upload infrastructure for Data Retention.

This commit prepares the file upload infrastructure for the data
retention feature that is under construction. Changes are:

* Move file management code to utils to allow access to it from jobs.

* From now on, store all file uploads in a top level folder which is the
  date of the day on which they were uploaded.

This commit is based on Harrison Healey's branch, but updated to work
with the latest master.

* Use NewAppError
grundleborg authored and hmhealey committed Aug 25, 2017
1 parent 99acf61 commit 50fc6e1
Showing 20 changed files with 678 additions and 299 deletions.
2 changes: 1 addition & 1 deletion api/emoji_test.go
Original file line number Diff line number Diff line change
@@ -266,7 +266,7 @@ func TestDeleteEmoji(t *testing.T) {
func createTestEmoji(t *testing.T, emoji *model.Emoji, imageData []byte) *model.Emoji {
emoji = store.Must(app.Srv.Store.Emoji().Save(emoji)).(*model.Emoji)

if err := app.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
if err := utils.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
store.Must(app.Srv.Store.Emoji().Delete(emoji.Id, time.Now().Unix()))
t.Fatalf("failed to write image: %v", err.Error())
}
10 changes: 5 additions & 5 deletions api/file.go
Original file line number Diff line number Diff line change
@@ -96,7 +96,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.Path); err != nil {
if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -118,7 +118,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.ThumbnailPath); err != nil {
if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, w, r); err != nil {
@@ -140,7 +140,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.PreviewPath); err != nil {
if data, err := utils.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, w, r); err != nil {
@@ -190,7 +190,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.Path); err != nil {
if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -285,7 +285,7 @@ func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.Path); err != nil {
if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
53 changes: 45 additions & 8 deletions api/file_test.go
Original file line number Diff line number Diff line change
@@ -81,20 +81,22 @@ func TestUploadFile(t *testing.T) {
t.Fatal("file preview path should be set in database")
}

date := time.Now().Format("20060102")

// This also makes sure that the relative path provided above is sanitized out
expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", team.Id, channel.Id, user.Id, info.Id)
expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, team.Id, channel.Id, user.Id, info.Id)
if info.Path != expectedPath {
t.Logf("file is saved in %v", info.Path)
t.Fatalf("file should've been saved in %v", expectedPath)
}

expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", team.Id, channel.Id, user.Id, info.Id)
expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, team.Id, channel.Id, user.Id, info.Id)
if info.ThumbnailPath != expectedThumbnailPath {
t.Logf("file thumbnail is saved in %v", info.ThumbnailPath)
t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath)
}

expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", team.Id, channel.Id, user.Id, info.Id)
expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, team.Id, channel.Id, user.Id, info.Id)
if info.PreviewPath != expectedPreviewPath {
t.Logf("file preview is saved in %v", info.PreviewPath)
t.Fatalf("file preview should've been saved in %v", expectedPreviewPath)
@@ -466,15 +468,23 @@ func TestGetPublicFileOld(t *testing.T) {
utils.Cfg.FileSettings.EnablePublicLink = true
*utils.Cfg.FileSettings.PublicLinkSalt = model.NewId()

Client := th.BasicClient
channel := th.BasicChannel

var fileId string
data, err := readTestFile("test.png")
if err != nil {
t.Fatal(err)
} else {
fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
//fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
fileId = model.NewId()
fileInfo := model.FileInfo{
Id: fileId,
CreateAt: model.GetMillis(),
CreatorId: th.BasicUser.Id,
Path: fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId, "test.png"),
}
store.Must(app.Srv.Store.FileInfo().Save(&fileInfo))
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId), "test.png")
}

// Hacky way to assign file to a post (usually would be done by CreatePost call)
@@ -619,7 +629,9 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId1), "test.png")
fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId2), "test.png")
}

// Bypass the Client whenever possible since we're trying to simulate a pre-3.5 post
@@ -686,6 +698,25 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) {
}
}

func uploadFileOld(t *testing.T, data []byte, dest string, filename string) {
os.MkdirAll(dest, os.ModePerm)
eFile, err := os.Create(dest + "/" + filename)
if err != nil {
t.Fatal(err)
}
defer eFile.Close()

_, err = io.Copy(eFile, bytes.NewReader(data)) // first var shows number of bytes
if err != nil {
t.Fatal(err)
}

err = eFile.Sync()
if err != nil {
t.Fatal(err)
}
}

func TestFindTeamIdForFilename(t *testing.T) {
th := Setup().InitBasic()

@@ -717,9 +748,11 @@ func TestFindTeamIdForFilename(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png")

Client.SetTeamId(team2.Id)
fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel2.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team2.Id, channel2.Id, user1.Id, fileId2), "test.png")
Client.SetTeamId(team1.Id)
}

@@ -732,6 +765,7 @@ func TestFindTeamIdForFilename(t *testing.T) {
})).(*model.Post)

if teamId := app.FindTeamIdForFilename(post1, post1.Filenames[0]); teamId != team1.Id {
t.Log(teamId)
t.Fatal("file should've been found under team1")
}

@@ -773,6 +807,7 @@ func TestGetInfoForFilename(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png")
path = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).Path
thumbnailPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).ThumbnailPath
previewPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).PreviewPath
@@ -786,6 +821,8 @@ func TestGetInfoForFilename(t *testing.T) {
Filenames: []string{fmt.Sprintf("/%s/%s/%s/%s", channel1.Id, user1.Id, fileId1, "test.png")},
})).(*model.Post)

date := time.Now().Format("20060102")

if info := app.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil {
t.Fatal("info shouldn't be nil")
} else if info.Id == "" {
@@ -794,11 +831,11 @@ func TestGetInfoForFilename(t *testing.T) {
t.Fatal("incorrect user id")
} else if info.PostId != post1.Id {
t.Fatal("incorrect user id")
} else if info.Path != path {
} else if fmt.Sprintf("%s/%s", date, info.Path) != path {
t.Fatal("incorrect path")
} else if info.ThumbnailPath != thumbnailPath {
} else if fmt.Sprintf("%s/%s", date, info.ThumbnailPath) != thumbnailPath {
t.Fatal("incorrect thumbnail path")
} else if info.PreviewPath != previewPath {
} else if fmt.Sprintf("%s/%s", date, info.PreviewPath) != previewPath {
t.Fatal("incorrect preview path")
} else if info.Name != "test.png" {
t.Fatal("incorrect name")
8 changes: 4 additions & 4 deletions api4/file.go
Original file line number Diff line number Diff line change
@@ -123,7 +123,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

data, err := app.ReadFile(info.Path)
data, err := utils.ReadFile(info.Path)
if err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
@@ -165,7 +165,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.ThumbnailPath); err != nil {
if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -237,7 +237,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.PreviewPath); err != nil {
if data, err := utils.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -299,7 +299,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}

if data, err := app.ReadFile(info.Path); err != nil {
if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, true, w, r); err != nil {
8 changes: 5 additions & 3 deletions api4/file_test.go
Original file line number Diff line number Diff line change
@@ -71,20 +71,22 @@ func TestUploadFile(t *testing.T) {
t.Fatal("file preview path should be set in database")
}

date := time.Now().Format("20060102")

// This also makes sure that the relative path provided above is sanitized out
expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.Path != expectedPath {
t.Logf("file is saved in %v", info.Path)
t.Fatalf("file should've been saved in %v", expectedPath)
}

expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.ThumbnailPath != expectedThumbnailPath {
t.Logf("file thumbnail is saved in %v", info.ThumbnailPath)
t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath)
}

expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.PreviewPath != expectedPreviewPath {
t.Logf("file preview is saved in %v", info.PreviewPath)
t.Fatalf("file preview should've been saved in %v", expectedPreviewPath)
10 changes: 5 additions & 5 deletions app/emoji.go
Original file line number Diff line number Diff line change
@@ -101,7 +101,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := gif.EncodeAll(newbuf, resized_gif); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest)
}
if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -113,13 +113,13 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := png.Encode(newbuf, resized_image); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest)
}
if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
}
} else {
if err := WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -159,7 +159,7 @@ func GetEmojiImage(emojiId string) (imageByte []byte, imageType string, err *mod
} else {
var img []byte

if data, err := ReadFile(getEmojiImagePath(emojiId)); err != nil {
if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil {
return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound)
} else {
img = data
@@ -219,7 +219,7 @@ func imageToPaletted(img image.Image) *image.Paletted {
}

func deleteEmojiImage(id string) {
if err := MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
l4g.Error("Failed to rename image when deleting emoji %v", id)
}
}
Loading

0 comments on commit 50fc6e1

Please sign in to comment.