Skip to content

Commit

Permalink
Merge pull request #266 from MG-RAST/develop
Browse files Browse the repository at this point in the history 
develop into master
  • Loading branch information
@jaredbischof
jaredbischof committed May 13, 2015
2 parents a0f861f + a0eb862 commit c9ee049
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 74 deletions.
44 changes: 26 additions & 18 deletions shock-server/controller/node/single.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
_, download_raw := query["download_raw"]
if _, ok := query["download"]; ok || download_raw {
if !n.HasFile() {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file")
return responder.RespondWithError(ctx, http.StatusBadRequest, e.NodeNoFile)
}

_, seek_ok := query["seek"]
Expand Down Expand Up @@ -207,7 +207,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}
} else {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index")
return responder.RespondWithError(ctx, http.StatusBadRequest, e.InvalidIndex)
}
}

Expand Down Expand Up @@ -251,7 +251,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
fullRange := "1-" + strconv.FormatInt(recordIdxInfo.TotalUnits, 10)
recSlice, err := recordIdx.Range(fullRange, n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
for _, rec := range recSlice {
size += rec[1]
Expand All @@ -262,15 +262,15 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
for _, p := range query["part"] {
chunkRecSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
// This gets us the parts of the chunkrecord index, but we still need to convert these to record indices.
for _, chunkRec := range chunkRecSlice {
start := (chunkRec[0] / 16) + 1
stop := (start - 1) + (chunkRec[1] / 16)
recSlice, err := recordIdx.Range(strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(stop, 10), n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
for _, rec := range recSlice {
size += rec[1]
Expand All @@ -288,7 +288,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
fullRange := "1-" + strconv.FormatInt(idxInfo.TotalUnits, 10)
recSlice, err := idx.Range(fullRange, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
for _, rec := range recSlice {
size += rec[1]
Expand All @@ -301,16 +301,20 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
if idxInfo.Type == "subset" {
recSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
for _, rec := range recSlice {
size += rec[1]
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
} else {
// empty node has no parts
if n.File.Size == 0 {
return responder.RespondWithError(ctx, http.StatusBadRequest, e.IndexOutBounds)
}
pos, length, err := idx.Part(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part")
return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
}
size += length
s.R = append(s.R, io.NewSectionReader(r, pos, length))
Expand Down Expand Up @@ -340,17 +344,21 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}

idx := index.New()

s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc, Compression: compressionFormat}

fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10)
recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusInternalServerError, "Invalid data index for subset node.")
}
for _, rec := range recSlice {
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
if n.File.Size == 0 {
// handle empty subset file
s.R = append(s.R, r)
} else {
idx := index.New()
fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10)
recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits)
if err != nil {
return responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
}
for _, rec := range recSlice {
s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1]))
}
}
if err = s.Stream(download_raw); err != nil {
// causes "multiple response.WriteHeader calls" error but better than no response
Expand Down Expand Up @@ -383,7 +391,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
}

if !n.HasFile() {
return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file")
return responder.RespondWithError(ctx, http.StatusBadRequest, e.NodeNoFile)
} else {
// add options
options := map[string]string{}
Expand Down
6 changes: 5 additions & 1 deletion shock-server/errors/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,13 @@ const (
AttrImut = "Node attributes immutable"
FileImut = "Node file immutable"
ProvenanceImut = "Provenance info immutable"
InvalidIndex = "Invalid Index"
InvalidIndex = "Invalid index type"
InvalidFileTypeForFilter = "Invalid file type for filter"
InvalidIndexRange = "Invalid index record range"
IndexOutBounds = "Index record out of bounds"
IndexNoFile = "Index file is missing"
NodeReferenced = "Node referenced by virtual node"
NodeDoesNotExist = "Node does not exist"
NodeNotFound = "Node not found"
NodeNoFile = "Node has no file"
)
11 changes: 7 additions & 4 deletions shock-server/node/file/index/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package index
import (
"encoding/binary"
"errors"
e "github.com/MG-RAST/Shock/shock-server/errors"
"io"
"os"
"strconv"
Expand Down Expand Up @@ -62,6 +63,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64,
// used for non-subset indices where the records are contiguous for the data file
f, err := os.Open(idxFilePath)
if err != nil {
err = errors.New(e.IndexNoFile)
return
}
defer f.Close()
Expand All @@ -71,7 +73,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64,
start, startEr := strconv.ParseInt(startend[0], 10, 64)
end, endEr := strconv.ParseInt(startend[1], 10, 64)
if startEr != nil || endEr != nil || start <= 0 || start > int64(idxLength) || end <= 0 || end > int64(idxLength) {
err = errors.New("Invalid part range")
err = errors.New(e.InvalidIndexRange)
return
}

Expand All @@ -92,7 +94,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64,
} else {
p, er := strconv.ParseInt(part, 10, 64)
if er != nil || p <= 0 || p > int64(idxLength) {
err = errors.New("")
err = errors.New(e.IndexOutBounds)
return
}

Expand All @@ -113,6 +115,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][]
// used for subset indices where the records are not contiguous for the data file
f, err := os.Open(idxFilePath)
if err != nil {
err = errors.New(e.IndexNoFile)
return
}
defer f.Close()
Expand All @@ -122,7 +125,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][]
start, startEr := strconv.ParseInt(startend[0], 10, 64)
end, endEr := strconv.ParseInt(startend[1], 10, 64)
if startEr != nil || endEr != nil || start <= 0 || start > int64(idxLength) || end <= 0 || end > int64(idxLength) {
err = errors.New("Invalid subset range")
err = errors.New(e.InvalidIndexRange)
return
}

Expand Down Expand Up @@ -169,7 +172,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][]
} else {
p, er := strconv.ParseInt(part, 10, 64)
if er != nil || p <= 0 || p > int64(idxLength) {
err = errors.New("")
err = errors.New(e.IndexOutBounds)
return
}

Expand Down
5 changes: 3 additions & 2 deletions shock-server/node/file/index/virtual.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package index

import (
"errors"
e "github.com/MG-RAST/Shock/shock-server/errors"
"strconv"
"strings"
)
Expand Down Expand Up @@ -52,7 +53,7 @@ func SizePart(part string, v *vIndex) (pos int64, length int64, err error) {
start, startEr := strconv.ParseInt(startend[0], 10, 64)
end, endEr := strconv.ParseInt(startend[1], 10, 64)
if startEr != nil || endEr != nil || start <= 0 || (start-1)*v.ChunkSize > v.size || end <= 0 || (end-1)*v.ChunkSize > v.size {
err = errors.New("")
err = errors.New(e.InvalidIndexRange)
return
}
pos = (start - 1) * v.ChunkSize
Expand All @@ -65,7 +66,7 @@ func SizePart(part string, v *vIndex) (pos int64, length int64, err error) {
} else {
p, er := strconv.ParseInt(part, 10, 64)
if er != nil || p <= 0 || (p-1)*v.ChunkSize > v.size {
err = errors.New("")
err = errors.New(e.IndexOutBounds)
return
}
pos = (p - 1) * v.ChunkSize
Expand Down
99 changes: 52 additions & 47 deletions shock-server/node/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,66 +222,71 @@ func (node *Node) Update(params map[string]string, files FormFiles) (err error)
node.File.Path = n.File.Path
}

err = node.Save()
if err != nil {
return
}
} else if isSubsetUpload {
_, hasParentIndex := params["parent_index"]
if !hasParentIndex {
return errors.New("parent_index is a required parameter for creating a subset node.")
}

var n *Node
n, err = Load(params["parent_node"])
if err != nil {
if err = node.Save(); err != nil {
return err
}

if n.File.Virtual {
return errors.New("parent_node parameter points to a virtual node, invalid operation.")
}

if _, indexExists := n.Indexes[params["parent_index"]]; !indexExists {
return errors.New("Index '" + params["parent_index"] + "' does not exist for parent node.")
}

parentIndexFile := n.IndexPath() + "/" + params["parent_index"] + ".idx"
if _, statErr := os.Stat(parentIndexFile); statErr != nil {
return errors.New("Could not stat index file for parent node where parent node = '" + params["parent_node"] + "' and index = '" + params["parent_index"] + "'.")
} else if isSubsetUpload {
fInfo, statErr := os.Stat(files["subset_indices"].Path)
if statErr != nil {
return errors.New("Could not stat uploaded subset_indices file.")
}

// Copy node file information
node.File.Name = n.File.Name
node.File.Format = n.File.Format
node.Type = "subset"
node.Subset.Parent.Id = params["parent_node"]
node.Subset.Parent.IndexName = params["parent_index"]

if n.File.Path == "" {
node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["parent_node"]), params["parent_node"])
} else {
node.File.Path = n.File.Path
}

if _, hasSubsetList := files["subset_indices"]; hasSubsetList {
if err = node.SetFileFromSubset(files["subset_indices"]); err != nil {
if fInfo.Size() == 0 {
// if upload file is empty, make a basic node with empty file
if err = node.SetFile(files["subset_indices"]); err != nil {
return err
}
delete(files, "subset_indices")
} else {
err = node.Save()
// process subset upload
_, hasParentIndex := params["parent_index"]
if !hasParentIndex {
return errors.New("parent_index is a required parameter for creating a subset node.")
}

var n *Node
n, err = Load(params["parent_node"])
if err != nil {
return
return err
}
}
}

if _, hasSubsetList := files["subset_indices"]; hasSubsetList {
if err = node.SetFileFromSubset(files["subset_indices"]); err != nil {
return err
if n.File.Virtual {
return errors.New("parent_node parameter points to a virtual node, invalid operation.")
}

if _, indexExists := n.Indexes[params["parent_index"]]; !indexExists {
return errors.New("Index '" + params["parent_index"] + "' does not exist for parent node.")
}

parentIndexFile := n.IndexPath() + "/" + params["parent_index"] + ".idx"
if _, statErr := os.Stat(parentIndexFile); statErr != nil {
return errors.New("Could not stat index file for parent node where parent node = '" + params["parent_node"] + "' and index = '" + params["parent_index"] + "'.")
}

// Copy node file information
node.File.Name = n.File.Name
node.File.Format = n.File.Format
node.Subset.Parent.Id = params["parent_node"]
node.Subset.Parent.IndexName = params["parent_index"]

if n.File.Path == "" {
node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["parent_node"]), params["parent_node"])
} else {
node.File.Path = n.File.Path
}

if _, hasSubsetList := files["subset_indices"]; hasSubsetList {
if err = node.SetFileFromSubset(files["subset_indices"]); err != nil {
return err
}
delete(files, "subset_indices")
} else {
if err = node.Save(); err != nil {
return err
}
}
}
delete(files, "subset_indices")
}

// set attributes from file
Expand Down
10 changes: 8 additions & 2 deletions shock-server/request/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,18 +117,24 @@ func ParseMultipartForm(r *http.Request) (params map[string]string, files node.F
}
params[part.FormName()] = fmt.Sprintf("%s", buffer[0:n])
} else {
// determine file type
isSubsetFile := false
if part.FormName() == "subset_indices" {
isSubsetFile = true
}
isPartsFile := false
if _, er := strconv.Atoi(part.FormName()); er == nil {
isPartsFile = true
}
if !isPartsFile && !util.IsValidFileName(part.FormName()) {
return nil, files, errors.New("invalid file param: " + part.FormName())
}
// download it
tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
files[part.FormName()] = node.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
if tmpFile, err := os.Create(tmpPath); err == nil {
defer tmpFile.Close()
if util.IsValidUploadFile(part.FormName()) || isPartsFile {
if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
// handle upload or parts files
var tmpform = files[part.FormName()]
md5h := md5.New()
Expand All @@ -146,7 +152,7 @@ func ParseMultipartForm(r *http.Request) (params map[string]string, files node.F
tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
files[part.FormName()] = tmpform
} else {
// handle "attributes" and "subset_indices" files
// handle file where md5 not needed
if _, err = io.Copy(tmpFile, part); err != nil {
return nil, files, err
}
Expand Down

0 comments on commit c9ee049

Please sign in to comment.