diff --git a/shock-server/controller/node/single.go b/shock-server/controller/node/single.go index b8b825f4..fc596046 100644 --- a/shock-server/controller/node/single.go +++ b/shock-server/controller/node/single.go @@ -96,7 +96,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { _, download_raw := query["download_raw"] if _, ok := query["download"]; ok || download_raw { if !n.HasFile() { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file") + return responder.RespondWithError(ctx, http.StatusBadRequest, e.NodeNoFile) } _, seek_ok := query["seek"] @@ -207,7 +207,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } } else { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index") + return responder.RespondWithError(ctx, http.StatusBadRequest, e.InvalidIndex) } } @@ -251,7 +251,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { fullRange := "1-" + strconv.FormatInt(recordIdxInfo.TotalUnits, 10) recSlice, err := recordIdx.Range(fullRange, n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } for _, rec := range recSlice { size += rec[1] @@ -262,7 +262,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { for _, p := range query["part"] { chunkRecSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } // This gets us the parts of the chunkrecord index, but we still need to convert these to record indices. for _, chunkRec := range chunkRecSlice { @@ -270,7 +270,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { stop := (start - 1) + (chunkRec[1] / 16) recSlice, err := recordIdx.Range(strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(stop, 10), n.IndexPath()+"/"+recordIdxName+".idx", recordIdxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } for _, rec := range recSlice { size += rec[1] @@ -288,7 +288,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { fullRange := "1-" + strconv.FormatInt(idxInfo.TotalUnits, 10) recSlice, err := idx.Range(fullRange, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index subset") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } for _, rec := range recSlice { size += rec[1] @@ -301,16 +301,20 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { if idxInfo.Type == "subset" { recSlice, err := idx.Range(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } for _, rec := range recSlice { size += rec[1] s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) } } else { + // empty node has no parts + if n.File.Size == 0 { + return responder.RespondWithError(ctx, http.StatusBadRequest, e.IndexOutBounds) + } pos, length, err := idx.Part(p, n.IndexPath()+"/"+idxName+".idx", idxInfo.TotalUnits) if err != nil { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid index part") + return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error()) } size += length s.R = append(s.R, io.NewSectionReader(r, pos, length)) @@ -340,17 +344,21 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg) } - idx := index.New() - s := &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc, Compression: compressionFormat} - fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10) - recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits) - if err != nil { - return responder.RespondWithError(ctx, http.StatusInternalServerError, "Invalid data index for subset node.") - } - for _, rec := range recSlice { - s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) + if n.File.Size == 0 { + // handle empty subset file + s.R = append(s.R, r) + } else { + idx := index.New() + fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10) + recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits) + if err != nil { + return responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error()) + } + for _, rec := range recSlice { + s.R = append(s.R, io.NewSectionReader(r, rec[0], rec[1])) + } } if err = s.Stream(download_raw); err != nil { // causes "multiple response.WriteHeader calls" error but better than no response @@ -383,7 +391,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error { } if !n.HasFile() { - return responder.RespondWithError(ctx, http.StatusBadRequest, "Node has no file") + return responder.RespondWithError(ctx, http.StatusBadRequest, e.NodeNoFile) } else { // add options options := map[string]string{} diff --git a/shock-server/errors/errors.go b/shock-server/errors/errors.go index 38bd25bc..e5b9ae38 100644 --- a/shock-server/errors/errors.go +++ b/shock-server/errors/errors.go @@ -16,9 +16,13 @@ const ( AttrImut = "Node attributes immutable" FileImut = "Node file immutable" ProvenanceImut = "Provenance info immutable" - InvalidIndex = "Invalid Index" + InvalidIndex = "Invalid index type" InvalidFileTypeForFilter = "Invalid file type for filter" + InvalidIndexRange = "Invalid index record range" + IndexOutBounds = "Index record out of bounds" + IndexNoFile = "Index file is missing" NodeReferenced = "Node referenced by virtual node" NodeDoesNotExist = "Node does not exist" NodeNotFound = "Node not found" + NodeNoFile = "Node has no file" ) diff --git a/shock-server/node/file/index/index.go b/shock-server/node/file/index/index.go index ab99066d..1ad635c3 100644 --- a/shock-server/node/file/index/index.go +++ b/shock-server/node/file/index/index.go @@ -3,6 +3,7 @@ package index import ( "encoding/binary" "errors" + e "github.com/MG-RAST/Shock/shock-server/errors" "io" "os" "strconv" @@ -62,6 +63,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64, // used for non-subset indices where the records are contiguous for the data file f, err := os.Open(idxFilePath) if err != nil { + err = errors.New(e.IndexNoFile) return } defer f.Close() @@ -71,7 +73,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64, start, startEr := strconv.ParseInt(startend[0], 10, 64) end, endEr := strconv.ParseInt(startend[1], 10, 64) if startEr != nil || endEr != nil || start <= 0 || start > int64(idxLength) || end <= 0 || end > int64(idxLength) { - err = errors.New("Invalid part range") + err = errors.New(e.InvalidIndexRange) return } @@ -92,7 +94,7 @@ func (i *Idx) Part(part string, idxFilePath string, idxLength int64) (pos int64, } else { p, er := strconv.ParseInt(part, 10, 64) if er != nil || p <= 0 || p > int64(idxLength) { - err = errors.New("") + err = errors.New(e.IndexOutBounds) return } @@ -113,6 +115,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][] // used for subset indices where the records are not contiguous for the data file f, err := os.Open(idxFilePath) if err != nil { + err = errors.New(e.IndexNoFile) return } defer f.Close() @@ -122,7 +125,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][] start, startEr := strconv.ParseInt(startend[0], 10, 64) end, endEr := strconv.ParseInt(startend[1], 10, 64) if startEr != nil || endEr != nil || start <= 0 || start > int64(idxLength) || end <= 0 || end > int64(idxLength) { - err = errors.New("Invalid subset range") + err = errors.New(e.InvalidIndexRange) return } @@ -169,7 +172,7 @@ func (i *Idx) Range(part string, idxFilePath string, idxLength int64) (recs [][] } else { p, er := strconv.ParseInt(part, 10, 64) if er != nil || p <= 0 || p > int64(idxLength) { - err = errors.New("") + err = errors.New(e.IndexOutBounds) return } diff --git a/shock-server/node/file/index/virtual.go b/shock-server/node/file/index/virtual.go index ff3449fc..154deb74 100644 --- a/shock-server/node/file/index/virtual.go +++ b/shock-server/node/file/index/virtual.go @@ -2,6 +2,7 @@ package index import ( "errors" + e "github.com/MG-RAST/Shock/shock-server/errors" "strconv" "strings" ) @@ -52,7 +53,7 @@ func SizePart(part string, v *vIndex) (pos int64, length int64, err error) { start, startEr := strconv.ParseInt(startend[0], 10, 64) end, endEr := strconv.ParseInt(startend[1], 10, 64) if startEr != nil || endEr != nil || start <= 0 || (start-1)*v.ChunkSize > v.size || end <= 0 || (end-1)*v.ChunkSize > v.size { - err = errors.New("") + err = errors.New(e.InvalidIndexRange) return } pos = (start - 1) * v.ChunkSize @@ -65,7 +66,7 @@ func SizePart(part string, v *vIndex) (pos int64, length int64, err error) { } else { p, er := strconv.ParseInt(part, 10, 64) if er != nil || p <= 0 || (p-1)*v.ChunkSize > v.size { - err = errors.New("") + err = errors.New(e.IndexOutBounds) return } pos = (p - 1) * v.ChunkSize diff --git a/shock-server/node/update.go b/shock-server/node/update.go index 4b4b4f84..61053eb3 100644 --- a/shock-server/node/update.go +++ b/shock-server/node/update.go @@ -222,66 +222,71 @@ func (node *Node) Update(params map[string]string, files FormFiles) (err error) node.File.Path = n.File.Path } - err = node.Save() - if err != nil { - return - } - } else if isSubsetUpload { - _, hasParentIndex := params["parent_index"] - if !hasParentIndex { - return errors.New("parent_index is a required parameter for creating a subset node.") - } - - var n *Node - n, err = Load(params["parent_node"]) - if err != nil { + if err = node.Save(); err != nil { return err } - - if n.File.Virtual { - return errors.New("parent_node parameter points to a virtual node, invalid operation.") - } - - if _, indexExists := n.Indexes[params["parent_index"]]; !indexExists { - return errors.New("Index '" + params["parent_index"] + "' does not exist for parent node.") - } - - parentIndexFile := n.IndexPath() + "/" + params["parent_index"] + ".idx" - if _, statErr := os.Stat(parentIndexFile); statErr != nil { - return errors.New("Could not stat index file for parent node where parent node = '" + params["parent_node"] + "' and index = '" + params["parent_index"] + "'.") + } else if isSubsetUpload { + fInfo, statErr := os.Stat(files["subset_indices"].Path) + if statErr != nil { + return errors.New("Could not stat uploaded subset_indices file.") } - - // Copy node file information - node.File.Name = n.File.Name - node.File.Format = n.File.Format node.Type = "subset" - node.Subset.Parent.Id = params["parent_node"] - node.Subset.Parent.IndexName = params["parent_index"] - if n.File.Path == "" { - node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["parent_node"]), params["parent_node"]) - } else { - node.File.Path = n.File.Path - } - - if _, hasSubsetList := files["subset_indices"]; hasSubsetList { - if err = node.SetFileFromSubset(files["subset_indices"]); err != nil { + if fInfo.Size() == 0 { + // if upload file is empty, make a basic node with empty file + if err = node.SetFile(files["subset_indices"]); err != nil { return err } delete(files, "subset_indices") } else { - err = node.Save() + // process subset upload + _, hasParentIndex := params["parent_index"] + if !hasParentIndex { + return errors.New("parent_index is a required parameter for creating a subset node.") + } + + var n *Node + n, err = Load(params["parent_node"]) if err != nil { - return + return err } - } - } - if _, hasSubsetList := files["subset_indices"]; hasSubsetList { - if err = node.SetFileFromSubset(files["subset_indices"]); err != nil { - return err + if n.File.Virtual { + return errors.New("parent_node parameter points to a virtual node, invalid operation.") + } + + if _, indexExists := n.Indexes[params["parent_index"]]; !indexExists { + return errors.New("Index '" + params["parent_index"] + "' does not exist for parent node.") + } + + parentIndexFile := n.IndexPath() + "/" + params["parent_index"] + ".idx" + if _, statErr := os.Stat(parentIndexFile); statErr != nil { + return errors.New("Could not stat index file for parent node where parent node = '" + params["parent_node"] + "' and index = '" + params["parent_index"] + "'.") + } + + // Copy node file information + node.File.Name = n.File.Name + node.File.Format = n.File.Format + node.Subset.Parent.Id = params["parent_node"] + node.Subset.Parent.IndexName = params["parent_index"] + + if n.File.Path == "" { + node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["parent_node"]), params["parent_node"]) + } else { + node.File.Path = n.File.Path + } + + if _, hasSubsetList := files["subset_indices"]; hasSubsetList { + if err = node.SetFileFromSubset(files["subset_indices"]); err != nil { + return err + } + delete(files, "subset_indices") + } else { + if err = node.Save(); err != nil { + return err + } + } } - delete(files, "subset_indices") } // set attributes from file diff --git a/shock-server/request/request.go b/shock-server/request/request.go index 5d914808..8105fe52 100644 --- a/shock-server/request/request.go +++ b/shock-server/request/request.go @@ -117,6 +117,11 @@ func ParseMultipartForm(r *http.Request) (params map[string]string, files node.F } params[part.FormName()] = fmt.Sprintf("%s", buffer[0:n]) } else { + // determine file type + isSubsetFile := false + if part.FormName() == "subset_indices" { + isSubsetFile = true + } isPartsFile := false if _, er := strconv.Atoi(part.FormName()); er == nil { isPartsFile = true @@ -124,11 +129,12 @@ func ParseMultipartForm(r *http.Request) (params map[string]string, files node.F if !isPartsFile && !util.IsValidFileName(part.FormName()) { return nil, files, errors.New("invalid file param: " + part.FormName()) } + // download it tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int()) files[part.FormName()] = node.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)} if tmpFile, err := os.Create(tmpPath); err == nil { defer tmpFile.Close() - if util.IsValidUploadFile(part.FormName()) || isPartsFile { + if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile { // handle upload or parts files var tmpform = files[part.FormName()] md5h := md5.New() @@ -146,7 +152,7 @@ func ParseMultipartForm(r *http.Request) (params map[string]string, files node.F tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil)) files[part.FormName()] = tmpform } else { - // handle "attributes" and "subset_indices" files + // handle file where md5 not needed if _, err = io.Copy(tmpFile, part); err != nil { return nil, files, err }