-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loop over pages to get requested object count. (#29)
Loop over pages to get requested object count
- Loading branch information
1 parent
2bb3c6c
commit 569e602
Showing
2 changed files
with
80 additions
and
87 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -315,8 +315,6 @@ func (cl *Client) List(prefix string, marker *string, count int32) ([]*internal. | |
| // Check for an empty path to prevent indexing to [-1] | ||
| findCommonPrefixes := listPath == "" || listPath[len(listPath)-1] == '/' | ||
|
|
||
| // create a map to keep track of all directories | ||
| var dirList = make(map[string]bool) | ||
| var newMarker *string | ||
| var token *string | ||
|
|
||
|
|
@@ -339,65 +337,64 @@ func (cl *Client) List(prefix string, marker *string, count int32) ([]*internal. | |
| paginator := s3.NewListObjectsV2Paginator(cl.awsS3Client, params) | ||
| // initialize list to be returned | ||
| objectAttrList := make([]*internal.ObjAttr, 0) | ||
| // fetch and process result pages | ||
|
|
||
| if paginator.HasMorePages() { | ||
| output, err := paginator.NextPage(context.Background()) | ||
| if err != nil { | ||
| log.Err("Client::List : Failed to list objects in bucket %v with prefix %v. Here's why: %v", prefix, bucketName, err) | ||
| return objectAttrList, nil, err | ||
| } | ||
| // fetch and process a single result page | ||
| output, err := paginator.NextPage(context.Background()) | ||
| if err != nil { | ||
| log.Err("Client::List : Failed to list objects in bucket %v with prefix %v. Here's why: %v", prefix, bucketName, err) | ||
| return objectAttrList, nil, err | ||
| } | ||
|
|
||
| if output.IsTruncated { | ||
| newMarker = output.NextContinuationToken | ||
| } else { | ||
| newMarker = nil | ||
| } | ||
| if output.IsTruncated { | ||
| newMarker = output.NextContinuationToken | ||
| } else { | ||
| newMarker = nil | ||
| } | ||
|
|
||
| // documentation for this S3 data structure: | ||
| // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]#ListObjectsV2Output | ||
| for _, value := range output.Contents { | ||
| // push object info into the list | ||
| name, isSymLink := cl.getFile(*value.Key) | ||
| // documentation for this S3 data structure: | ||
| // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]#ListObjectsV2Output | ||
| for _, value := range output.Contents { | ||
| // push object info into the list | ||
| name, isSymLink := cl.getFile(*value.Key) | ||
|
|
||
| path := split(cl.Config.prefixPath, name) | ||
| attr := createObjAttr(path, value.Size, *value.LastModified, isSymLink) | ||
| objectAttrList = append(objectAttrList, attr) | ||
| } | ||
| path := split(cl.Config.prefixPath, name) | ||
| attr := createObjAttr(path, value.Size, *value.LastModified, isSymLink) | ||
| objectAttrList = append(objectAttrList, attr) | ||
| } | ||
|
|
||
| if findCommonPrefixes { | ||
| // documentation for CommonPrefixes: | ||
| // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]/types#CommonPrefix | ||
| for _, value := range output.CommonPrefixes { | ||
| dir := *value.Prefix | ||
| dirList[dir] = true | ||
| // let's extract and add intermediate directories | ||
| // first cut the listPath (the full prefix path) off of the directory path | ||
| _, intermediatePath, listPathFound := strings.Cut(dir, listPath) | ||
| // if the listPath isn't here, that's weird | ||
| if !listPathFound { | ||
| log.Warn("Prefix mismatch with path %v when listing objects in %v.", dir, listPath) | ||
| if findCommonPrefixes { | ||
| // documentation for CommonPrefixes: | ||
| // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]/types#CommonPrefix | ||
| // create a map to keep track of all directories | ||
| var dirList = make(map[string]bool) | ||
| for _, value := range output.CommonPrefixes { | ||
| dir := *value.Prefix | ||
| dirList[dir] = true | ||
| // let's extract and add intermediate directories | ||
| // first cut the listPath (the full prefix path) off of the directory path | ||
| _, intermediatePath, listPathFound := strings.Cut(dir, listPath) | ||
| // if the listPath isn't here, that's weird | ||
| if !listPathFound { | ||
| log.Warn("Prefix mismatch with path %v when listing objects in %v.", dir, listPath) | ||
| } | ||
| // get an array of intermediate directories | ||
| intermediateDirectories := strings.Split(intermediatePath, "/") | ||
| // walk up the tree and add each one until we find an already existing parent | ||
| // we have to iterate in descending order | ||
| suffixToTrim := "" | ||
| for i := len(intermediateDirectories) - 1; i >= 0; i-- { | ||
| // ignore empty strings (split does not omit them) | ||
| if intermediateDirectories[i] == "" { | ||
| continue | ||
| } | ||
| // get an array of intermediate directories | ||
| intermediateDirectories := strings.Split(intermediatePath, "/") | ||
| // walk up the tree and add each one until we find an already existing parent | ||
| // we have to iterate in descending order | ||
| suffixToTrim := "" | ||
| for i := len(intermediateDirectories) - 1; i >= 0; i-- { | ||
| // ignore empty strings (split does not omit them) | ||
| if intermediateDirectories[i] == "" { | ||
| continue | ||
| } | ||
| // add to the suffix we're trimming off | ||
| suffixToTrim = intermediateDirectories[i] + "/" + suffixToTrim | ||
| // get the trimmed (parent) directory | ||
| parentDir := strings.TrimSuffix(dir, suffixToTrim) | ||
| // have we seen this one already? | ||
| if dirList[parentDir] { | ||
| break | ||
| } | ||
| dirList[parentDir] = true | ||
| // add to the suffix we're trimming off | ||
| suffixToTrim = intermediateDirectories[i] + "/" + suffixToTrim | ||
| // get the trimmed (parent) directory | ||
| parentDir := strings.TrimSuffix(dir, suffixToTrim) | ||
| // have we seen this one already? | ||
| if dirList[parentDir] { | ||
| break | ||
| } | ||
| dirList[parentDir] = true | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -411,17 +408,15 @@ func (cl *Client) List(prefix string, marker *string, count int32) ([]*internal. | |
| attr := internal.CreateObjAttrDir(path) | ||
| objectAttrList = append(objectAttrList, attr) | ||
| } | ||
|
|
||
| // values should be returned in ascending order by key | ||
| // sort the list before returning it | ||
| sort.Slice(objectAttrList, func(i, j int) bool { | ||
| return objectAttrList[i].Path < objectAttrList[j].Path | ||
| }) | ||
|
|
||
| } | ||
|
|
||
| return objectAttrList, newMarker, nil | ||
| // values should be returned in ascending order by key | ||
| // sort the list before returning it | ||
| sort.Slice(objectAttrList, func(i, j int) bool { | ||
| return objectAttrList[i].Path < objectAttrList[j].Path | ||
| }) | ||
|
|
||
| return objectAttrList, newMarker, nil | ||
| } | ||
|
|
||
| // create an object attributes struct | ||
|
|
||