Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix path prefix handling in S3-to-S3 copy #396

Merged
merged 14 commits into from
Jun 10, 2024
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ fixture:
aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/foo/
aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/bar/baz/
aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination
aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination2
aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-escaped
aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload
aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-upload/dest_only_file
Expand Down
10 changes: 6 additions & 4 deletions s3sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
Expand Down Expand Up @@ -253,15 +253,17 @@ func (m *Manager) syncS3ToLocal(ctx context.Context, chJob chan func(), sourcePa
}

func (m *Manager) copyS3ToS3(ctx context.Context, file *fileInfo, sourcePath *s3Path, destPath *s3Path) error {
println("Copying", file.name, "to", destPath.String())
copySource := filepath.ToSlash(filepath.Join(sourcePath.bucket, sourcePath.bucketPrefix, file.name))
destinationKey := filepath.ToSlash(filepath.Join(destPath.bucketPrefix, file.name))
println("Copying from", copySource, "to key", destinationKey, "in bucket", destPath.bucket)
if m.dryrun {
return nil
}

_, err := m.s3.CopyObject(&s3.CopyObjectInput{
Bucket: aws.String(destPath.bucket),
CopySource: aws.String(sourcePath.bucket + "/" + file.name),
Key: aws.String(file.name),
CopySource: aws.String(copySource),
Key: aws.String(destinationKey),
ACL: m.acl,
})

Expand Down
29 changes: 19 additions & 10 deletions s3sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,6 @@ func TestS3syncNotImplemented(t *testing.T) {
}
}

func TestS3ToS3(t *testing.T) {
m := New(getSession())

err := m.Sync("s3://s3-source", "s3://s3-destination")

if err != nil {
t.Fatal(err.Error())
}
}

func TestS3sync(t *testing.T) {
data, err := ioutil.ReadFile(dummyFilename)
if err != nil {
Expand Down Expand Up @@ -163,6 +153,25 @@ func TestS3sync(t *testing.T) {
}
})

t.Run("S3ToS3CopyWithPrefix", func(t *testing.T) {
if err := New(getSession()).Sync("s3://s3-source/bar", "s3://s3-destination2/hoge"); err != nil {
t.Fatal("Sync should be successful", err)
}

objs := listObjectsSorted(t, "s3-destination2")
if n := len(objs); n != 1 {
t.Fatalf("Number of the files should be 1 (result: %v)", objs)
}
for _, obj := range objs {
if obj.size != dummyFileSize {
t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size)
}
}
if objs[0].path != "hoge/baz/README.md" {
t.Error("Unexpected keys", objs)
}
})

t.Run("Upload", func(t *testing.T) {
temp, err := ioutil.TempDir("", "s3synctest")
defer os.RemoveAll(temp)
Expand Down