Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix #16377 #16473

Merged
merged 6 commits into from
Nov 7, 2024
+120 −102
Merged

Fix #16377 #16473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f44a2d3
Test credentials using sts:GetCallerIdentity. Skip querying root for …
dkocher Oct 31, 2024
ea534ae
Return all available regions for profiles with custom default region.
dkocher Oct 31, 2024
634beaa
Logging.
dkocher Oct 31, 2024
98e9435
Allow retry of POST and PUT requests on 302 with x-amz-bucket-region.
dkocher Nov 4, 2024
b526ab2
Review case for region specific AWS profile. Apply default region onl…
dkocher Nov 4, 2024
0233042
Fix tests.
dkocher Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 37 additions & 34 deletions s3/src/main/java/ch/cyberduck/core/s3/RequestEntityRestStorageService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,47 +143,50 @@ public HttpUriRequest setupConnection(final HTTP_METHOD method, final String buc
String endpoint = host.getHostname();
// Apply default configuration
if(S3Session.isAwsHostname(host.getHostname(), false)) {
if(StringUtils.isNotBlank(host.getRegion())) {
log.debug("Apply default region {} to endpoint", host.getRegion());
// Apply default region
endpoint = createRegionSpecificEndpoint(host, host.getRegion());
}
else {
if(StringUtils.isNotBlank(bucketName)) {
// Only for AWS set endpoint to region specific
if(preferences.getBoolean(String.format("s3.transferacceleration.%s.enable", bucketName))) {
// Already set to accelerated endpoint
log.debug("Use accelerated endpoint {}", S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME);
endpoint = S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME;
}
else {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
// Check if not already request to query bucket location
if(requestParameters == null || !requestParameters.containsKey("location")) {
try {
// Determine region for bucket using cache
final Location.Name region = new S3LocationFeature(session, regionEndpointCache).getLocation(bucketName);
if(Location.unknown == region) {
// Missing permission or not supported
log.warn("Failure determining bucket location for {}", bucketName);
endpoint = host.getHostname();
}
else {
log.debug("Determined region {} for bucket {}", region, bucketName);
endpoint = createRegionSpecificEndpoint(host, region.getIdentifier());
}
if(StringUtils.isNotBlank(bucketName)) {
// Only for AWS set endpoint to region specific
if(preferences.getBoolean(String.format("s3.transferacceleration.%s.enable", bucketName))) {
// Already set to accelerated endpoint
log.debug("Use accelerated endpoint {}", S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME);
endpoint = S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME;
}
else {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
// Check if not already request to query bucket location
if(requestParameters == null || !requestParameters.containsKey("location")) {
try {
// Determine region for bucket using cache
final Location.Name region = new S3LocationFeature(session, regionEndpointCache).getLocation(bucketName);
if(Location.unknown == region) {
// Missing permission or not supported
log.warn("Failure determining bucket location for {}", bucketName);
endpoint = host.getHostname();
}
catch(BackgroundException e) {
// Ignore failure reading location for bucket
log.error("Failure {} determining bucket location for {}", e, bucketName);
endpoint = createRegionSpecificEndpoint(host, preferences.getProperty("s3.location"));
else {
log.debug("Determined region {} for bucket {}", region, bucketName);
endpoint = createRegionSpecificEndpoint(host, region.getIdentifier());
}
}
catch(BackgroundException e) {
// Ignore failure reading location for bucket
log.error("Failure {} determining bucket location for {}", e, bucketName);
endpoint = createRegionSpecificEndpoint(host, preferences.getProperty("s3.location"));
}
}
}
}
}
else {
if(StringUtils.isNotBlank(host.getRegion())) {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
log.debug("Apply default region {} to endpoint", host.getRegion());
// Apply default region
endpoint = createRegionSpecificEndpoint(host, host.getRegion());
}
}
}
}
log.debug("Set endpoint to {}", endpoint);
// Virtual host style endpoint including bucket name
Expand Down
3 changes: 3 additions & 0 deletions s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public void process(final HttpRequest request, final HttpContext context) throws
if(host != null) {
try {
region = SignatureUtils.awsRegionForRequest(new URI(host.toURI()));
log.debug("Determined region {} from URI {}", region, host.toURI());
}
catch(URISyntaxException e) {
throw new IOException(e);
Expand All @@ -95,9 +96,11 @@ public void process(final HttpRequest request, final HttpContext context) throws
}
if(null == region) {
region = session.getHost().getRegion();
log.debug("Determined region {} from {}", region, session.getHost());
}
if(null == region) {
region = new HostPreferences(session.getHost()).getProperty("s3.location");
log.debug("Determined region {} from defaults", region);
}
final HttpUriRequest message = (HttpUriRequest) request;
String requestPayloadHexSHA256Hash =
Expand Down
10 changes: 10 additions & 0 deletions s3/src/main/java/ch/cyberduck/core/s3/S3BucketRegionRedirectStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolException;
import org.apache.http.client.RedirectException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.DefaultRedirectStrategy;
Expand All @@ -39,6 +43,12 @@ public class S3BucketRegionRedirectStrategy extends DefaultRedirectStrategy {
private final Host host;

public S3BucketRegionRedirectStrategy(final RequestEntityRestStorageService service, final Host host) {
super(new String[] {
HttpHead.METHOD_NAME,
HttpGet.METHOD_NAME,
HttpPost.METHOD_NAME,
HttpPut.METHOD_NAME
});
this.service = service;
this.host = host;
}
Expand Down
22 changes: 6 additions & 16 deletions s3/src/main/java/ch/cyberduck/core/s3/S3LocationFeature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,31 +61,16 @@ public Name getDefault() {

@Override
public Set<Name> getLocations() {
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
final S3Region region = new S3Region(session.getHost().getRegion());
log.debug("Return single region {} set in bookmark", region);
return Collections.singleton(region);
}
if(StringUtils.isNotEmpty(RequestEntityRestStorageService.findBucketInHostname(session.getHost()))) {
log.debug("Return empty set for hostname {}", session.getHost());
// Connected to single bucket
return Collections.emptySet();
}
if(!S3Session.isAwsHostname(session.getHost().getHostname(), false)) {
if(new S3Protocol().getRegions().equals(session.getHost().getProtocol().getRegions())) {
// Return empty set for unknown provider
log.debug("Return empty set for unknown provider {}", session.getHost());
return Collections.emptySet();
}
}
return session.getHost().getProtocol().getRegions();
}

@Override
public Name getLocation(final Path file) throws BackgroundException {
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
return new S3Region(session.getHost().getRegion());
}
final Path bucket = containerService.getContainer(file);
return this.getLocation(bucket.isRoot() ? StringUtils.EMPTY : bucket.getName());
}
Expand All @@ -105,7 +90,12 @@ protected Name getLocation(final String bucketname) throws BackgroundException {
final S3Region region;
if(StringUtils.isBlank(location)) {
log.warn("No region known for bucket {}", bucketname);
region = DEFAULT_REGION;
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
region = new S3Region(session.getHost().getRegion());
}
else {
region = DEFAULT_REGION;
}
}
else {
switch(location) {
Expand Down
65 changes: 50 additions & 15 deletions s3/src/main/java/ch/cyberduck/core/s3/S3Session.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
*/

import ch.cyberduck.core.Credentials;
import ch.cyberduck.core.DisabledListProgressListener;
import ch.cyberduck.core.Host;
import ch.cyberduck.core.HostKeyCallback;
import ch.cyberduck.core.ListService;
Expand All @@ -28,15 +29,15 @@
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.Scheme;
import ch.cyberduck.core.UrlProvider;
import ch.cyberduck.core.auth.AWSCredentialsConfigurator;
import ch.cyberduck.core.auth.AWSSessionCredentialsRetriever;
import ch.cyberduck.core.aws.CustomClientConfiguration;
import ch.cyberduck.core.cdn.Distribution;
import ch.cyberduck.core.cdn.DistributionConfiguration;
import ch.cyberduck.core.cloudfront.CloudFrontDistributionConfigurationPreloader;
import ch.cyberduck.core.cloudfront.WebsiteCloudFrontDistributionConfiguration;
import ch.cyberduck.core.date.RFC822DateFormatter;
import ch.cyberduck.core.exception.AccessDeniedException;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.exception.InteroperabilityException;
import ch.cyberduck.core.exception.LoginCanceledException;
import ch.cyberduck.core.features.*;
import ch.cyberduck.core.http.CustomServiceUnavailableRetryStrategy;
Expand All @@ -49,7 +50,6 @@
import ch.cyberduck.core.preferences.PreferencesReader;
import ch.cyberduck.core.proxy.ProxyFinder;
import ch.cyberduck.core.restore.Glacier;
import ch.cyberduck.core.shared.DefaultHomeFinderService;
import ch.cyberduck.core.shared.DefaultPathHomeFeature;
import ch.cyberduck.core.shared.DelegatingHomeFeature;
import ch.cyberduck.core.shared.DisabledBulkFeature;
Expand All @@ -59,8 +59,10 @@
import ch.cyberduck.core.ssl.X509KeyManager;
import ch.cyberduck.core.ssl.X509TrustManager;
import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor;
import ch.cyberduck.core.sts.STSExceptionMappingService;
import ch.cyberduck.core.threading.CancelCallback;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
Expand All @@ -87,6 +89,12 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException;
import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
import com.amazonaws.services.securitytoken.model.GetCallerIdentityResult;

import static com.amazonaws.services.s3.Headers.*;

public class S3Session extends HttpSession<RequestEntityRestStorageService> {
Expand Down Expand Up @@ -323,20 +331,47 @@ public void login(final LoginCallback prompt, final CancelCallback cancel) throw
log.warn("Skip verifying credentials with previous successful authentication event for {}", this);
return;
}
try {
final Path home = new DelegatingHomeFeature(new DefaultPathHomeFeature(host)).find();
final Location.Name location = new S3LocationFeature(S3Session.this, regions).getLocation(home);
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
//
host.setProperty("s3.location", location.getIdentifier());
final Path home = new DelegatingHomeFeature(new DefaultPathHomeFeature(host)).find();
if(S3Session.isAwsHostname(host.getHostname(), false)) {
if(StringUtils.isEmpty(RequestEntityRestStorageService.findBucketInHostname(host))) {
final CustomClientConfiguration configuration = new CustomClientConfiguration(host,
new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key);
final AWSSecurityTokenServiceClientBuilder builder = AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials(AWSCredentialsConfigurator.toAWSCredentialsProvider(client.getProviderCredentials()))
.withClientConfiguration(configuration);
final AWSSecurityTokenService service = builder.build();
// Returns details about the IAM user or role whose credentials are used to call the operation.
// No permissions are required to perform this operation.
try {
final GetCallerIdentityResult identity = service.getCallerIdentity(new GetCallerIdentityRequest());
log.debug("Successfully verified credentials for {}", identity);
}
catch(AWSSecurityTokenServiceException e) {
throw new STSExceptionMappingService().map(e);
}
}
else {
final Location.Name location = new S3LocationFeature(this, regions).getLocation(home);
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
host.setProperty("s3.location", location.getIdentifier());
}
}
}
catch(AccessDeniedException | InteroperabilityException e) {
log.warn("Failure {} querying region", e.getMessage());
final Path home = new DefaultHomeFinderService(this).find();
log.debug("Retrieved {}", home);
else {
if(home.isRoot() && StringUtils.isEmpty(RequestEntityRestStorageService.findBucketInHostname(host))) {
log.debug("Skip querying region for {}", home);
new S3ListService(this, acl).list(home, new DisabledListProgressListener());
}
else {
final Location.Name location = new S3LocationFeature(this, regions).getLocation(home);
dkocher marked this conversation as resolved.
Show resolved Hide resolved
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
host.setProperty("s3.location", location.getIdentifier());
}
}
}
}

Expand Down
5 changes: 2 additions & 3 deletions s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,17 @@
import ch.cyberduck.core.exception.LoginFailureException;

import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException;
import com.amazonaws.services.securitytoken.model.InvalidIdentityTokenException;

public class STSExceptionMappingService implements ExceptionMappingService<AWSSecurityTokenServiceException> {

@Override
public BackgroundException map(final AWSSecurityTokenServiceException e) {
if(e instanceof com.amazonaws.services.securitytoken.model.ExpiredTokenException) {
if("RequestExpired".equals(e.getErrorCode())) {
// The web identity token that was passed is expired or is not valid. Get a new identity token from the identity
// provider and then retry the request.
return new ExpiredTokenException(e.getErrorMessage(), e);
}
if(e instanceof InvalidIdentityTokenException) {
if("InvalidClientTokenId".equals(e.getErrorCode())) {
// The web identity token that was passed could not be validated by Amazon Web Services. Get a new identity token from
// the identity provider and then retry the request.
return new ExpiredTokenException(e.getErrorMessage(), e);
Expand Down
2 changes: 1 addition & 1 deletion s3/src/test/java/ch/cyberduck/core/s3/RequestEntityRestStorageServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void testSetupConnection() throws Exception {
@Test
public void testSetupConnectionVirtualHost() throws Exception {
final RequestEntityRestStorageService service = new RequestEntityRestStorageService(virtualhost, new HttpConnectionPoolBuilder(virtualhost.getHost(),
new ThreadLocalHostnameDelegatingTrustManager(new DisabledX509TrustManager(), session.getHost().getHostname()),
new ThreadLocalHostnameDelegatingTrustManager(new DisabledX509TrustManager(), virtualhost.getHost().getHostname()),
new DefaultX509KeyManager(), new DisabledProxyFinder()).build(new DisabledProxyFinder(), new DisabledTranscriptListener(), new DisabledLoginCallback()));
final RegionEndpointCache cache = service.getRegionEndpointCache();
cache.clear();
Expand Down
4 changes: 2 additions & 2 deletions s3/src/test/java/ch/cyberduck/core/s3/S3AccessControlListFeatureTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,15 @@ public void testWrite() throws Exception {
@Test
public void testWriteVirtualHostBucket() throws Exception {
final Path test = new Path(new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
new S3TouchFeature(virtualhost, new S3AccessControlListFeature(session)).touch(test, new TransferStatus());
new S3TouchFeature(virtualhost, new S3AccessControlListFeature(virtualhost)).touch(test, new TransferStatus());
final S3AccessControlListFeature f = new S3AccessControlListFeature(virtualhost);
final Acl acl = new Acl();
acl.addAll(new Acl.Owner("80b9982b7b08045ee86680cc47f43c84bf439494a89ece22b5330f8a49477cf6"), new Acl.Role(Acl.Role.FULL));
acl.addAll(new Acl.GroupUser(Acl.GroupUser.EVERYONE), new Acl.Role(Acl.Role.READ));
acl.addAll(new Acl.GroupUser(Acl.GroupUser.AUTHENTICATED), new Acl.Role(Acl.Role.READ));
f.setPermission(test, acl);
assertEquals(acl, f.getPermission(test));
new S3DefaultDeleteFeature(virtualhost, new S3AccessControlListFeature(session)).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
new S3DefaultDeleteFeature(virtualhost, new S3AccessControlListFeature(virtualhost)).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

@Test
Expand Down
Loading