From 6d476500a9e8be1b03f8f5d5aa343ca72bccd537 Mon Sep 17 00:00:00 2001 From: Johan Guldmyr Date: Thu, 25 Apr 2019 07:41:59 +0300 Subject: [PATCH] Support non AWS S3 endpoints Only tested with CEPH Rados Gateway (Luminous 12.2.x) which has better AWS4 support. Older CEPH Rados Gateway might need more boto 3 settings to use the older AWS2 signature. This adds a fifth option when initializing bigstore for 1 AWS S3 called endpoint_url From what I can tell this is needed until one can specify endpoints in better places: https://github.com/aws/aws-cli/issues/1270 --- bigstore/backends/s3.py | 4 ++-- bigstore/bigstore.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bigstore/backends/s3.py b/bigstore/backends/s3.py index 3f1be92..89ab44c 100644 --- a/bigstore/backends/s3.py +++ b/bigstore/backends/s3.py @@ -20,10 +20,10 @@ pass class S3Backend(object): - def __init__(self, bucket_name, key=None, secret=None, profile_name=None): + def __init__(self, bucket_name, key=None, secret=None, profile_name=None, endpoint_url=None): self.bucket = bucket_name self.session = boto3.Session(aws_access_key_id=key, aws_secret_access_key=secret, profile_name=profile_name) - self.s3_client = self.session.client('s3') + self.s3_client = self.session.client('s3', endpoint_url=endpoint_url) @property def name(self): diff --git a/bigstore/bigstore.py b/bigstore/bigstore.py index e89c0da..e6c4732 100644 --- a/bigstore/bigstore.py +++ b/bigstore/bigstore.py @@ -106,7 +106,8 @@ def backend_for_name(name): access_key_id = config('bigstore.s3.key') secret_access_key = config('bigstore.s3.secret') profile_name = config('bigstore.s3.profile-name') - return S3Backend(bucket_name, access_key_id, secret_access_key, profile_name) + endpoint_url = config('bigstore.s3.endpoint-url') + return S3Backend(bucket_name, access_key_id, secret_access_key, profile_name, endpoint_url) elif name == 'cloudfiles': username = config('bigstore.cloudfiles.username') api_key = config('bigstore.cloudfiles.key') @@ -467,6 +468,7 @@ def request_s3_credentials(): s3_key = input("Access Key: ") s3_secret = input("Secret Key: ") s3_profile_name = input("Profile Name: ") + s3_endpoint_url = input("Endpoint URL: ") g().config("bigstore.backend", "s3", file=config_filename) g().config("bigstore.s3.bucket", s3_bucket, file=config_filename) @@ -476,6 +478,8 @@ def request_s3_credentials(): g().config("bigstore.s3.secret", s3_secret, file=config_filename) if s3_profile_name != '': g().config("bigstore.s3.profile-name", s3_profile_name, file=config_filename) + if s3_endpoint_url != '': + g().config("bigstore.s3.endpoint-url", s3_endpoint_url, file=config_filename) def request_google_cloud_storage_credentials():