Merge pull request #6 from Multiplicom/0.1.9

allow object prefix to be passed when uploading

.gitignore

@@ -101,4 +101,6 @@ ENV/
 .mypy_cache/
 
 .pytest_cache
-.vscode
+.vscode
+
+.DS_Store

tests/integration_test.py

@@ -0,0 +1,74 @@
+# from __future__ import print_function
+
+# import random
+
+# import os
+# import sys
+# from os import path
+
+# from wanna import Transfer
+
+# import logging
+
+# logging.basicConfig(format="%(levelname)-8s %(message)s")
+
+# logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
+
+# console_handler = logging.StreamHandler(sys.stdout)
+# console_handler.setLevel(logging.DEBUG)
+# logger.addHandler(console_handler)
+
+# application_profile_space = Transfer(
+#     vendor="aws", bucket="mr17-external-prd-15", profile="mr-app-user-prd")
+
+# user_profile_space = Transfer(
+#     vendor="aws", bucket="mr17-external-prd-15", profile="mr17-external-prd-15")
+
+# module_dir = os.path.dirname(os.path.realpath(__file__))
+
+# random_file_name = "%032x" % random.getrandbits(128)
+# file_path = os.path.join(module_dir, random_file_name)
+
+
+# try:
+#     shared_secret = "cf243a7c213899020c18b76a2dff7226a570723100e49d9d067cd4759e0a02bb"
+
+#     logger.debug('creating file %s' % file_path)
+#     file_stub = open(file_path, "w+")
+#     file_stub.write(random_file_name)
+#     file_stub.close()
+
+#     # Upload file and remove locally
+#     logger.debug("uploading %s" % file_path)
+
+#     application_profile_space.upload_files(
+#         file_path, encryption_key=shared_secret)
+
+#     # Download file and assert
+#     logger.debug("downloading %s" % random_file_name)
+
+#     user_profile_space.download_file(
+#         random_file_name, dst=file_path + ".download", encryption_key=shared_secret)
+
+#     assert os.path.isfile(file_path) == True
+
+#     with open("{}.download".format(file_path)) as f:
+#         assert f.readline() == random_file_name
+
+#     # Clean files
+#     user_profile_space.delete_file(random_file_name)
+#     logger.debug("removing %s" % file_path)
+#     os.remove(file_path)
+#     os.remove("{}.download".format(file_path))
+
+#     logger.info('test succeeded')
+# except OSError as e:
+#     print("ERROR %s" % e.message)
+#     raise
+# except IOError as e:
+#     print("ERROR %s" % e.message)
+#     raise
+# except:
+#     print("Unexpected error:", sys.exc_info()[0])
+#     raise

wanna/__init__.py

@@ -1,7 +1,7 @@
 from wanna.vendors.aws import _AWS
 
 
-__version__ = '0.1.9-beta1'
+__version__ = '0.1.9-beta3'
 
 ALIASES = {
     's3': _AWS,
@@ -23,7 +23,7 @@ def setup_vendor(vendor_str, bucket=None, use_encryption=True, ignore_prefix=Fal
         bucket=bucket, use_encryption=use_encryption, ignore_prefix=ignore_prefix, profile=profile, **other)
 
 
-def Transfer(vendor='aws', bucket=None, use_encrpytion=True, ignore_prefix=True, profile=None):
+def Transfer(vendor='aws', bucket=None, use_encrpytion=True, ignore_prefix=False, profile=None):
     """A proxy to vendor"""
     return setup_vendor(
         vendor, bucket=bucket, use_encryption=use_encrpytion, ignore_prefix=ignore_prefix, profile=profile

wanna/vendors/aws/__init__.py

@@ -53,7 +53,7 @@ def _get_config(self, config):
     def __init__(self, bucket=None, use_encryption=True, ignore_prefix=False, humanized=False, profile=None):
         config = Config(self, profile=profile)
         self._bucket = config.BUCKET if not bucket else bucket
-        self._prefix = os.path.join(config.UPLOAD_PREFIX, config.PARTNER_NAME)
+        self._default_prefix = os.path.join(config.UPLOAD_PREFIX, config.PARTNER_NAME)
         self._encrypt = use_encryption
         self.client = boto3.client(self.service, **self._get_config(config))
         self.resource = boto3.resource('s3', **self._get_config(config))
@@ -63,7 +63,7 @@ def __init__(self, bucket=None, use_encryption=True, ignore_prefix=False, humani
         self.config = config
         self._humanized = humanized
 
-        if ignore_prefix:
+        if self.ignore_prefix:
             self._ignore_prefix()
 
     def get_encryption_key(self, key=None):
@@ -77,7 +77,7 @@ def get_encryption_key(self, key=None):
 
     def _ignore_prefix(self):
         LOG.debug('ignore prefix mode')
-        self._prefix = ''
+        self._default_prefix = ''
 
     def _get_extra_args(self, encryption_key=None):
         """Extra parameters"""
@@ -90,12 +90,13 @@ def _get_extra_args(self, encryption_key=None):
             })
         return args
 
-    def get_obj_key(self, path, md5=False, ignore_prefix=False):
+    def get_obj_key(self, path, md5=False, ignore_prefix=False, prefix=None):
         """Get file object key"""
         if ignore_prefix is True or self.ignore_prefix:
             key = path
         else:
-            key = os.path.join(self._prefix, os.path.basename(path))
+            prefix = prefix or self._default_prefix or ""
+            key = os.path.join(prefix, os.path.basename(path))
         if md5 is True:
             key = key + self.hash_checksum
         return key
@@ -114,7 +115,7 @@ def upload_checksum(self, path):
         LOG.info('checksum ({}): {}'.format(self.hash_checksum, checksum))
         return response
 
-    def upload_files(self, path, add_checksum=False, progress=False, encryption_key=None):
+    def upload_files(self, path, add_checksum=False, progress=False, encryption_key=None, prefix=None):
         """Upload files"""
 
         def get_files():
@@ -124,7 +125,7 @@ def get_files():
 
         for item in get_files():
             itemname = os.path.basename(item)
-            key = self.get_obj_key(itemname)
+            key = self.get_obj_key(itemname, prefix=prefix)
             progress_callback = ProgressPercentage(item, humanized=self._humanized) if progress else lambda x: None
             extra_args = {} if self._encrypt is False else self._get_extra_args(encryption_key=encryption_key)
 
@@ -207,7 +208,7 @@ def download_file(self, path, dst='.', progress=False, use_encryption=None, encr
 
     def list_files(self):
         """List files"""
-        resp = self.client.list_objects_v2(Bucket=self._bucket, Prefix=self._prefix)
+        resp = self.client.list_objects_v2(Bucket=self._bucket, Prefix=self._default_prefix)
         if 'Contents' in resp:
             for el in resp['Contents']:
                 yield {'date': el['LastModified'], 'size': el['Size'], 'name': el['Key']}