diff --git a/README.md b/README.md index 950cde7..ed0089f 100644 --- a/README.md +++ b/README.md @@ -162,13 +162,33 @@ save_file.write(downloaded_file.read()) save_file.close() ``` -#### Delete a file +#### Delete a file version ```python file.delete() ``` -NOTE: There is no confirmation and this will delete all of a file's versions. +This deletes a single version of a file. (See the [docs on File Versions](https://www.backblaze.com/b2/docs/b2_delete_file_version.html) at Backblaze for explanation) + +#### Hide (aka "Soft-delete") a file + +```python +file.hide() +``` + +This hides a file (aka "soft-delete") so that downloading by name will not find the file, but previous versions of the file are still stored. (See the [docs on Hiding file](https://www.backblaze.com/b2/docs/b2_hide_file.html) at Backblaze for details) + +## Testing + +Unit testing with pytest +Before running, you must set the environment variables: `B2_KEY_ID` and `B2_APPLICATION_KEY` + +** Run tests ** + +``` bash +python3 ./tests.py +``` + ## LICENSE diff --git a/b2blaze/__init__.py b/b2blaze/__init__.py index dd65c78..cc61261 100644 --- a/b2blaze/__init__.py +++ b/b2blaze/__init__.py @@ -1 +1,2 @@ -from b2blaze.b2lib import B2 \ No newline at end of file +from b2blaze.b2lib import B2 +from .api import API_VERSION, BASE_URL, API \ No newline at end of file diff --git a/b2blaze/api.py b/b2blaze/api.py new file mode 100644 index 0000000..80ed85a --- /dev/null +++ b/b2blaze/api.py @@ -0,0 +1,22 @@ +# api.py +# BackBlaze API endpoints + +API_VERSION = '/b2api/v2' +BASE_URL = 'https://api.backblazeb2.com' + API_VERSION + + +class API(): + authorize = '/b2_authorize_account' + delete_file = '/b2_hide_file' + delete_file_version = '/b2_delete_file_version' + file_info = '/b2_get_file_info' + download_file_by_id = '/b2_download_file_by_id' + list_all_files = '/b2_list_file_names' + list_file_versions = '/b2_list_file_versions' + upload_url = '/b2_get_upload_url' + upload_large = '/b2_start_large_file' + upload_large_part = '/b2_get_upload_part_url' + upload_large_finish = '/b2_finish_large_file' + create_bucket = '/b2_create_bucket' + delete_bucket = '/b2_delete_bucket' + list_all_buckets = '/b2_list_buckets' \ No newline at end of file diff --git a/b2blaze/b2_exceptions.py b/b2blaze/b2_exceptions.py index 6cd00ae..ff28887 100644 --- a/b2blaze/b2_exceptions.py +++ b/b2blaze/b2_exceptions.py @@ -1,53 +1,116 @@ """ Copyright George Sibble 2018 """ -class B2ApplicationKeyNotSet(Exception): - """ - """ +import json + + +class B2ApplicationKeyNotSet(Exception): + """ You must set the B2_KEY_ID environment variable before running the application """ pass + class B2KeyIDNotSet(Exception): - """ + """ You must set the B2_APPLICATION_KEY environment variable before running the application """ + pass - """ + +class B2Exception(Exception): + """ Base exception class for the Backblaze API """ + + @staticmethod + def parse(response): + """ Parse the response error code and return the related error type. """ + + API_EXCEPTION_CODES = { + 400 : B2RequestError, + 401 : B2UnauthorizedError, + 403 : B2ForbiddenError, + 404 : B2FileNotFoundError, + 408 : B2RequestTimeoutError, + 429 : B2TooManyRequestsError, + 500 : B2InternalError, + 503 : B2ServiceUnavailableError, + } + + try: + response_json = response.json() + message = response_json['message'] + code = response_json['code'] + status = int(response_json['status']) + + # Return B2Exception if unrecognized status code + if not status in API_EXCEPTION_CODES: + return B2Exception('{} - {}: {}'.format(status, code, message)) + + ErrorClass = API_EXCEPTION_CODES[status] + return ErrorClass('{} - {}: {}'.format(status, code, message)) + + except: + return Exception('error parsing response. status code - {} Response JSON: {}'.format(response.status_code, response_json) ) + +class B2FileNotFoundError(Exception): + """ 404 Not Found """ pass -class B2InvalidBucketName(Exception): - """ - """ +class B2RequestError(Exception): + """ There is a problem with a passed in request parameters. See returned message for details """ pass -class B2InvalidBucketConfiguration(Exception): - """ +class B2UnauthorizedError(Exception): + """ When calling b2_authorize_account, this means that there was something wrong with the accountId/applicationKeyId or with the applicationKey that was provided. The code unauthorized means that the application key is bad. The code unsupported means that the application key is only valid in a later version of the API. + + The code unauthorized means that the auth token is valid, but does not allow you to make this call with these parameters. When the code is either bad_auth_token or expired_auth_token you should call b2_authorize_account again to get a new auth token. """ pass -class B2AuthorizationError(Exception): - """ +class B2ForbiddenError(Exception): + """ You have a reached a storage cap limit, or account access may be impacted in some other way; see the human-readable message. """ pass -class B2BucketCreationError(Exception): - """ - """ +class B2RequestTimeoutError(Exception): + """ The service timed out trying to read your request. """ pass -class B2RequestError(Exception): - """ +class B2OutOfRangeError(Exception): + """ The Range header in the request is outside the size of the file.. """ + pass - """ -class B2InvalidRequestType(Exception): - """ +class B2TooManyRequestsError(Exception): + """ B2 may limit API requests on a per-account basis. """ + pass - """ -class B2FileNotFound(Exception): +class B2InternalError(Exception): + """ An unexpected error has occurred. """ + pass + + +class B2ServiceUnavailableError(Exception): + """ The service is temporarily unavailable. The human-readable message identifies the nature of the issue, in general we recommend retrying with an exponential backoff between retries in response to this error. """ + pass + - """ \ No newline at end of file +class B2InvalidBucketName(Exception): + """ Bucket name must be alphanumeric or '-' """ + pass + + +class B2InvalidBucketConfiguration(Exception): + """ Value error in bucket configuration """ + pass + +class B2AuthorizationError(Exception): + """ An error with the authorization request """ + pass + +class B2InvalidRequestType(Exception): + """ Request type must be get or post """ + pass diff --git a/b2blaze/b2lib.py b/b2blaze/b2lib.py index ecf5a54..a5eebdd 100644 --- a/b2blaze/b2lib.py +++ b/b2blaze/b2lib.py @@ -2,10 +2,8 @@ Copyright George Sibble 2018 """ import os -from b2blaze.b2_exceptions import B2ApplicationKeyNotSet, B2KeyIDNotSet, B2InvalidBucketName, B2InvalidBucketConfiguration -from b2blaze.b2_exceptions import B2BucketCreationError +from b2blaze.b2_exceptions import B2ApplicationKeyNotSet, B2KeyIDNotSet from b2blaze.connector import B2Connector - from b2blaze.models.bucket_list import B2Buckets class B2(object): diff --git a/b2blaze/connector.py b/b2blaze/connector.py index 2d9efd4..be4a3e0 100644 --- a/b2blaze/connector.py +++ b/b2blaze/connector.py @@ -4,17 +4,16 @@ import requests import datetime from requests.auth import HTTPBasicAuth -from b2blaze.b2_exceptions import B2AuthorizationError, B2RequestError, B2InvalidRequestType +from b2blaze.b2_exceptions import B2Exception, B2AuthorizationError, B2InvalidRequestType import sys from hashlib import sha1 from b2blaze.utilities import b2_url_encode, decode_error, get_content_length, StreamWithHashProgress +from .api import BASE_URL, API_VERSION, API class B2Connector(object): """ """ - auth_url = 'https://api.backblazeb2.com/b2api/v1' - def __init__(self, key_id, application_key): """ @@ -53,22 +52,23 @@ def _authorize(self): :return: """ - path = self.auth_url + '/b2_authorize_account' + path = BASE_URL + API.authorize + result = requests.get(path, auth=HTTPBasicAuth(self.key_id, self.application_key)) if result.status_code == 200: result_json = result.json() self.authorized_at = datetime.datetime.utcnow() self.account_id = result_json['accountId'] self.auth_token = result_json['authorizationToken'] - self.api_url = result_json['apiUrl'] + '/b2api/v1' - self.download_url = result_json['downloadUrl'] + '/file/' + self.api_url = result_json['apiUrl'] + API_VERSION + self.download_url = result_json['downloadUrl'] + API_VERSION + API.download_file_by_id self.recommended_part_size = result_json['recommendedPartSize'] self.api_session = requests.Session() self.api_session.headers.update({ 'Authorization': self.auth_token }) else: - raise B2AuthorizationError(decode_error(result)) + raise B2Exception.parse(result) def make_request(self, path, method='get', headers={}, params={}, account_id_required=False): @@ -164,7 +164,7 @@ def download_file(self, file_id): :param file_id: :return: """ - download_by_id_url = self.download_url.split('file/')[0] + '/b2api/v1/b2_download_file_by_id' + url = self.download_url params = { 'fileId': file_id } @@ -172,5 +172,5 @@ def download_file(self, file_id): 'Authorization': self.auth_token } - return requests.get(download_by_id_url, headers=headers, params=params) + return requests.get(url, headers=headers, params=params) diff --git a/b2blaze/models/b2_file.py b/b2blaze/models/b2_file.py index c52dea3..320ca34 100644 --- a/b2blaze/models/b2_file.py +++ b/b2blaze/models/b2_file.py @@ -3,7 +3,8 @@ """ from io import BytesIO from ..utilities import b2_url_encode, b2_url_decode, decode_error -from ..b2_exceptions import B2RequestError +from ..b2_exceptions import B2Exception +from ..api import API class B2File(object): """ @@ -27,8 +28,8 @@ def __init__(self, connector, parent_list, fileId, fileName, contentSha1, conten :param kwargs: """ self.file_id = fileId - #self.file_name = b2_url_decode(fileName) - #TODO: Find out if this is necessary + # self.file_name_decoded = b2_url_decode(fileName) + #TODO: Find out if this is necessary self.file_name = fileName self.content_sha1 = contentSha1 self.content_length = contentLength @@ -40,41 +41,107 @@ def __init__(self, connector, parent_list, fileId, fileName, contentSha1, conten self.parent_list = parent_list self.deleted = False - def delete(self): - """ + def get_versions(self, limit=None): + """ Fetch list of all versions of the current file. + Params: + limit: (int) Limit number of results returned (optional, default 10000) - :return: + Returns: + file_versions (list) B2FileObject of all file versions """ - path = '/b2_delete_file_version' + bucket_id = self.parent_list.bucket.bucket_id + + path = API.list_file_versions + file_versions = [] params = { - 'fileId': self.file_id, + 'bucketId': bucket_id, + 'maxFileCount': limit or 10000, + 'startFileId': self.file_id, + 'startFileName': self.file_name, + } + + response = self.connector.make_request(path=path, method='post', params=params) + if response.status_code == 200: + files_json = response.json() + for file_json in files_json['files']: + new_file = B2File(connector=self.connector, parent_list=self, **file_json) + file_versions.append(new_file) + else: + raise B2Exception.parse(response) + return file_versions + + + def hide(self): + """ Soft-delete a file (hide it from files list, but previous versions are saved.) """ + path = API.delete_file + params = { + 'bucketId': self.parent_list.bucket.bucket_id, 'fileName': b2_url_encode(self.file_name) } response = self.connector.make_request(path=path, method='post', params=params) if response.status_code == 200: self.deleted = True - del self.parent_list._files_by_name[self.file_name] - del self.parent_list._files_by_id[self.file_id] + # Delete from parent list if exists + self.parent_list._files_by_name.pop(self.file_name) + self.parent_list._files_by_id.pop(self.file_id) else: - raise B2RequestError(decode_error(response)) - #TODO: Raise Error + raise B2Exception.parse(response) - def download(self): - """ - :return: + def delete_all_versions(self, confirm=False): + """ Delete completely all versions of a file. + ** NOTE THAT THIS CAN BE VERY EXPENSIVE IN TERMS OF YOUR API LIMITS ** + Each call to delete_all_versions will result in multiple API calls: + One API call per file version to be deleted, per file. + 1. Call '/b2_list_file_versions' to get file versions + 2. Call '/b2_delete_file_version' once for each version of the file + + This means: if you have 10 files with 50 versions each and call delete_all_versions, + you will spend (10 + 1) x 50 == 550 API calls against your BackBlaze b2 API limit. + + ** You have been warned! BE CAREFUL!!! ** """ + print(self.delete_all_versions.__name__, self.delete_all_versions.__doc__) # Print warnings at call time. + + # Confirm deletion + if not confirm: + print('To call this function, use delete_all_versions(confirm=True)') + return False + + versions = self.get_versions() + + version_count = len(versions) + if not version_count > 0: + print('No file versions') + else: + print(version_count, 'file versions') + for count, v in enumerate(versions): + print('deleting [{}/{}]'.format(count + 1 , version_count)) + v.delete() + + + def delete(self): + """ Delete a file version (Does not delete entire file history: only most recent version) """ + path = API.delete_file_version + params = { + 'fileId': self.file_id, + 'fileName': b2_url_encode(self.file_name) + } + response = self.connector.make_request(path=path, method='post', params=params) + if not response.status_code == 200: + raise B2Exception.parse(response) + self.deleted = True + + + def download(self): + """ Download latest file version """ response = self.connector.download_file(file_id=self.file_id) if response.status_code == 200: return BytesIO(response.content) else: - raise B2RequestError(decode_error(response)) + raise B2Exception.parse(response) @property def url(self): - """ - - :return: file download url - """ - return self.connector.download_url.split('file/')[0] \ - + 'b2api/v1/b2_download_file_by_id?fileId=' + self.file_id \ No newline at end of file + """ Return file download URL """ + return self.connector.download_url + '?fileId=' + self.file_id diff --git a/b2blaze/models/bucket.py b/b2blaze/models/bucket.py index d1ca1e3..693ff32 100644 --- a/b2blaze/models/bucket.py +++ b/b2blaze/models/bucket.py @@ -2,9 +2,8 @@ Copyright George Sibble 2018 """ from .file_list import B2FileList -from ..b2_exceptions import B2RequestError -from ..utilities import decode_error - +from ..b2_exceptions import B2Exception +from ..api import API class B2Bucket(object): """ @@ -37,15 +36,23 @@ def __init__(self, connector, parent_list, bucketId, bucketName, bucketType, buc self.parent_list = parent_list self.deleted = False - def delete(self): - """ + def delete(self, delete_files=False, confirm_non_empty=False): + """ Delete a bucket. - :return: + Params: + delete_files: (bool) Delete all files first. + confirm_non_empty: (bool) Confirm deleting on bucket not empty. """ - path = '/b2_delete_bucket' - files = self.files.all() - for file in files: - file.delete() + path = API.delete_bucket + files = self.files.all(include_hidden=True) + if delete_files: + if not confirm_non_empty: + raise B2Exception('Bucket is not empty! Must confirm deletion of all files with confirm_non_empty=True') + else: + print("Deleting all files from bucket. Beware API limits!") + self.files.delete_all(confirm=True) + + params = { 'bucketId': self.bucket_id } @@ -55,7 +62,7 @@ def delete(self): del self.parent_list._buckets_by_name[self.bucket_name] del self.parent_list._buckets_by_id[self.bucket_id] else: - raise B2RequestError(decode_error(response)) + raise B2Exception.parse(response) def edit(self): #TODO: Edit details @@ -63,8 +70,5 @@ def edit(self): @property def files(self): - """ - - :return: - """ + """ List of files in the bucket. B2FileList instance. """ return B2FileList(connector=self.connector, bucket=self) \ No newline at end of file diff --git a/b2blaze/models/bucket_list.py b/b2blaze/models/bucket_list.py index e8eac2c..5ea558f 100644 --- a/b2blaze/models/bucket_list.py +++ b/b2blaze/models/bucket_list.py @@ -2,9 +2,9 @@ Copyright George Sibble 2018 """ -from ..b2_exceptions import B2InvalidBucketName, B2InvalidBucketConfiguration, B2BucketCreationError, B2RequestError -from ..utilities import decode_error +from ..b2_exceptions import B2Exception, B2InvalidBucketName, B2InvalidBucketConfiguration from .bucket import B2Bucket +from ..api import API class B2Buckets(object): @@ -36,7 +36,7 @@ def _update_bucket_list(self, retrieve=False): :param retrieve: :return: """ - path = '/b2_list_buckets' + path = API.list_all_buckets response = self.connector.make_request(path=path, method='post', account_id_required=True) if response.status_code == 200: response_json = response.json() @@ -51,7 +51,7 @@ def _update_bucket_list(self, retrieve=False): if retrieve: return buckets else: - raise B2RequestError(decode_error(response)) + raise B2Exception.parse(response) def get(self, bucket_name=None, bucket_id=None): """ @@ -73,9 +73,9 @@ def create(self, bucket_name, security, configuration=None): :param configuration: :return: """ - path = '/b2_create_bucket' + path = API.create_bucket if type(bucket_name) != str and type(bucket_name) != bytes: - raise B2InvalidBucketName + raise B2InvalidBucketName("Bucket name must be alphanumeric or '-") if type(configuration) != dict and configuration is not None: raise B2InvalidBucketConfiguration params = { @@ -93,4 +93,4 @@ def create(self, bucket_name, security, configuration=None): self._buckets_by_id[bucket_json['bucketId']] = new_bucket return new_bucket else: - raise B2RequestError(decode_error(response)) \ No newline at end of file + raise B2Exception.parse(response) \ No newline at end of file diff --git a/b2blaze/models/file_list.py b/b2blaze/models/file_list.py index 952dd62..ecf8c1e 100644 --- a/b2blaze/models/file_list.py +++ b/b2blaze/models/file_list.py @@ -2,11 +2,11 @@ Copyright George Sibble 2018 """ -from ..b2_exceptions import B2InvalidBucketName, B2InvalidBucketConfiguration, B2BucketCreationError from .b2_file import B2File from ..utilities import b2_url_encode, get_content_length, get_part_ranges, decode_error, RangeStream, StreamWithHashProgress -from ..b2_exceptions import B2RequestError, B2FileNotFound +from ..b2_exceptions import B2Exception, B2FileNotFoundError from multiprocessing.dummy import Pool as ThreadPool +from ..api import API class B2FileList(object): """ @@ -23,25 +23,56 @@ def __init__(self, connector, bucket): self._files_by_name = {} self._files_by_id = {} - def all(self): - """ + def all(self, include_hidden=False, limit=None): + """ Return an updated list of all files. + (This does not include hidden files unless include_hidden flag set to True) - :return: - """ - return self._update_files_list(retrieve=True) + Parameters: + include_hidden: (bool) Include hidden files + limit: (int) Limit number of file results - def _update_files_list(self, retrieve=False): """ + if not include_hidden: + return self._update_files_list(retrieve=True, limit=limit) + else: + results = self.all_file_versions(limit=limit) + versions = results['file_versions'] + file_ids = results['file_ids'] + if versions: + # Return only the first file from a given file with multiple versions + files = [versions[f][0] for f in file_ids] + return files + return [] # Return empty set on no results - :param retrieve: - :return: + def delete_all(self, confirm=False): + """ Delete all files in the bucket. + Parameters: + confirm: (bool) Safety check. Confirm deletion + """ + if not confirm: + raise Exception('This will delete all files! Pass confirm=True') + + all_files = self.all(include_hidden=True) + try: + for f in all_files: + f.delete_all_versions(confirm=True) + except Exception as E: + raise B2Exception.parse(E) + return [] + + + def _update_files_list(self, retrieve=False, limit=None): + """ Retrieve list of all files in bucket + Parameters: + limit: (int) Max number of file results, default 10000 + retrieve: (bool) Refresh local store. (default: false) """ - path = '/b2_list_file_names' + path = API.list_all_files files = [] new_files_to_retrieve = True params = { 'bucketId': self.bucket.bucket_id, - 'maxFileCount': 10000 + 'maxFileCount': limit or 10000 } while new_files_to_retrieve: response = self.connector.make_request(path=path, method='post', params=params) @@ -59,53 +90,140 @@ def _update_files_list(self, retrieve=False): else: params['startFileName'] = files_json['nextFileName'] else: - raise B2RequestError(decode_error(response)) + raise B2Exception.parse(response) if retrieve: return files + def get(self, file_name=None, file_id=None): - """ + """ Get a file by file name or id. + Required: + file_name or file_id - :param file_name: - :param file_id: - :return: + Parameters: + file_name: (str) File name + file_id: (str) File ID """ - if file_name is not None: - path = '/b2_list_file_names' - params = { - 'prefix': b2_url_encode(file_name), - 'bucketId': self.bucket.bucket_id - } + if file_name: + file = self._get_by_name(file_name) + + elif file_id: + file = self._get_by_id(file_id) + else: + raise ValueError('file_name or file_id must be passed') + + return file + + + def get_versions(self, file_name=None, file_id=None, limit=None): + """ Return list of all the versions of one file in current bucket. + Required: + file_id or file_name (either) + + Params: + file_id: (str) File id + file_name: (str) File id + limit: (int) Limit number of results returned (optional) + + Returns: + file_versions (list) B2FileObject of all file versions + """ + if file_name: + file = self.get(file_name) + + elif file_id: + file = self.get(file_id=file_id) + else: + raise ValueError('Either file_id or file_name required for get_versions') + return file.get_versions() + + + def all_file_versions(self, limit=None): + """ Return all the versions of all files in a given bucket. + + Params: + limit: (int) Limit number of results returned (optional). Defaults to 10000 + + Returns dict: + 'file_names': (list) String filenames + 'file_ids': (list) File IDs + 'file_versions': (dict) b2blaze File objects, keyed by file name + """ + + path = API.list_file_versions + file_versions = dict() + file_names = [] + file_ids = [] + new_files_to_retrieve = True + params = { + 'bucketId': self.bucket.bucket_id, + 'maxFileCount': 10000 + } + + # Limit files + if limit: + params['maxFileCount'] = limit + + while new_files_to_retrieve: - response = self.connector.make_request(path, method='post', params=params) + response = self.connector.make_request(path=path, method='post', params=params) if response.status_code == 200: - file_json = response.json() - if len(file_json['files']) > 0: - return B2File(connector=self.connector, parent_list=self, **file_json['files'][0]) + files_json = response.json() + for file_json in files_json['files']: + new_file = B2File(connector=self.connector, parent_list=self, **file_json) + + # Append file_id, file_name to lists + file_name, file_id = file_json['fileName'], file_json['fileId'] + file_names.append(file_name) + file_ids.append(file_id) + + # Add file to list keyed by file_id + if file_id in file_versions: + file_versions[file_id].append(new_file) + else: + file_versions[file_id] = [new_file] + + if files_json['nextFileName'] is None: + new_files_to_retrieve = False else: - raise B2FileNotFound('fileName - ' + file_name) - else: - raise B2RequestError(decode_error(response)) - elif file_id is not None: - path = '/b2_get_file_info' - params = { - 'fileId': file_id - } - response = self.connector.make_request(path, method='post', params=params) - if response.status_code == 200: - file_json = response.json() - return B2File(connector=self.connector, parent_list=self, **file_json) + params['startFileName'] = files_json['nextFileName'] else: - raise B2RequestError(decode_error(response)) + raise B2Exception.parse(response) + return {'file_names': file_names, 'file_versions': file_versions, 'file_ids': file_ids} + + + def _get_by_name(self, file_name): + """ Internal method, return single file by file name """ + path = API.list_all_files + params = { + 'prefix': b2_url_encode(file_name), + 'bucketId': self.bucket.bucket_id + } + + response = self.connector.make_request(path, method='post', params=params) + file_json = response.json() + + # Handle errors and empty files + if not response.status_code == 200: + raise B2Exception.parse(response) + if not len(file_json['files']) > 0: + raise B2FileNotFoundError('Filename {} not found'.format(file_name)) else: - raise ValueError('file_name or file_id must be passed') + return B2File(connector=self.connector, parent_list=self, **file_json['files'][0]) - # self._update_files_list() - # if file_name is not None: - # return self._files_by_name.get(file_name, None) - # else: - # return self._files_by_id.get(file_id, None) - # pass + def _get_by_id(self, file_id): + """ Internal method, return single file by file id """ + path = API.file_info + params = { + 'fileId': file_id + } + response = self.connector.make_request(path, method='post', params=params) + if response.status_code == 200: + file_json = response.json() + return B2File(connector=self.connector, parent_list=self, **file_json) + else: + raise B2Exception.parse(response) + def upload(self, contents, file_name, mime_content_type=None, content_length=None, progress_listener=None): """ @@ -119,7 +237,7 @@ def upload(self, contents, file_name, mime_content_type=None, content_length=Non """ if file_name[0] == '/': file_name = file_name[1:] - get_upload_url_path = '/b2_get_upload_url' + get_upload_url_path = API.upload_url params = { 'bucketId': self.bucket.bucket_id } @@ -132,11 +250,13 @@ def upload(self, contents, file_name, mime_content_type=None, content_length=Non content_length=content_length, progress_listener=progress_listener) if upload_response.status_code == 200: new_file = B2File(connector=self.connector, parent_list=self, **upload_response.json()) + # Update file list after upload + self._update_files_list() return new_file else: - raise B2RequestError(decode_error(upload_response)) + raise B2Exception.parse(upload_response) else: - raise B2RequestError(decode_error(upload_url_response)) + raise B2Exception.parse(upload_url_response) def upload_large_file(self, contents, file_name, part_size=None, num_threads=4, mime_content_type=None, content_length=None, progress_listener=None): @@ -157,7 +277,7 @@ def upload_large_file(self, contents, file_name, part_size=None, num_threads=4, part_size = self.connector.recommended_part_size if content_length == None: content_length = get_content_length(contents) - start_large_file_path = '/b2_start_large_file' + start_large_file_path = API.upload_large params = { 'bucketId': self.bucket.bucket_id, 'fileName': b2_url_encode(file_name), @@ -166,7 +286,7 @@ def upload_large_file(self, contents, file_name, part_size=None, num_threads=4, large_file_response = self.connector.make_request(path=start_large_file_path, method='post', params=params) if large_file_response.status_code == 200: file_id = large_file_response.json().get('fileId', None) - get_upload_part_url_path = '/b2_get_upload_part_url' + get_upload_part_url_path = API.upload_large_part params = { 'fileId': file_id } @@ -187,13 +307,13 @@ def upload_part_worker(args): if upload_part_response.status_code == 200: return upload_part_response.json().get('contentSha1', None) else: - raise B2RequestError(decode_error(upload_part_response)) + raise B2Exception.parse(upload_part_response) else: - raise B2RequestError(decode_error(upload_part_url_response)) + raise B2Exception.parse(upload_part_url_response) sha_list = pool.map(upload_part_worker, enumerate(get_part_ranges(content_length, part_size), 1)) pool.close() pool.join() - finish_large_file_path = '/b2_finish_large_file' + finish_large_file_path = API.upload_large_finish params = { 'fileId': file_id, 'partSha1Array': sha_list @@ -203,6 +323,6 @@ def upload_part_worker(args): new_file = B2File(connector=self.connector, parent_list=self, **finish_large_file_response.json()) return new_file else: - raise B2RequestError(decode_error(finish_large_file_response)) + raise B2Exception.parse(finish_large_file_response) else: - raise B2RequestError(decode_error(large_file_response)) + raise B2Exception.parse(large_file_response) diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..e82f12d --- /dev/null +++ b/changelog.md @@ -0,0 +1,29 @@ + +## Changelog + +#### v0.2.1 + +**Python version update** +Python version: +- Python 3 required! + +**Nonbreaking API changes:** + +B2File: +- hide added +- delete_all_versions(confirm=False) added + +B2FileList +- all(introduced new parameter: include_hidden) +- delete_all(confirm=False) added +- get_versions(file_name=None, file_id=None, limit=None) added +- all_file_versions(limit=None) added + +b2_exceptions.py +- Changed API error classes to match Backblaze API docs +- added base API exception B2Exception. +- Handle non-200 status code responses with 'raise B2Exception.parse(response)` + +tests.py +- added and cleaned up integration tests +- tests can now be run with `python tests.py` diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..c8c1cf7 --- /dev/null +++ b/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash +echo Building and installing $(pwd) + +echo "Clean build..." +/usr/bin/python3 setup.py clean --all && echo -e "OK\n" + +echo "Build..." +/usr/bin/python3 setup.py build && echo -e "OK\n" + +echo "Install" +sudo echo "Enter sudo password for install" +sudo -H /usr/bin/python3 setup.py install && echo -e "OK\n" + diff --git a/requirements.txt b/requirements.txt index 9e8b7cd..3fd3082 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,48 +1,13 @@ -asn1crypto==0.24.0 -astroid==1.6.5 -atomicwrites==1.1.5 -attrs==18.1.0 -backports.functools-lru-cache==1.5 -blessings==1.7 -bpython==0.17.1 -certifi==2018.4.16 -cffi==1.11.5 +atomicwrites==1.2.1 +attrs==18.2.0 +certifi==2018.10.15 chardet==3.0.4 -configparser==3.5.0 coverage==4.5.1 -cryptography==2.3 -curtsies==0.3.0 -enum34==1.1.6 -funcsigs==1.0.2 -futures==3.2.0 -greenlet==0.4.13 idna==2.7 -ipaddress==1.0.22 -isort==4.3.4 -lazy-object-proxy==1.3.1 -mccabe==0.6.1 -mock==2.0.0 -more-itertools==4.2.0 -ndg-httpsclient==0.5.0 +more-itertools==4.3.0 nose==1.3.7 -pbr==4.0.4 -pkginfo==1.4.2 pluggy==0.6.0 py==1.5.4 -pyasn1==0.4.3 -pycparser==2.18 -Pygments==2.2.0 -pylint==1.9.2 -pyOpenSSL==18.0.0 pytest==3.6.2 requests==2.19.1 -requests-toolbelt==0.8.0 -singledispatch==3.4.0.3 -six==1.11.0 -sure==1.4.11 -tqdm==4.23.4 -twine==1.11.0 -typing==3.6.4 urllib3==1.23 -wcwidth==0.1.7 -wrapt==1.10.11 diff --git a/setup.py b/setup.py index d572c19..5d33deb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.10' +VERSION = '0.2.1' from os import path this_directory = path.abspath(path.dirname(__file__)) @@ -11,14 +11,14 @@ setup(name='b2blaze', version=VERSION, - description='b2blaze library for connecting to Backblaze B2', + description='Forked from George Sibble\'s B2Blaze (0.1.10). All credits to author. Original package: https://github.com/sibblegp/b2blaze', long_description=long_description, long_description_content_type='text/markdown', packages=find_packages(), author='George Sibble', author_email='gsibble@gmail.com', python_requires='>=2.7', - url='https://github.com/sibblegp/b2blaze', + url='https://github.com/fieldse/b2blaze-fork', install_requires=[ 'requests==2.19.1' ], diff --git a/tests.py b/tests.py index b793e67..ce026a9 100644 --- a/tests.py +++ b/tests.py @@ -2,17 +2,13 @@ Copyright George Sibble 2018 """ import b2blaze.b2lib -import sure -from sure import expect -import random -import string +from datetime import datetime import pytest -from b2blaze.b2_exceptions import B2RequestError, B2FileNotFound +from b2blaze.b2_exceptions import B2Exception, B2RequestError, B2FileNotFoundError class TestB2(object): - """ + """ Tests for the b2blaze library """ - """ @classmethod def setup_class(cls): """ @@ -20,119 +16,337 @@ def setup_class(cls): :return: None """ cls.b2 = b2blaze.b2lib.B2() - cls.bucket_name = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase) for _ in range(7)) - print(cls.bucket_name) + timestamp = datetime.now().strftime('%Y-%m-%d-%H%M%S') + cls.bucket_name = 'testbucket-' + timestamp + print('test bucket: ', cls.bucket_name) + # Helper methods def test_create_b2_instance(self): - """ - - :return: None - """ + """Create a B2 instance """ b2 = b2blaze.b2lib.B2() - def test_create_bucket(self): - """ - :return: None - """ + @classmethod + def create_bucket(cls): + return cls.b2.buckets.create(cls.bucket_name, security=cls.b2.buckets.public) + + @classmethod + def getbucket(cls): + return cls.b2.buckets.get(bucket_name=cls.bucket_name) or cls.create_bucket() + + @classmethod + def upload_textfile(cls, contents="hello there", file_name='test/hello.txt'): + """ Upload text file with name 'test/hello.txt' """ + contents=contents.encode('utf-8') # These fail unless encoded to UTF8 + bucket = cls.getbucket() + return bucket.files.upload(contents=contents, file_name=file_name) + + + @classmethod + def is_b2_file(cls, obj): + """ hacky method for checking object class/type is B2File""" + if 'B2File' in str(type(obj)): + return True + return False + + ## Tests ## + @pytest.mark.bucket + @pytest.mark.files + @pytest.mark.versions + def test_create_bucket(self): + """ Create a bucket by name. """ self.bucket = self.b2.buckets.create(self.bucket_name, security=self.b2.buckets.public) + assert self.bucket - def test_create_file_and_retrieve_by_id(self): - """ + @pytest.mark.bucket + def test_get_bucket(self): + """ Get a bucket by name """ + bucket = self.getbucket() + assert bucket - :return: None - """ - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) - file = bucket.files.upload(contents='Hello World!', file_name='test/hello.txt') + @pytest.mark.bucket + def test_get_all_buckets(self): + """ Get buckets. Number of buckets returned should be >1 """ + buckets = self.b2.buckets.all() + assert len(buckets) > 1, "Number of buckets returned should be >1" + + @pytest.mark.bucket + def test_get_nonexistent_bucket(self): + """ Get a bucket which doesn't exist should return None """ + bucket = self.b2.buckets.get(bucket_name='this doesnt exist') + assert not bucket + + @pytest.mark.files + def test_create_file_and_retrieve_by_id(self): + """ Create a file and retrieve by ID """ + bucket = self.getbucket() + contents='Hello World!'.encode('utf-8') # These fail unless encoded to UTF8 + file = bucket.files.upload(contents=contents, file_name='test/hello.txt') file2 = bucket.files.get(file_id=file.file_id) - def test_create_z_binary_file(self): - """ + # It should be a B2File + assert self.is_b2_file(file2), 'Should be a B2File object' - :return: - """ - #from time import sleep - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) - binary_file = open('b2blaze/test_pic.jpg', 'rb') - uploaded_file = bucket.files.upload(contents=binary_file.read(), file_name='test_pic.jpg') - binary_file.close() - #sleep(3) - #downloaded_file = uploaded_file.download() - #save_file = open('save_pic.jpg', 'wb') - #save_file.write(downloaded_file.read()) - #save_file.close() + @pytest.mark.files def test_direct_upload_file(self): - """ - - :return: - """ - #from time import sleep - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) + """ Upload binary file """ + bucket = self.getbucket() binary_file = open('b2blaze/test_pic.jpg', 'rb') uploaded_file = bucket.files.upload(contents=binary_file, file_name='test_pic2.jpg') binary_file.close() + assert self.is_b2_file(uploaded_file) - def test_download_file(self): - """ - :return: None + @pytest.mark.files + def test_get_all_files(self): + """ Get all files from a bucket. Returned objects are B2Files """ + bucket = self.getbucket() + files = bucket.files.all() + print('test_get_files: all files: ', len(files)) + + # check type + assert self.is_b2_file(files[0]), 'Should be a B2File object' + + + + @pytest.mark.versions + @pytest.mark.files + def test_get_all_file_versions(self): + """ Get all file versions from a bucket """ + bucket = self.getbucket() + file = self.upload_textfile() + files = bucket.files.all_file_versions() + print('test_get_all_file_versions: all versions: ', len(files['file_versions'])) + assert len(files['file_versions']) > 0, 'File versions should exist' + + + @pytest.mark.files + def test_get_file_by_name(self): + """ Get file by name """ + bucket = self.getbucket() + file = self.upload_textfile() + + # check type + assert self.is_b2_file(file), 'Should be a B2File object' + + + @pytest.mark.files + def test_get_file_by_id(self): + """ Get file by id """ + bucket = self.getbucket() + file = self.upload_textfile() + + # check type + assert self.is_b2_file(file), 'Should be a B2File object' + + + @pytest.mark.versions + @pytest.mark.files + def test_get_file_versions(self): + """ Get all versions of a file via the file.get_versions method. + Returned data should be a list, and items should be of type B2File """ - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) + bucket = self.getbucket() file = bucket.files.get(file_name='test/hello.txt') - file.download() + versions = file.get_versions() + assert len(versions) > 0, 'File should have multiple versions' + + # check type + assert self.is_b2_file(versions[0]), 'Should be a B2File object' - def test_download_url(self): + + @pytest.mark.versions + @pytest.mark.files + def test_bucket_get_file_versions_by_name(self): + """ Get all versions of a file by name file_list.get_versions method. + Returned data should be a list, and items should be of type B2File """ + bucket = self.getbucket() + versions = bucket.files.get_versions(file_name='test/hello.txt') + assert len(versions) > 0, 'File should have multiple versions' + assert self.is_b2_file(versions[0]), 'Should be a B2File object' - :return: None + + @pytest.mark.versions + @pytest.mark.files + def test_bucket_get_file_versions_by_id(self): + """ Get all versions of a file by id file_list.get_versions method. + Returned data should be a list, and items should be of type B2File """ - import requests - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) + bucket = self.getbucket() file = bucket.files.get(file_name='test/hello.txt') + versions = bucket.files.get_versions(file_id=file.file_id) + assert len(versions) > 0, 'File should have multiple versions' + assert self.is_b2_file(versions[0]), 'Should be a B2File object' + + + @pytest.mark.files + @pytest.mark.b2errors + def test_get_file_doesnt_exist(self): + """ Get file which doesn't exist should raise B2FileNotFoundError, get by ID should raise B2RequestError """ + bucket = self.getbucket() + with pytest.raises(B2FileNotFoundError): + file = bucket.files.get(file_name='nope.txt') + with pytest.raises(B2RequestError): + file2 = bucket.files.get(file_id='abcd') + + + @pytest.mark.files + def test_download_file(self): + """ Get file by id """ + bucket = self.getbucket() + file = self.upload_textfile() + data = file.download() + assert len(data.read()) > 0 + + + @pytest.mark.files + def test_download_url(self): + """ Download file url should be publicly GET accessible """ + import requests + bucket = self.getbucket() + file = self.upload_textfile() url = file.url downloaded_file = requests.get(url) if downloaded_file.status_code != 200: print(downloaded_file.json()) raise ValueError - def test_get_buckets(self): - """ - :return: None + @pytest.mark.files + @pytest.mark.b2errors + def test_hide_file(self): + """ Should create + upload, then hide / soft-delete a file by name. + File should no longer be returned when searched by name in bucket. """ - buckets = self.b2.buckets.all() - expect(len(buckets)).should.be.greater_than(1) + bucket = self.getbucket() + upload = self.upload_textfile(contents='Delete this', file_name='test/deleteme.txt') + + # Delete + print('test_delete_file: upload.file_id', upload.file_id) + print('test_delete_file: upload.file_name', upload.file_name) + upload.hide() - def test_get_files(self): - """ + # Refresh bucket; getting the the file should fail + with pytest.raises(B2FileNotFoundError): + bucket = self.getbucket() + file = bucket.files.get(file_name=upload.file_name) + assert not file, 'Deleted file should not be in files list' + + + @pytest.mark.files + @pytest.mark.versions + def test_delete_file_version(self): + """ Delete a file version by name. It should still exist when searched.""" + bucket = self.getbucket() + + # Upload file & delete + file = self.upload_textfile() + file2 = self.upload_textfile() + + # Update versions + versions = file.get_versions() + + assert len(versions) > 1, 'File should should have multiple version' + + # Delete version + print('test_delete_file_version: file_name', file.file_name) + print('test_delete_file_version: file_id', file.file_id) + file.delete() + + # Refresh bucket; getting the the file should fail + file2 = bucket.files.get(file_name=file.file_name) + assert file2, 'Deleted file version only, file should still exist' + assert self.is_b2_file(file2), 'Should be a B2File object' + + + # @pytest.mark.versions + @pytest.mark.files + def test_delete_all_file_versions(self): + """ Delete all versions of a file. It should be gone completely from bucket.""" + bucket = self.getbucket() + + # Create file, make sure we have multiple versions + contents='Hello World!'.encode('utf-8') # These fail unless encoded to UTF8 + upload = bucket.files.upload(contents=contents, file_name='test/hello.txt') + + # Get + # versions = bucket.files.get_versions(file_name='test/hello.txt') + file = bucket.files.get(file_name='test/hello.txt') + versions = file.get_versions() + assert len(versions) > 0, 'File should should have multiple version' + + # Delete + print('test_delete_all_file_versions: file_name', file.file_name) + print('test_delete_all_file_versions: file_id', file.file_id) + file.delete_all_versions(confirm=True) + + # Refresh bucket; getting the the file should fail + with pytest.raises(B2FileNotFoundError): + bucket = self.getbucket() + file2 = bucket.files.get(file_name=file.file_name) + assert not file2, 'Deleted all file versions, file should not exist' + + + @pytest.mark.bucket + def test_delete_non_empty_bucket(self): + """ Delete bucket should fail on bucket non-empty """ + bucket = self.getbucket() + + # Upload file + self.upload_textfile() + assert len(bucket.files.all()) > 0, "Bucket should still contain files" + + # Should raise exception on non-empty without confirm + with pytest.raises(B2RequestError): + bucket.delete() # Try to delete without confirmation + + # Bucket should still exist + assert self.b2.buckets.get(bucket_name=bucket.bucket_name), 'bucket should still exist' + + # # Delete with confirmation + # bucket.delete(delete_files=True, confirm_non_empty=True) + + # # Bucket should be gone + # assert self.b2.buckets.get(bucket_name=bucket.bucket_name), 'bucket should not exist' + + + @pytest.mark.bucket + @pytest.mark.files + @pytest.mark.versions + def test_bucket_delete_all_files(self): + """ Delete all files from bucket. """ + bucket = self.getbucket() + self.upload_textfile() - :return: None - """ - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) files = bucket.files.all() + assert len(files) > 0, 'Bucket should still contain files' + + # Delete all files + bucket.files.delete_all(confirm=True) + assert len(bucket.files.all()) == 0, 'Bucket should be empty' - def test_get_file_doesnt_exist(self): - """ - :return: - """ - bucket = self.b2.buckets.get(bucket_name=self.bucket_name) - with pytest.raises(B2FileNotFound): - file = bucket.files.get(file_name='nope.txt') - with pytest.raises(B2RequestError): - file2 = bucket.files.get(file_id='abcd') + @pytest.mark.bucket + def test_delete_bucket(self): + """ Delete empty bucket""" + bucket = self.getbucket() - def test_z_delete_bucket(self): - """ + # Ascertain it's empty + files_new = bucket.files.all(include_hidden=True) + assert len(files_new) == 0, "Bucket should contain no files but contains {}".format(len(files_new)) + + # Delete + bucket.delete() - :return: None - """ - self.bucket = self.b2.buckets.get(bucket_name=self.bucket_name) - self.bucket.delete() - #TODO: Assert cannot retrieve bucket by ID or name - - # def test_failure_to_create_bucket(self): - # expect(self.b2.create_bucket( - # ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase) - # for _ in range(4)))).should.have.raised(Exception) + # Confirm bucket is gone. bucket.get() nonexistent should return None. + assert not self.b2.buckets.get(bucket_name=bucket.bucket_name), 'Deleted bucket still exists' + +def main(): + import pytest + pytest_args = [ __file__, '--verbose'] + pytest.main(pytest_args) + +if __name__ == '__main__': + main() \ No newline at end of file