From 075541d9719ff7d5a325f79bdc72112bdc0260bb Mon Sep 17 00:00:00 2001 From: voidrank Date: Sat, 16 May 2015 16:51:08 +0800 Subject: [PATCH 1/2] Add multiple checksum algorithms support & make command delete_expired_uploads compatible with python3 --- chunked_upload/constants.py | 11 +++ .../commands/delete_expired_uploads.py | 6 +- chunked_upload/models.py | 84 ++++++++++++++++++- chunked_upload/views.py | 40 +++++---- 4 files changed, 122 insertions(+), 19 deletions(-) diff --git a/chunked_upload/constants.py b/chunked_upload/constants.py index 152f0bc..0fc17bc 100644 --- a/chunked_upload/constants.py +++ b/chunked_upload/constants.py @@ -16,3 +16,14 @@ class http_status: (COMPLETE, _('Complete')), (FAILED, _('Failed')), ) + +SUPPORTED_CHUCKSUM_ALGORITHMS = ( + 'md5', + 'sha1', + 'sha224' + 'sha256', + 'sha384', + 'sha512', + 'crc32', + 'adler32', +) diff --git a/chunked_upload/management/commands/delete_expired_uploads.py b/chunked_upload/management/commands/delete_expired_uploads.py index 458671f..f6f8a8d 100644 --- a/chunked_upload/management/commands/delete_expired_uploads.py +++ b/chunked_upload/management/commands/delete_expired_uploads.py @@ -46,6 +46,6 @@ def handle(self, *args, **options): # Deleting objects individually to call delete method explicitly chunked_upload.delete() - print '%i complete uploads were deleted.' % count[COMPLETE] - print '%i incomplete uploads were deleted.' % count[UPLOADING] - print '%i failed uploads were deleted.' % count[FAILED] + print('%i complete uploads were deleted.' % count[COMPLETE]) + print('%i incomplete uploads were deleted.' % count[UPLOADING]) + print('%i failed uploads were deleted.' % count[FAILED]) diff --git a/chunked_upload/models.py b/chunked_upload/models.py index 20dbe2d..28ac55a 100644 --- a/chunked_upload/models.py +++ b/chunked_upload/models.py @@ -1,6 +1,7 @@ import time import os.path import hashlib +import zlib import uuid from django.db import models @@ -9,7 +10,7 @@ from django.utils import timezone from .settings import EXPIRATION_DELTA, UPLOAD_PATH, STORAGE, ABSTRACT_MODEL -from .constants import CHUNKED_UPLOAD_CHOICES, UPLOADING +from .constants import CHUNKED_UPLOAD_CHOICES, UPLOADING, SUPPORTED_CHUCKSUM_ALGORITHMS AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') @@ -53,6 +54,81 @@ def md5(self): self._md5 = md5.hexdigest() return self._md5 + @property + def sha1(self): + print(self.__dict__) + print(123) + if getattr(self, '_sha1', None) is None: + sha1 = hashlib.sha1() + for chunk in self.file.chunks(): + sha1.update(chunk) + self._sha1 = sha1.hexdigest() + return self._sha1 + + @property + def sha224(self): + if getattr(self, '_sha224', None) is None: + sha224 = hashlib.sha224() + for chunk in self.file.chunks(): + sha224.update(chunk) + self._sha224 = sha224.hexdigest() + return self._sha224 + + @property + def sha256(self): + if getattr(self, '_sha256', None) is None: + sha256 = hashlib.sha256() + for chunk in self.file.chunks(): + sha256.update(chunk) + self._sha256 = sha256.hexdigest() + return self._sha256 + + @property + def sha384(self): + if getattr(self, '_sha384', None) is None: + sha384 = hashlib.sha384() + for chunk in self.file.chunks(): + sha384.update(chunk) + self._sha384 = sha384.hexdigest() + return self._sha384 + + @property + def sha512(self): + if getattr(self, '_sha512', None) is None: + sha512 = hashlib.sha512() + for chunk in self.file.chunks(): + sha512.update(chunk) + self._sha512 = sha512.hexdigest() + return self._sha512 + + @property + def sha512(self): + if getattr(self, '_sha512', None) is None: + sha512 = hashlib.sha512() + for chunk in self.file.chunks(): + sha512.update(chunk) + self._sha512 = sha512.hexdigest() + return self._sha512 + + @property + def adler32(self): + if getattr(self, "_adler32", None) is None: + adler32 = 0 + for chunk in self.file.chunks(): + adler32 = zlib.adler32(chunk, adler32) + self._adler32 = str(adler32) + return self._adler32 + + @property + def crc32(self): + if getattr(self, "_crc32", None) is None: + crc32 = 0 + for chunk in self.file.chunks(): + crc32 = zlib.crc32(chunk, crc32) + self._crc32 = str(crc32) + return self._crc32 + + def delete(self, delete_file=True, *args, **kwargs): storage, path = self.file.storage, self.file.path super(ChunkedUpload, self).delete(*args, **kwargs) @@ -85,7 +161,11 @@ def append_chunk(self, chunk, chunk_size=None, save=True): self.offset += chunk.size else: self.offset = self.file.size - self._md5 = None # Clear cached md5 + # clear cached checksum + for checksum_type in SUPPORTED_CHUCKSUM_ALGORITHMS: + checksum = getattr(self, "_" + checksum_type, None) + if checksum is not None: + checksum = None if save: self.save() self.close_file() # Flush diff --git a/chunked_upload/views.py b/chunked_upload/views.py index 6760f72..7e80799 100644 --- a/chunked_upload/views.py +++ b/chunked_upload/views.py @@ -8,7 +8,8 @@ from .settings import MAX_BYTES from .models import ChunkedUpload from .response import Response -from .constants import http_status, COMPLETE, FAILED +from .constants import (http_status, COMPLETE, FAILED, + SUPPORTED_CHUCKSUM_ALGORITHMS) from .exceptions import ChunkedUploadError @@ -227,9 +228,13 @@ class ChunkedUploadCompleteView(ChunkedUploadBaseView): define what to do when upload is complete. """ - # I wouldn't recommend to turn off the md5 check, unless is really + # I wouldn't recommend to turn off the checksum check, unless is really # impacting your performance. Proceed at your own risk. - do_md5_check = True + do_checksum_check = True + + # You can choose any checksum type in 'SUPPORTED_CHECKSUM_ALGORITHM'. + # And multiple checksums check is supoorted. + supported_checksums = SUPPORTED_CHUCKSUM_ALGORITHMS def on_completion(self, uploaded_file, request): """ @@ -245,25 +250,23 @@ def is_valid_chunked_upload(self, chunked_upload): return ChunkedUploadError(status=http_status.HTTP_400_BAD_REQUEST, detail=error_msg) - def md5_check(self, chunked_upload, md5): + def checksum_check(self, checksum_alg, chunked_upload, checksum): """ - Verify if md5 checksum sent by client matches generated md5. + Verify if checksum sent by client matches generated checksum. """ - if chunked_upload.md5 != md5: + if getattr(chunked_upload, checksum_alg, None) != checksum: chunked_upload.status = FAILED self._save(chunked_upload) raise ChunkedUploadError(status=http_status.HTTP_400_BAD_REQUEST, - detail='md5 checksum does not match') + detail='%s check does not match. ' + '%s expected.' % + (checksum_alg, getattr(chunked_upload, checksum_alg, None))) def _post(self, request, *args, **kwargs): upload_id = request.POST.get('upload_id') - md5 = request.POST.get('md5') error_msg = None - if self.do_md5_check: - if not upload_id or not md5: - error_msg = "Both 'upload_id' and 'md5' are required" - elif not upload_id: + if not upload_id: error_msg = "'upload_id' is required" if error_msg: raise ChunkedUploadError(status=http_status.HTTP_400_BAD_REQUEST, @@ -274,8 +277,17 @@ def _post(self, request, *args, **kwargs): self.validate(request) self.is_valid_chunked_upload(chunked_upload) - if self.do_md5_check: - self.md5_check(chunked_upload, md5) + + if self.do_checksum_check: + for checksum_alg in self.supported_checksums: + if checksum_alg in request.POST: + self.checksum_check(checksum_alg, chunked_upload, request.POST[checksum_alg]) + break + else: # Didn't find any checksum + raise ChunkedUploadError(status=http_status.HTTP_400_BAD_REQUEST, + detail="Didn't find any checksum, " + "checksum in %s is required." % + self.supported_checksums) chunked_upload.status = COMPLETE chunked_upload.completed_on = timezone.now() From 146b9a43ca3d7dce11d47eccd4840927da99bcc4 Mon Sep 17 00:00:00 2001 From: voidrank Date: Sun, 9 Aug 2015 15:10:14 +0800 Subject: [PATCH 2/2] fix some bugs. Thank @pmdarrow --- chunked_upload/constants.py | 2 +- chunked_upload/models.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/chunked_upload/constants.py b/chunked_upload/constants.py index 0fc17bc..5af44a0 100644 --- a/chunked_upload/constants.py +++ b/chunked_upload/constants.py @@ -17,7 +17,7 @@ class http_status: (FAILED, _('Failed')), ) -SUPPORTED_CHUCKSUM_ALGORITHMS = ( +SUPPORTED_CHECKSUM_ALGORITHMS = ( 'md5', 'sha1', 'sha224' diff --git a/chunked_upload/models.py b/chunked_upload/models.py index 28ac55a..2dd2633 100644 --- a/chunked_upload/models.py +++ b/chunked_upload/models.py @@ -56,8 +56,6 @@ def md5(self): @property def sha1(self): - print(self.__dict__) - print(123) if getattr(self, '_sha1', None) is None: sha1 = hashlib.sha1() for chunk in self.file.chunks():