From ac2dfaddbbb3d0d15f42852d35811e02b666bf1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zaoral?= Date: Tue, 22 Aug 2023 09:49:09 +0200 Subject: [PATCH 1/2] django: use BigIntegerField for file upload sizes Otherwise, we cannot log uploads that are larger than 2 GB. Resolves: https://github.com/release-engineering/kobo/issues/216 --- .../migrations/0002_alter_fileupload_size.py | 19 +++++++++++++++++++ kobo/django/upload/models.py | 7 ++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 kobo/django/upload/migrations/0002_alter_fileupload_size.py diff --git a/kobo/django/upload/migrations/0002_alter_fileupload_size.py b/kobo/django/upload/migrations/0002_alter_fileupload_size.py new file mode 100644 index 00000000..f5648eee --- /dev/null +++ b/kobo/django/upload/migrations/0002_alter_fileupload_size.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.20 on 2023-08-22 09:45 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('upload', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='fileupload', + name='size', + field=models.BigIntegerField(validators=[django.core.validators.MinValueValidator(0)]), + ), + ] diff --git a/kobo/django/upload/models.py b/kobo/django/upload/models.py index f5f23b2f..68d65c6f 100644 --- a/kobo/django/upload/models.py +++ b/kobo/django/upload/models.py @@ -4,6 +4,7 @@ import os from django.conf import settings +from django.core.validators import MinValueValidator from django.db import models from django.contrib.auth.models import User @@ -25,7 +26,8 @@ class FileUpload(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) name = models.CharField(max_length=255) checksum = models.CharField(max_length=255) - size = models.PositiveIntegerField() + # models.PositiveBigIntegerField would be even better but it was introduced only in Django 3.1 + size = models.BigIntegerField(validators=[MinValueValidator(0)]) target_dir = models.CharField(max_length=255) upload_key = models.CharField(max_length=255) state = models.PositiveIntegerField(default=0, choices=UPLOAD_STATES.get_mapping()) @@ -66,6 +68,9 @@ def save(self, *args, **kwargs): self.state = UPLOAD_STATES['FAILED'] if "update_fields" in kwargs: kwargs["update_fields"] = {"state"}.union(kwargs["update_fields"]) + + # execute validators + self.full_clean() super(FileUpload, self).save(*args, **kwargs) def delete(self): From f277aaeed206fda07920a3d1afa4fab908e4dd52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zaoral?= Date: Tue, 22 Aug 2023 09:57:08 +0200 Subject: [PATCH 2/2] client: allow file upload size to be sent a string ... for uploads which sizes exceed the xmlrpc.client.MAXINT limit for int values. This change should not be backwards incompatible. Old client with newer hub should work seamlessly. And new hub with older client would crash with the following traceback anyway. Fixes the following traceback: ``` $ truncate -s 2G test $ osh/client/osh-cli mock-build --tarball-build-script=test ./test Traceback (most recent call last): File "/src/osh/client/osh-cli", line 79, in main() File "/src/osh/client/osh-cli", line 72, in main parser.run() File "/src/kobo/kobo/cli.py", line 296, in run cmd.run(*cmd_args, **cmd_kwargs) File "/src/osh/client/commands/cmd_diff_build.py", line 157, in run target_dir, self.parser) File "/src/osh/client/commands/shortcuts.py", line 145, in upload_file return hub.upload_file(os.path.expanduser(srpm), target_dir) File "/src/kobo/kobo/client/__init__.py", line 473, in upload_file upload_id, upload_key = self.upload.register_upload(os.path.basename(file_name), checksum, fsize, target_dir) File "/usr/lib64/python3.6/xmlrpc/client.py", line 1112, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python3.6/xmlrpc/client.py", line 1446, in __request allow_none=self.__allow_none).encode(self.__encoding, 'xmlcharrefreplace') File "/usr/lib64/python3.6/xmlrpc/client.py", line 971, in dumps data = m.dumps(params) File "/usr/lib64/python3.6/xmlrpc/client.py", line 502, in dumps dump(v, write) File "/usr/lib64/python3.6/xmlrpc/client.py", line 524, in __dump f(self, value, write) File "/usr/lib64/python3.6/xmlrpc/client.py", line 540, in dump_long raise OverflowError("int exceeds XML-RPC limits") OverflowError: int exceeds XML-RPC limits ``` Resolves: https://github.com/release-engineering/kobo/issues/216 --- kobo/client/__init__.py | 4 ++++ kobo/django/upload/xmlrpc.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/kobo/client/__init__.py b/kobo/client/__init__.py index dce65778..9b47b4f1 100644 --- a/kobo/client/__init__.py +++ b/kobo/client/__init__.py @@ -470,6 +470,10 @@ def upload_file(self, file_name, target_dir): checksum = sum.hexdigest().lower() fsize = os.path.getsize(file_name) + # use str only for large uploads to not break compatibility with older hubs + if fsize > xmlrpclib.MAXINT: + fsize = str(fsize) + upload_id, upload_key = self.upload.register_upload(os.path.basename(file_name), checksum, fsize, target_dir) secure = (scheme == "https") diff --git a/kobo/django/upload/xmlrpc.py b/kobo/django/upload/xmlrpc.py index 39a55e07..fdf54a4d 100644 --- a/kobo/django/upload/xmlrpc.py +++ b/kobo/django/upload/xmlrpc.py @@ -27,7 +27,8 @@ def register_upload(request, name, checksum, size, target_dir): upload.owner = request.user upload.name = name upload.checksum = checksum.lower() - upload.size = size + # size may be sent as a string to workaround the xmlrpc.client.MAXINT limit + upload.size = int(size) upload.target_dir = target_dir upload.save() return (upload.id, upload.upload_key)