Merge pull request #107 from rinse-inc/feat/customize-global-soft-tim…

…e-limit

Add configurable soft and hard time limits for import and export tasks

README.rst

@@ -58,7 +58,7 @@ By default a dry run of the import is initiated when the import object is create
         IMPORT_DRY_RUN_FIRST_TIME = False
 
 
-Preforming an import
+Performing an import
 --------------------
 
 You will find an example django application that uses django-import-export-celery for importing data. There are instructions for running the example application in the example directory's README file. Once you have it running, you can perform an import with the following steps.
@@ -178,6 +178,21 @@ Define a custom storage backend by adding the `IMPORT_EXPORT_CELERY_STORAGE` to
 
         IMPORT_EXPORT_CELERY_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 
+Customizing Task Time Limits
+----------------------------
+
+By default, there is no time limit on celery import/export tasks. This can be customized by setting the following variables in your Django settings file.
+
+    ::
+
+        # set import time limits (in seconds)
+        IMPORT_EXPORT_CELERY_IMPORT_SOFT_TIME_LIMIT = 300  # 5 minutes
+        IMPORT_EXPORT_CELERY_IMPORT_HARD_TIME_LIMIT = 360  # 6 minutes
+
+        # set export time limits (in seconds)
+        IMPORT_EXPORT_CELERY_EXPORT_SOFT_TIME_LIMIT = 300  # 5 minutes
+        IMPORT_EXPORT_CELERY_EXPORT_HARD_TIME_LIMIT = 360  # 6 minutes
+
 Customizing email template for export job completion email
 ----------------------------------------------------------
 

example/project/settings.py

@@ -136,3 +136,11 @@
 
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 IMPORT_EXPORT_CELERY_STORAGE = "django.core.files.storage.FileSystemStorage"
+
+# Default import time limits (in seconds)
+IMPORT_EXPORT_CELERY_IMPORT_SOFT_TIME_LIMIT = 300  # 5 minutes
+IMPORT_EXPORT_CELERY_IMPORT_HARD_TIME_LIMIT = 360  # 6 minutes
+
+# Default export time limits (in seconds)
+IMPORT_EXPORT_CELERY_EXPORT_SOFT_TIME_LIMIT = 300  # 5 minutes
+IMPORT_EXPORT_CELERY_EXPORT_HARD_TIME_LIMIT = 360  # 6 minutes

import_export_celery/tasks.py

@@ -188,7 +188,11 @@ def before_import_row(self, row, **kwargs):
     import_job.save()
 
 
-@shared_task(bind=False)
+@shared_task(
+    bind=False,
+    soft_time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_IMPORT_SOFT_TIME_LIMIT", 0),
+    time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_IMPORT_HARD_TIME_LIMIT", 0),
+)
 def run_import_job(pk, dry_run=True):
     log.info(f"Importing {pk} dry-run {dry_run}")
     import_job = models.ImportJob.objects.get(pk=pk)
@@ -201,7 +205,11 @@ def run_import_job(pk, dry_run=True):
         return
 
 
-@shared_task(bind=False)
+@shared_task(
+    bind=False,
+    soft_time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_EXPORT_SOFT_TIME_LIMIT", 0),
+    time_limit=getattr(settings, "IMPORT_EXPORT_CELERY_EXPORT_HARD_TIME_LIMIT", 0),
+)
 def run_export_job(pk):
     log.info("Exporting %s" % pk)
     export_job = models.ExportJob.objects.get(pk=pk)