Add batching to import and sync queue jobs

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Release Notes for Campaign
 
+## 2.13.0 - Unreleased
+
+### Added
+
+- Added batching to import queue jobs.
+- Added batching to sync queue jobs.
+
+### Changed
+
+- Campaign now requires Craft CMS 4.4.0 or later.
+
+### Deprecated
+
+- Deprecated the `memoryLimit` config setting.
+- Deprecated the `memoryThreshold` config setting.
+- Deprecated the `timeLimit` config setting.
+- Deprecated the `timeThreshold` config setting.
+
 ## 2.12.2 - 2024-03-05
 
 ### Changed

composer.json

@@ -1,7 +1,7 @@
 {
   "name": "putyourlightson/craft-campaign",
   "description": "Send and manage email campaigns, contacts and mailing lists.",
-  "version": "2.12.2",
+  "version": "2.13.0",
   "type": "craft-plugin",
   "homepage": "https://putyourlightson.com/plugins/campaign",
   "license": "proprietary",
@@ -19,7 +19,7 @@
     "php": "^8.0.2",
     "ext-dom": "*",
     "aws/aws-php-sns-message-validator": "^1.5",
-    "craftcms/cms": "^4.0",
+    "craftcms/cms": "^4.4.0",
     "elvanto/litemoji": "^3.0.1|^4.0",
     "html2text/html2text": "^4.3.1",
     "matomo/device-detector": "^3.9.1|^4.0|^5.0|^6.0",

src/batchers/PendingRecipientBatcher.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace putyourlightson\campaign\batchers;
+
+use craft\base\Batchable;
+
+/**
+ * @since 2.13.0
+ */
+class PendingRecipientBatcher implements Batchable
+{
+    public function __construct(
+        private array $pendingRecipients,
+        private ?int $limit = null,
+    ) {
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function count(): int
+    {
+        return min(count($this->pendingRecipients), $this->limit ?? 0);
+    }
+
+    /**
+     * Returns a batch of pending recipients, using the limit and ignoring the offset, since the pending recipients array is calculated fresh each time.
+     */
+    public function getSlice(int $offset, int $limit): iterable
+    {
+        return array_slice($this->pendingRecipients, 0, $limit);
+    }
+}

src/batchers/RowBatcher.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace putyourlightson\campaign\batchers;
+
+use craft\base\Batchable;
+
+/**
+ * @since 2.13.0
+ */
+class RowBatcher implements Batchable
+{
+    public function __construct(
+        private array $rows,
+    ) {
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function count(): int
+    {
+        return count($this->rows);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function getSlice(int $offset, int $limit): iterable
+    {
+        return array_slice($this->rows, $offset, $limit);
+    }
+}

src/jobs/ImportJob.php

@@ -6,34 +6,36 @@
 namespace putyourlightson\campaign\jobs;
 
 use Craft;
-use craft\queue\BaseJob;
+use craft\queue\BaseBatchedJob;
 use DateTime;
+use putyourlightson\campaign\batchers\RowBatcher;
 use putyourlightson\campaign\Campaign;
 use putyourlightson\campaign\events\ImportEvent;
+use putyourlightson\campaign\models\ImportModel;
 use putyourlightson\campaign\services\ImportsService;
 
-/**
- * @property-read int $ttr
- */
-class ImportJob extends BaseJob
+class ImportJob extends BaseBatchedJob
 {
     /**
      * @var int
      */
     public int $importId;
 
+    /**
+     * @var ImportModel|null
+     */
+    private ?ImportModel $_import = null;
+
     /**
      * @inheritdoc
      */
     public function execute($queue): void
     {
-        $import = Campaign::$plugin->imports->getImportById($this->importId);
-
+        $import = $this->_getImport();
         if ($import === null) {
             return;
         }
 
-        // Fire a before event
         $event = new ImportEvent([
             'import' => $import,
         ]);
@@ -46,38 +48,61 @@ public function execute($queue): void
         // Call for max power
         Campaign::$plugin->maxPowerLieutenant();
 
-        // Get rows
-        $rows = Campaign::$plugin->imports->getRows($import);
-        $total = count($rows);
+        parent::execute($queue);
 
-        // Loop as long as the there are lines
-        foreach ($rows as $i => $row) {
-            // Set progress
-            $this->setProgress($queue, $i / $total);
+        // TODO: move what’s below this into the `BaseBatchedJob::after` method in Campaign 3.
 
-            // Import row
-            $import = Campaign::$plugin->imports->importRow($import, $row, $i + 1);
+        if ($this->itemOffset < $this->totalItems()) {
+            return;
         }
 
         $import->dateImported = new DateTime();
 
-        // Save import
         Campaign::$plugin->imports->saveImport($import);
-
-        // Update the search indexes
         Campaign::$plugin->imports->updateSearchIndexes();
 
-        // Fire an after event
         if (Campaign::$plugin->imports->hasEventHandlers(ImportsService::EVENT_AFTER_IMPORT)) {
             Campaign::$plugin->imports->trigger(ImportsService::EVENT_AFTER_IMPORT, $event);
         }
     }
 
+    /**
+     * @inheritdoc
+     */
+    protected function loadData(): RowBatcher
+    {
+        $import = $this->_getImport();
+        if ($import === null) {
+            return new RowBatcher([]);
+        }
+
+        $rows = Campaign::$plugin->imports->getRows($import);
+
+        return new RowBatcher($rows);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    protected function processItem(mixed $item): void
+    {
+        Campaign::$plugin->imports->importRow($this->_getImport(), $item);
+    }
+
     /**
      * @inheritdoc
      */
     protected function defaultDescription(): string
     {
         return Craft::t('campaign', 'Importing contacts.');
     }
+
+    private function _getImport(): ?ImportModel
+    {
+        if ($this->_import === null) {
+            $this->_import = Campaign::$plugin->imports->getImportById($this->importId);
+        }
+
+        return $this->_import;
+    }
 }