Merge pull request #278 from DavisPuciriuss/parallel-import-fixes

PullFromSourceParallel small bug fixes

CHANGELOG.md

@@ -6,9 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [Unreleased]
 
-## [8.0.0-alpha.2] - 2024-
+## [8.0.0-alpha.2] - 2024-06-20
 ### Added
 - ElasticParams trait, that adds 'getElasticsearchScore' and 'getElasticsearchHighlight' to the model after performing a search.
+- 'elasticsearch.queue.name' config parameter to set a custom parallel import queue name.
 
 ## [8.0.0-alpha.1] - 2024-05-29
 ### Added

README.md

@@ -1,6 +1,10 @@
 <p align="center">
-  <a href="https://savelife.in.ua/en/donate/">
-    <img alt="Support Ukraine" src="http://supportua.org.ua/wp-content/uploads/2015/05/content-logo-main.png" >
+    <h1>Stay with Ukraine</h1>
+    <p><a href="https://u24.gov.ua/robots_fight">UNITED24</a> launches the first fundraiser towards terrestrial robotic platforms. Squads of robots will save the lives of our military and civilians. They will become logistics devices, tow trucks, minelayers and deminers, as well as self-destructive robots. They will fight alongside people and for people.
+
+The first robots are already proving their effectiveness on the battlefield. There will be more of them soon. Many more.</p>
+  <a href="https://u24.gov.ua/robots_fight">
+    <img alt="Support Ukraine" src="https://files.u24.gov.ua/pages/robotsFight/_processed/robots-og-en.jpg" >
   </a>
 <!--   <a href="https://github.com/matchish/laravel-scout-elasticsearch">
     <img alt="Scout ElasticSearch" src="https://raw.githubusercontent.com/matchish/laravel-scout-elasticsearch/master/docs/banner.svg?sanitize=true" >
@@ -37,10 +41,10 @@ Don't forget to :star: the package if you like it. :pray:
 - [Search amongst multiple models](#search-amongst-multiple-models)
 - [**Zero downtime** reimport](#zero-downtime-reimport) - it’s a breeze to import data in production.
 - [Eager load relations](#eager-load) - speed up your import.
+- Parallel import to make your import as fast as possible (in [alpha version](https://github.com/matchish/laravel-scout-elasticsearch/releases/tag/8.0.0-alpha.1) for now)
 - Import all searchable models at once.
 - A fully configurable mapping for each model.
 - Full power of ElasticSearch in your queries.
-
 ## :warning: Requirements
 
 - PHP version >= 8.0

config/elasticsearch.php

@@ -8,6 +8,7 @@
     'api_key' => env('ELASTICSEARCH_API_KEY'),
     'queue' => [
         'timeout' => env('SCOUT_QUEUE_TIMEOUT'),
+        'name' => env('SCOUT_QUEUE_NAME'),
     ],
     'indices' => [
         'mappings' => [

src/Jobs/Stages/PullFromSourceParallel.php

@@ -19,6 +19,11 @@ final class PullFromSourceParallel implements StageInterface
      */
     const DEFAULT_HANDLER_COUNT = 1;
 
+    /**
+     * @var string
+     */
+    const DEFAULT_QUEUE_NAME = 'elasticsearch-parallel';
+
     /**
      * @var ImportSource
      */
@@ -42,19 +47,19 @@ final class PullFromSourceParallel implements StageInterface
     /**
      * @var array<string>
      */
-    private $queues = [
-        'import_1',
-    ];
+    private $queues = [];
 
     /**
      * @param  ImportSource  $source
      */
     public function __construct(ImportSource $source)
     {
         $this->source = $source;
-        $this->queues = collect(config('scout.chunk.handlers', self::DEFAULT_HANDLER_COUNT))->map(function ($i) {
-            return 'import_'.$i;
-        })->toArray();
+        $this->queues = [];
+
+        foreach (range(1, config('scout.chunk.handlers', self::DEFAULT_HANDLER_COUNT)) as $i) {
+            $this->queues[] = config('elasticsearch.queue.name', self::DEFAULT_QUEUE_NAME).'-'.$i;
+        }
     }
 
     /**
@@ -95,7 +100,7 @@ public function handle(Client $elasticsearch = null): void
             })->pluck('id')->toArray();
         }
 
-        if (count($this->dispatchedJobIds) >= $this->source->getTotalChunks()) {
+        if (count($this->handledJobs) + count($this->dispatchedJobIds) > $this->source->getTotalChunks()) {
             return;
         }