Merge pull request #33 from matchish/import-speed

Improve import speed

.scrutinizer.yml

@@ -11,18 +11,25 @@ build:
   environment:
     php:
       version: '7.2'
+    variables:
+      ELASTICSEARCH_HOST: 'localhost:9200'
+      DB_HOST: '127.0.0.1'
+      DB_USERNAME: 'root'
+      DB_PASSWORD: ''
+      DB_DATABASE: 'my_database'
+  project_setup:
+    before:
+      - mysql -e "CREATE DATABASE my_database"
   nodes:
     analysis:
-      environment:
-        variables:
-          ELASTICSEARCH_HOST: 'localhost:9200'
       services:
         elasticsearch: 6.6.2
+        mysql: 5.7
       tests:
         override:
           - php-scrutinizer-run
           -
             command: 'vendor/bin/phpunit --coverage-clover=coverage.clover'
             coverage:
               file: 'coverage.clover'
-              format: 'clover'
+              format: 'clover'

.travis.yml

@@ -1,7 +1,10 @@
 language: php
 
+services:
+  - mysql
+
 env:
-  - ELASTICSEARCH_HOST=localhost:9200
+  - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD=""
 
 php:
   - 7.1
@@ -18,6 +21,7 @@ matrix:
     - php: nightly
 
 before_script:
+  - mysql -e "create database $DB_DATABASE;"
   - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.2.deb && sudo dpkg -i --force-confnew elasticsearch-6.6.2.deb && sudo service elasticsearch restart
   - sleep 30
   - curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=50s"

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [Unreleased]
 
+## [2.1.0] - 2019-11-13
+### Added
+- Import source factory
+- Using global scopes only for import
+
 ## [2.0.4] - 2019-11-10
 ### Fixed
 - Throw more descriptive exception if there are elasticsearch errors on update

Makefile

@@ -21,6 +21,7 @@ help: ## Show this help
 ---------------: ## ---------------
 
 up: ## Start all containers (in background) for development
+	sudo sysctl -w vm.max_map_count=262144
 	$(docker_compose_bin) up --no-recreate -d
 
 down: ## Stop all started for development containers

README.md

@@ -29,6 +29,7 @@ If you need any help, [stack overflow](https://stackoverflow.com/questions/tagge
 
 - [Search amongst multiple models](#search-amongst-multiple-models)
 - [**Zero downtime** reimport](#zero-downtime-reimport) - it’s a breeze to import data in production.
+- [Eager load relations](#eager-load) - speed up your import.
 - Elasticsearch **7.0** ready - Use [elasticsearch-7](https://github.com/matchish/laravel-scout-elasticsearch/tree/elasticsearch-7) branch instead.
 - Import all searchable models at once.
 - A fully configurable mapping for each model.
@@ -95,6 +96,45 @@ For index `products` it will be
 And for default settings  
 `elasticsearch.indices.settings.default`
 
+### Eager load
+To speed up import you can eager load relations on import using global scopes.
+
+You should configure `ImportSourceFactory` in your service provider(`register` method)
+```
+use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory;
+...
+public function register(): void
+{
+$this->app->bind(ImportSourceFactory::class, MyImportSourceFactory::class);
+``` 
+Here is an example of `MyImportSourceFactory`
+```
+namespace Matchish\ScoutElasticSearch\Searchable;
+
+final class MyImportSourceFactory implements ImportSourceFactory
+{
+    public static function from(string $className): ImportSource
+    {
+        //Add all required scopes
+        return new DefaultImportSource($className, [new WithCommentsScope()]);
+    }
+}
+
+class WithCommentsScope implements Scope {
+
+    /**
+     * Apply the scope to a given Eloquent query builder.
+     *
+     * @param \Illuminate\Database\Eloquent\Builder $builder
+     * @param \Illuminate\Database\Eloquent\Model $model
+     * @return void
+     */
+    public function apply(Builder $builder, Model $model)
+    {
+        $builder->with('comments');
+    }
+}
+```
 ### Zero downtime reimport
 While working in production, to keep your existing search experience available while reimporting your data, you also can use `scout:import` Artisan command:  
 

composer.json

@@ -1,7 +1,7 @@
 {
     "name": "matchish/laravel-scout-elasticsearch",
     "description": "This package extends Laravel Scout adding full power of ElasticSearch",
-    "version": "2.0.4",
+    "version": "2.1.0",
     "keywords": [
         "laravel",
         "scout",

docker-compose.yml

@@ -20,6 +20,19 @@ services:
       XDEBUG_CONFIG: ${XDEBUG_CONFIG}
       PHP_IDE_CONFIG: ${PHP_IDE_CONFIG}
       ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST}
+      DB_HOST: ${DB_HOST}
+      DB_DATABASE: ${DB_DATABASE}
+      DB_USERNAME: ${DB_USERNAME}
+      DB_PASSWORD: ${DB_PASSWORD}
+    networks:
+      - default
+  db:
+    image: mysql:5.7.22
+    environment:
+      MYSQL_DATABASE: ${DB_DATABASE}
+      MYSQL_USER: ${DB_USERNAME}
+      MYSQL_PASSWORD: ${DB_PASSWORD}
+      MYSQL_ROOT_PASSWORD: ${DB_PASSWORD}
     networks:
       - default
   elasticsearch:

src/Console/Commands/ImportCommand.php

@@ -7,6 +7,8 @@
 use Illuminate\Console\Command;
 use Matchish\ScoutElasticSearch\Jobs\Import;
 use Matchish\ScoutElasticSearch\Jobs\QueueableJob;
+use Matchish\ScoutElasticSearch\Searchable\ImportSource;
+use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory;
 use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory;
 
 final class ImportCommand extends Command
@@ -42,7 +44,9 @@ private function searchableList($argument)
 
     private function import($searchable)
     {
-        $job = new Import($searchable);
+        $sourceFactory = app(ImportSourceFactory::class);
+        $source = $sourceFactory::from($searchable);
+        $job = new Import($source);
 
         if (config('scout.queue')) {
             $job = (new QueueableJob())->chain([$job]);
@@ -54,8 +58,9 @@ private function import($searchable)
         $startMessage = trans('scout::import.start', ['searchable' => "<comment>$searchable</comment>"]);
         $this->line($startMessage);
 
-        dispatch($job)->allOnQueue((new $searchable)->syncWithSearchUsingQueue())
-            ->allOnConnection(config((new $searchable)->syncWithSearchUsing()));
+        /* @var ImportSource $source */
+        dispatch($job)->allOnQueue($source->syncWithSearchUsingQueue())
+            ->allOnConnection($source->syncWithSearchUsing());
 
         $doneMessage = trans(config('scout.queue') ? 'scout::import.done.queue' : 'scout::import.done', [
             'searchable' => $searchable,

src/Database/Scopes/ChunkScope.php

@@ -0,0 +1,55 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Database\Scopes;
+
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Scope;
+
+class ChunkScope implements Scope
+{
+    /**
+     * @var mixed
+     */
+    private $start;
+    /**
+     * @var mixed
+     */
+    private $end;
+
+    /**
+     * ChunkScope constructor.
+     * @param mixed $start
+     * @param mixed $end
+     */
+    public function __construct($start, $end)
+    {
+        $this->start = $start;
+        $this->end = $end;
+    }
+
+    /**
+     * Apply the scope to a given Eloquent query builder.
+     *
+     * @param  \Illuminate\Database\Eloquent\Builder  $builder
+     * @param  \Illuminate\Database\Eloquent\Model  $model
+     * @return void
+     */
+    public function apply(Builder $builder, Model $model)
+    {
+        $start = $this->start;
+        $end = $this->end;
+        $builder
+            ->when(! is_null($start), function ($query) use ($start, $model) {
+                return $query->where($model->getKeyName(), '>', $start);
+            })
+            ->when(! is_null($end), function ($query) use ($end, $model) {
+                return $query->where($model->getKeyName(), '<=', $end);
+            });
+    }
+
+    public function key()
+    {
+        return static::class;
+    }
+}

src/Database/Scopes/PageScope.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Database\Scopes;
+
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Scope;
+
+class PageScope implements Scope
+{
+    /**
+     * @var int
+     */
+    private $page;
+    /**
+     * @var int
+     */
+    private $perPage;
+
+    /**
+     * PageScope constructor.
+     * @param int $page
+     * @param int $perPage
+     */
+    public function __construct(int $page, int $perPage)
+    {
+        $this->page = $page;
+        $this->perPage = $perPage;
+    }
+
+    /**
+     * Apply the scope to a given Eloquent query builder.
+     *
+     * @param \Illuminate\Database\Eloquent\Builder $builder
+     * @param \Illuminate\Database\Eloquent\Model $model
+     * @return void
+     */
+    public function apply(Builder $builder, Model $model)
+    {
+        $builder->forPage($this->page, $this->perPage);
+    }
+}

src/ElasticSearch/EloquentHitsIteratorAggregate.php

@@ -1,21 +1,16 @@
 <?php
-/**
- * Created by PhpStorm.
- * User: matchish
- * Date: 20.03.19
- * Time: 12:39.
- */
 
 namespace Matchish\ScoutElasticSearch\ElasticSearch;
 
+use IteratorAggregate;
 use Laravel\Scout\Builder;
 use Laravel\Scout\Searchable;
 use Traversable;
 
 /**
  * @internal
  */
-final class EloquentHitsIteratorAggregate implements \IteratorAggregate
+final class EloquentHitsIteratorAggregate implements IteratorAggregate
 {
     /**
      * @var array

src/ElasticSearch/Index.php

@@ -2,6 +2,8 @@
 
 namespace Matchish\ScoutElasticSearch\ElasticSearch;
 
+use Matchish\ScoutElasticSearch\Searchable\ImportSource;
+
 /**
  * @internal
  */
@@ -81,11 +83,11 @@ public function config(): array
         return $config;
     }
 
-    public static function fromSearchable($searchable): Index
+    public static function fromSource(ImportSource $source): Index
     {
-        $name = $searchable->searchableAs().'_'.time();
-        $settingsConfigKey = "elasticsearch.indices.settings.{$searchable->searchableAs()}";
-        $mappingsConfigKey = "elasticsearch.indices.mappings.{$searchable->searchableAs()}";
+        $name = $source->searchableAs().'_'.time();
+        $settingsConfigKey = "elasticsearch.indices.settings.{$source->searchableAs()}";
+        $mappingsConfigKey = "elasticsearch.indices.mappings.{$source->searchableAs()}";
         $defaultSettings = [
             'number_of_shards' => 1,
             'number_of_replicas' => 0,

src/Jobs/Import.php

@@ -6,6 +6,7 @@
 use Illuminate\Bus\Queueable;
 use Illuminate\Support\Collection;
 use Matchish\ScoutElasticSearch\ProgressReportable;
+use Matchish\ScoutElasticSearch\Searchable\ImportSource;
 
 /**
  * @internal
@@ -16,16 +17,16 @@ final class Import
     use ProgressReportable;
 
     /**
-     * @var string
+     * @var ImportSource
      */
-    private $searchable;
+    private $source;
 
     /**
-     * @param string $searchable
+     * @param ImportSource $source
      */
-    public function __construct(string $searchable)
+    public function __construct(ImportSource $source)
     {
-        $this->searchable = $searchable;
+        $this->source = $source;
     }
 
     /**
@@ -45,6 +46,6 @@ public function handle(Client $elasticsearch): void
 
     private function stages(): Collection
     {
-        return ImportStages::fromSearchable(new $this->searchable);
+        return ImportStages::fromSource($this->source);
     }
 }