[6.x] Improve search indexing performance (#13126)

Author: duncanmcclean

config/search.php

@@ -79,4 +79,31 @@
         'fields' => ['title'],
     ],
 
+    /*
+    |--------------------------------------------------------------------------
+    | Indexing Queue
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure the queue name and connection used when indexing
+    | documents.
+    |
+    */
+
+    'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+    'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+    /*
+    |--------------------------------------------------------------------------
+    | Chunk Size
+    |--------------------------------------------------------------------------
+    |
+    | Here you can configure the chunk size used when indexing documents.
+    | The higher you make it, the more memory it will use, but the quicker
+    | the indexing process will be.
+    |
+    */
+
+    'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
         return (new Query($this))->query($query);
     }
 
-    protected function insertDocuments(Documents $documents)
+    public function insertDocuments(Documents $documents)
     {
         $documents = $documents->map(function ($item, $id) {
             $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
         $this->save($data);
     }
 
-    protected function insertDocuments(Documents $documents)
+    public function insertDocuments(Documents $documents)
     {
         try {
             $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
     abstract public function exists();
 
-    abstract protected function insertDocuments(Documents $documents);
+    abstract public function insertDocuments(Documents $documents);
 
     abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
     public function insert($document)
     {
-        return $this->insertMultiple(Arr::wrap($document));
+        return $this->insertMultiple(collect($document));
     }
 
     public function insertMultiple($documents)
     {
-        $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
-            return [$item->getSearchReference() => $this->searchables()->fields($item)];
-        });
-
-        $this->insertDocuments($documents);
+        $documents
+            ->chunk(config('statamic.search.chunk_size'))
+            ->each(fn ($documents) => InsertMultipleJob::dispatch(
+                name: Str::beforeLast($this->name, '_'),
+                locale: $this->locale,
+                documents: $documents
+            ));
 
         return $this;
     }
 
+    public function fields(Searchable $searchable)
+    {
+        return $this->searchables()->fields($searchable);
+    }
+
     public function shouldIndex($searchable)
     {
         return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+    use Queueable;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(
+        public string $name,
+        public ?string $locale,
+        public Collection|LazyCollection $documents,
+    ) {
+        $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+        $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $providers = app(Providers::class);
+        $index = Search::index($this->name, $this->locale);
+
+        $documents = $this->documents
+            ->groupBy(fn ($document) => explode('::', $document)[0])
+            ->flatMap(function ($documents, $prefix) use ($providers) {
+                return $providers
+                    ->getByPrefix($prefix)
+                    ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+                    ->all();
+            })
+            ->mapWithKeys(function (Searchable $item) use ($index) {
+                return [$item->getSearchReference() => $index->fields($item)];
+            });
+
+        $index->insertDocuments(new Documents($documents));
+    }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
         return true;
     }
 
-    protected function insertDocuments(Documents $documents)
+    public function insertDocuments(Documents $documents)
     {
         //
     }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
             $exists = $index->exists();
 
             if ($shouldIndex && $exists) {
-                $index->insert($searchable);
+                $index->insert($searchable->getSearchReference());
             } elseif ($shouldIndex && ! $exists) {
                 $index->update();
             } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
             : AssetCollection::make($this->keys)
                 ->flatMap(fn ($key) => Asset::whereContainer($key));
 
-        return $assets->filter($this->filter())->values();
+        // TODO: query scope support?
+
+        return $assets->filter($this->filter())->values()->map->reference();
     }
 
     public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
             $query->where('site', $site);
         }
 
-        return $query->lazy(100)->filter($this->filter())->values();
+        $this->applyQueryScope($query);
+
+        if ($this->hasFilter()) {
+            return $query
+                ->lazy(config('statamic.search.chunk_size'))
+                ->filter($this->filter())
+                ->values()
+                ->map->reference();
+        }
+
+        $query->where('status', 'published');
+
+        return $query->pluck('reference');
     }
 
     public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
             return false;
         }
 
-        return $this->filter()($searchable);
+        if ($this->hasFilter()) {
+            return $this->filter()($searchable);
+        }
+
+        $query = Entry::query()
+            ->where('status', 'published')
+            ->where('id', $searchable->id());
+
+        $this->applyQueryScope($query);
+
+        return $query->exists();
     }
 
     public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
         return in_array('*', $this->keys);
     }
 
+    protected function applyQueryScope($query)
+    {
+        if (! $scope = $this->index->config()['query_scope'] ?? null) {
+            return;
+        }
+
+        Scope::find($scope)->apply($query, []);
+    }
+
+    protected function hasFilter()
+    {
+        return Arr::has($this->index->config(), 'filter');
+    }
+
     protected function filter()
     {
         $filter = $this->index->config()['filter'] ?? null;