feat(medusa): Move search indexing into a separate subscriber to defer the work load (#1874)

**What** Move the preliminary indexing action at boot time to a separate subscriber in order to defer the work load in the background and therefore to avoid increasing the load time when the number of products increase with time. **Tests** Add 10k products (since it is our limit, tried with 50k before getting the error limit) using ```sal do $$ declare counter integer := 0; begin while counter < 10000 loop INSERT INTO product (id, title, description, handle, profile_id) (SELECT * FROM ((SELECT random(), random(), random(), random(), 'sp_01FNB9K7FXB0SZMKXD013RJYSP')) as T); counter := counter + 1; end loop; end$$; ``` then start the server and while the server is starting, hit the search end point repeatedly FIXES CORE-258 Co-authored-by: Oliver Windall Juhl <59018053+olivermrbl@users.noreply.github.com>
2022-07-26 12:58:28 +00:00
parent 0e0b131488
commit b8ddb31f6f
6 changed files with 122 additions and 61 deletions
@@ -0,0 +1,94 @@
+import EventBusService from "../services/event-bus"
+import { SEARCH_INDEX_EVENT } from "../loaders/search-index"
+import ProductService from "../services/product"
+import { indexTypes } from "medusa-core-utils"
+import { Product } from "../models"
+import { SearchService } from "../services"
+
+type InjectedDependencies = {
+  eventBusService: EventBusService
+  searchService: SearchService
+  productService: ProductService
+}
+
+class SearchIndexingSubscriber {
+  private readonly eventBusService_: EventBusService
+  private readonly searchService_: SearchService
+  private readonly productService_: ProductService
+
+  constructor({
+    eventBusService,
+    searchService,
+    productService,
+  }: InjectedDependencies) {
+    this.eventBusService_ = eventBusService
+    this.searchService_ = searchService
+    this.productService_ = productService
+
+    this.eventBusService_.subscribe(SEARCH_INDEX_EVENT, this.indexDocuments)
+  }
+
+  indexDocuments = async (): Promise<void> => {
+    const TAKE = this.searchService_?.options?.batch_size ?? 1000
+    let hasMore = true
+
+    let lastSeenId = ""
+
+    while (hasMore) {
+      const products = await this.retrieveNextProducts(lastSeenId, TAKE)
+
+      if (products.length > 0) {
+        await this.searchService_.addDocuments(
+          ProductService.IndexName,
+          products,
+          indexTypes.products
+        )
+        lastSeenId = products[products.length - 1].id
+      } else {
+        hasMore = false
+      }
+    }
+  }
+
+  protected async retrieveNextProducts(
+    lastSeenId: string,
+    take: number
+  ): Promise<Product[]> {
+    return await this.productService_.list(
+      { id: { gt: lastSeenId } },
+      {
+        select: [
+          "id",
+          "title",
+          "status",
+          "subtitle",
+          "description",
+          "handle",
+          "is_giftcard",
+          "discountable",
+          "thumbnail",
+          "profile_id",
+          "collection_id",
+          "type_id",
+          "origin_country",
+          "created_at",
+          "updated_at",
+        ],
+        relations: [
+          "variants",
+          "tags",
+          "type",
+          "collection",
+          "variants.prices",
+          "images",
+          "variants.options",
+          "options",
+        ],
+        take: take,
+        order: { id: "ASC" },
+      }
+    )
+  }
+}
+
+export default SearchIndexingSubscriber