chore(): Emit events in batch and index process event ids in batch (#12097)

**What**
First iteration to prevent events from overwhelming the systems.
- Group emitted event ids when possible instead of creating a message per id which leads to reduced amount of events to process massively in cases of import for example
- Update the index engine to process event data in batches of 100
- Update event handling by the index engine to be able to upsert by batch as well
- Fix index engine build config for intermediate listeners inferrence
This commit is contained in:
Adrien de Peretti authored and GitHub committed 2025-04-08 16:57:08 +00:00
1 parent b05807bfc1
commit 74381addc3
21 files changed
+548 -463

No files matched your search

@@ -141,6 +141,7 @@ const beforeAll_ = async () => {
;(index as any).eventBusModuleService_ = eventBusMock
await globalApp.onApplicationStart()
await setTimeout(3000)
;(index as any).storageProvider_.query_ = queryMock
return globalApp
@@ -196,6 +197,8 @@ describe("IndexModuleService", function () {
await dbUtils.shutdown(dbName)
})
afterEach(afterEach_)
describe("on created or attached events", function () {
let manager
@@ -222,7 +225,7 @@ describe("IndexModuleService", function () {
},
},
{
name: "PriceSet.created",
name: "pricing.price-set.created",
data: {
id: priceSetId,
},
@@ -247,7 +250,6 @@ describe("IndexModuleService", function () {
]
beforeEach(async () => {
await setTimeout(1000)
await beforeEach_(eventDataToEmit)
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
@@ -392,7 +394,7 @@ describe("IndexModuleService", function () {
},
},
{
name: "PriceSet.created",
name: "pricing.price-set.created",
data: {
id: priceSetId,
},
@@ -689,7 +691,7 @@ describe("IndexModuleService", function () {
},
},
{
name: "PriceSet.created",
name: "pricing.price-set.created",
data: {
id: priceSetId,
},
@@ -761,8 +763,6 @@ describe("IndexModuleService", function () {
await eventBusMock.emit(deleteEventDataToEmit)
})
afterEach(afterEach_)
it("should consume all deleted events and delete the index entries", async () => {
const indexEntries = await manager.find(toMikroORMEntity(IndexData), {})
const indexRelationEntries = await manager.find(