asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(): Emit events in batch and index process event ids in batch (#12097)

Browse Source 
**What**
First iteration to prevent events from overwhelming the systems.
- Group emitted event ids when possible instead of creating a message per id which leads to reduced amount of events to process massively in cases of import for example
- Update the index engine to process event data in batches of 100
- Update event handling by the index engine to be able to upsert by batch as well
- Fix index engine build config for intermediate listeners inferrence
This commit is contained in:
Adrien de Peretti
2025-04-08 16:57:08 +00:00
committed by GitHub
parent b05807bfc1
commit 74381addc3
21 changed files with 548 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/modules/fulfillment/integration-tests/__tests__/fulfillment-module-service/fulfillment-set.spec.ts
+56 -20
View File
@@ -218,7 +218,9 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_CREATED,
action: "created",
object: "fulfillment_set",
data: { id: fulfillmentSets[i].id },
data: {
id: expect.arrayContaining([fulfillmentSets[i].id]),
},
}),
]),
{
@@ -335,13 +337,19 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_CREATED,
action: "created",
object: "fulfillment_set",
data: { id: fulfillmentSets[i].id },
data: {
id: expect.arrayContaining([fulfillmentSets[i].id]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SERVICE_ZONE_CREATED,
action: "created",
object: "service_zone",
data: { id: fulfillmentSets[i].service_zones[0].id },
data: {
id: expect.arrayContaining([
fulfillmentSets[i].service_zones[0].id,
]),
},
}),
]),
{
@@ -505,20 +513,28 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_CREATED,
action: "created",
object: "fulfillment_set",
data: { id: fulfillmentSets[i].id },
data: {
id: expect.arrayContaining([fulfillmentSets[i].id]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SERVICE_ZONE_CREATED,
action: "created",
object: "service_zone",
data: { id: fulfillmentSets[i].service_zones[0].id },
data: {
id: expect.arrayContaining([
fulfillmentSets[i].service_zones[0].id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.GEO_ZONE_CREATED,
action: "created",
object: "geo_zone",
data: {
id: fulfillmentSets[i].service_zones[0].geo_zones[0].id,
id: expect.arrayContaining([
fulfillmentSets[i].service_zones[0].geo_zones[0].id,
]),
},
}),
]),
@@ -708,7 +724,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
})
expect(updatedFulfillmentSets).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(1)
for (const data_ of updateData) {
const currentFullfillmentSet = fullfillmentSets.find(
@@ -729,7 +745,9 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_UPDATED,
action: "updated",
object: "fulfillment_set",
data: { id: currentFullfillmentSet.id },
data: {
id: expect.arrayContaining([currentFullfillmentSet.id]),
},
}),
]),
{
@@ -1057,7 +1075,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
)
expect(updatedFulfillmentSets).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(10)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(5)
for (const data_ of updateData) {
const expectedFulfillmentSet = updatedFulfillmentSets.find(
@@ -1097,36 +1115,48 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_UPDATED,
action: "updated",
object: "fulfillment_set",
data: { id: expectedFulfillmentSet.id },
data: {
id: expect.arrayContaining([expectedFulfillmentSet.id]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SERVICE_ZONE_CREATED,
action: "created",
object: "service_zone",
data: { id: expectedFulfillmentSet.service_zones[0].id },
data: {
id: expect.arrayContaining([
expectedFulfillmentSet.service_zones[0].id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.GEO_ZONE_CREATED,
action: "created",
object: "geo_zone",
data: {
id: expectedFulfillmentSet.service_zones[0].geo_zones[0]
.id,
id: expect.arrayContaining([
expectedFulfillmentSet.service_zones[0].geo_zones[0].id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SERVICE_ZONE_DELETED,
action: "deleted",
object: "service_zone",
data: { id: originalFulfillmentSet.service_zones[0].id },
data: {
id: expect.arrayContaining([
originalFulfillmentSet.service_zones[0].id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.GEO_ZONE_DELETED,
action: "deleted",
object: "geo_zone",
data: {
id: originalFulfillmentSet.service_zones[0].geo_zones[0]
.id,
id: expect.arrayContaining([
originalFulfillmentSet.service_zones[0].geo_zones[0].id,
]),
},
}),
]),
@@ -1215,7 +1245,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
)
expect(updatedFulfillmentSets).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(6)
expect(eventBusEmitSpy.mock.calls[1][0]).toHaveLength(3)
for (const data_ of updateData) {
const expectedFulfillmentSet = updatedFulfillmentSets.find(
@@ -1259,20 +1289,26 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.FULFILLMENT_SET_UPDATED,
action: "updated",
object: "fulfillment_set",
data: { id: expectedFulfillmentSet.id },
data: {
id: expect.arrayContaining([expectedFulfillmentSet.id]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SERVICE_ZONE_CREATED,
action: "created",
object: "service_zone",
data: { id: createdServiceZone.id },
data: {
id: expect.arrayContaining([createdServiceZone.id]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.GEO_ZONE_CREATED,
action: "created",
object: "geo_zone",
data: {
id: createdServiceZone.geo_zones[0].id,
id: expect.arrayContaining([
createdServiceZone.geo_zones[0].id,
]),
},
}),
]),
packages/modules/fulfillment/integration-tests/__tests__/fulfillment-module-service/geo-zone.spec.ts
+4 -4
View File
@@ -154,7 +154,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
const geoZones = await service.createGeoZones(data)
expect(geoZones).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(1)
let i = 0
for (const data_ of data) {
@@ -172,7 +172,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.GEO_ZONE_CREATED,
action: "created",
object: "geo_zone",
data: { id: geoZones[i].id },
data: { id: expect.arrayContaining([geoZones[i].id]) },
}),
]),
{
@@ -331,7 +331,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
const updatedGeoZones = await service.updateGeoZones(updateData)
expect(updatedGeoZones).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(1)
for (const data_ of updateData) {
const expectedGeoZone = updatedGeoZones.find(
@@ -352,7 +352,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.GEO_ZONE_UPDATED,
action: "updated",
object: "geo_zone",
data: { id: expectedGeoZone.id },
data: { id: expect.arrayContaining([expectedGeoZone.id]) },
}),
]),
{
packages/modules/fulfillment/integration-tests/__tests__/fulfillment-module-service/service-zone.spec.ts
+1 -1
View File
@@ -510,7 +510,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
expect(updatedServiceZones).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(6) // Since the update only calls create and update which are already tested, only check the length
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(3) // Since the update only calls create and update which are already tested, only check the length
for (const data_ of updateData) {
const expectedServiceZone = updatedServiceZones.find(
packages/modules/fulfillment/integration-tests/__tests__/fulfillment-module-service/shipping-option.spec.ts
+16 -4
View File
@@ -582,7 +582,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
)
expect(createdShippingOptions).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(6)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(3)
let i = 0
for (const data_ of createData) {
@@ -619,19 +619,31 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.SHIPPING_OPTION_CREATED,
action: "created",
object: "shipping_option",
data: { id: createdShippingOptions[i].id },
data: {
id: expect.arrayContaining([
createdShippingOptions[i].id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SHIPPING_OPTION_TYPE_CREATED,
action: "created",
object: "shipping_option_type",
data: { id: createdShippingOptions[i].type.id },
data: {
id: expect.arrayContaining([
createdShippingOptions[i].type.id,
]),
},
}),
buildExpectedEventMessageShape({
eventName: FulfillmentEvents.SHIPPING_OPTION_RULE_CREATED,
action: "created",
object: "shipping_option_rule",
data: { id: createdShippingOptions[i].rules[0].id },
data: {
id: expect.arrayContaining([
createdShippingOptions[i].rules[0].id,
]),
},
}),
]),
{
packages/modules/fulfillment/integration-tests/__tests__/fulfillment-module-service/shipping-profile.spec.ts
+6 -2
View File
@@ -100,7 +100,7 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
await service.createShippingProfiles(createData)
expect(createdShippingProfiles).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(2)
expect(eventBusEmitSpy.mock.calls[0][0]).toHaveLength(1)
let i = 0
for (const data_ of createData) {
@@ -117,7 +117,11 @@ moduleIntegrationTestRunner<IFulfillmentModuleService>({
eventName: FulfillmentEvents.SHIPPING_PROFILE_CREATED,
action: "created",
object: "shipping_profile",
data: { id: createdShippingProfiles[i].id },
data: {
id: expect.arrayContaining([
createdShippingProfiles[i].id,
]),
},
}),
]),
{