feat(workflows): Data aggregation (#4732)

* apply the aggregator automatically

* add comment

* apply aggregate

* improve pipe aggregation

* improve test cases

* improvements

* clean tests

* renameing to merge

* fix merge apply

* move merge apply

* cleanup cart workflow and end point

* fixes and naming

packages/workflows/src/helper/__tests__/merge-data.spec.ts

@@ -1,8 +1,8 @@
-import { aggregateData } from "../aggregate"
+import { mergeData } from "../merge-data"
 import { WorkflowStepMiddlewareReturn } from "../pipe"
 
-describe("aggregate", function () {
-  it("should aggregate a new object from the source into a specify target", async function () {
+describe("merge", function () {
+  it("should merge a new object from the source into a specify target", async function () {
     const source = {
       stringProp: "stringProp",
       anArray: ["anArray"],
@@ -14,13 +14,14 @@ describe("aggregate", function () {
       },
     }
 
-    const { value: result } = (await aggregateData(
+    const result = (await mergeData(
       ["input", "another", "stringProp", "anArray"],
       "payload"
     )({ data: source } as any)) as unknown as WorkflowStepMiddlewareReturn
 
     expect(result).toEqual({
-      payload: {
+      alias: "payload",
+      value: {
         ...source.input,
         ...source.another,
         anArray: source.anArray,
@@ -29,7 +30,7 @@ describe("aggregate", function () {
     })
   })
 
-  it("should aggregate a new object from the entire source into the resul object", async function () {
+  it("should merge a new object from the entire source into the resul object", async function () {
     const source = {
       stringProp: "stringProp",
       anArray: ["anArray"],
@@ -41,7 +42,7 @@ describe("aggregate", function () {
       },
     }
 
-    const { value: result } = (await aggregateData()({
+    const { value: result } = (await mergeData()({
       data: source,
     } as any)) as unknown as WorkflowStepMiddlewareReturn
 

packages/workflows/src/helper/__tests__/pipe.spec.ts

@@ -46,6 +46,152 @@ describe("Pipe", function () {
     expect(result).toEqual(output)
   })
 
+  it("should evaluate the input object and append the values to the data object using the merge and return the result from the handler", async function () {
+    const payload = { input: "input" }
+    const output = { test: "test" }
+    const invoke = {
+      input: payload,
+      step1: { step1Data: { test: "test" } },
+      step2: [{ test: "test" }],
+    }
+
+    const handler = jest.fn().mockImplementation(async () => output)
+    const input = {
+      inputAlias: "payload",
+      merge: true,
+      invoke: [
+        {
+          from: "payload",
+        },
+        {
+          from: "step1",
+        },
+        {
+          from: "step2",
+          alias: "step2Data",
+        },
+      ],
+    }
+
+    const result = await pipe(input, handler)({ invoke, payload } as any)
+
+    expect(handler).toHaveBeenCalled()
+    expect(handler).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: {
+          ...payload,
+          ...invoke.step1,
+          step2Data: invoke.step2,
+        },
+      })
+    )
+
+    expect(result).toBeDefined()
+    expect(result).toEqual(output)
+  })
+
+  it("should evaluate the input object and append the values to the data object using the merge to store on the merge alias and return the result from the handler", async function () {
+    const payload = { input: "input" }
+    const output = { test: "test" }
+    const invoke = {
+      input: payload,
+      step1: { step1Data: { test: "test" } },
+      step2: { step2Data: { test: "test" } },
+    }
+
+    const handler = jest.fn().mockImplementation(async () => output)
+    const input = {
+      inputAlias: "payload",
+      mergeAlias: "mergedData",
+      invoke: [
+        {
+          from: "payload",
+          alias: "input",
+        },
+        {
+          from: "step1",
+          alias: "step1Data",
+        },
+        {
+          from: "step2",
+          alias: "step2Data",
+        },
+      ],
+    }
+
+    const result = await pipe(input, handler)({ invoke, payload } as any)
+
+    expect(handler).toHaveBeenCalled()
+    expect(handler).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: {
+          input: payload,
+          step1Data: invoke.step1,
+          step2Data: invoke.step2,
+          mergedData: {
+            ...payload,
+            ...invoke.step1,
+            ...invoke.step2,
+          },
+        },
+      })
+    )
+
+    expect(result).toBeDefined()
+    expect(result).toEqual(output)
+  })
+
+  it("should evaluate the input object and append the values to the data object using the merge to store on the merge alias from the merge from values and return the result from the handler", async function () {
+    const payload = { input: "input" }
+    const output = { test: "test" }
+    const invoke = {
+      input: payload,
+      step1: { step1Data: { test: "test" } },
+      step2: { step2Data: { test: "test" } },
+    }
+
+    const handler = jest.fn().mockImplementation(async () => output)
+    const input = {
+      inputAlias: "payload",
+      mergeAlias: "mergedData",
+      mergeFrom: ["input", "step1Data"],
+      invoke: [
+        {
+          from: "payload",
+          alias: "input",
+        },
+        {
+          from: "step1",
+          alias: "step1Data",
+        },
+        {
+          from: "step2",
+          alias: "step2Data",
+        },
+      ],
+    }
+
+    const result = await pipe(input, handler)({ invoke, payload } as any)
+
+    expect(handler).toHaveBeenCalled()
+    expect(handler).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: {
+          input: payload,
+          step1Data: invoke.step1,
+          step2Data: invoke.step2,
+          mergedData: {
+            ...payload,
+            ...invoke.step1,
+          },
+        },
+      })
+    )
+
+    expect(result).toBeDefined()
+    expect(result).toEqual(output)
+  })
+
   it("should execute onComplete function if available but the output result shouldn't change", async function () {
     const payload = { input: "input" }
     const output = { test: "test" }

packages/workflows/src/helper/index.ts

@@ -1,4 +1,4 @@
-export * from "./aggregate"
+export * from "./merge-data"
 export * from "./empty-handler"
 export * from "./pipe"
 export * from "./workflow-export"

packages/workflows/src/helper/merge-data.ts

@@ -2,12 +2,12 @@ import { PipelineHandler, WorkflowArguments } from "./pipe"
 import { isObject } from "@medusajs/utils"
 
 /**
- * Pipe utils that aggregates data from an object into a new object.
- * The new object will have a target key with the aggregated data from the keys.
+ * Pipe utils that merges data from an object into a new object.
+ * The new object will have a target key with the merged data from the keys if specified.
  * @param keys
  * @param target
  */
-export function aggregateData<
+export function mergeData<
   T extends Record<string, unknown> = Record<string, unknown>,
   TKeys extends keyof T = keyof T,
   Target extends "payload" | string = string
@@ -43,7 +43,7 @@ export function aggregateData<
 
     return {
       alias: target,
-      value,
+      value: target ? value[target as string] : value,
     }
   }
 }

packages/workflows/src/helper/pipe.ts

@@ -1,11 +1,11 @@
 import {
+  DistributedTransaction,
   TransactionMetadata,
   WorkflowStepHandler,
 } from "@medusajs/orchestration"
 import { Context, MedusaContainer, SharedContext } from "@medusajs/types"
-
-import { DistributedTransaction } from "@medusajs/orchestration"
 import { InputAlias } from "../definitions"
+import { mergeData } from "./merge-data"
 
 export type WorkflowStepMiddlewareReturn = {
   alias?: string
@@ -18,10 +18,28 @@ export type WorkflowStepMiddlewareInput = {
 }
 
 interface PipelineInput {
+  /**
+   * The alias of the input data to store in
+   */
   inputAlias?: InputAlias | string
+  /**
+   * Descriptors to get the data from
+   */
   invoke?: WorkflowStepMiddlewareInput | WorkflowStepMiddlewareInput[]
   compensate?: WorkflowStepMiddlewareInput | WorkflowStepMiddlewareInput[]
-  onComplete?: (args: WorkflowOnCompleteArguments) => {}
+  onComplete?: (args: WorkflowOnCompleteArguments) => Promise<void>
+  /**
+   * Apply the data merging
+   */
+  merge?: boolean
+  /**
+   * Store the merged data in a new key, if this is present no need to set merge: true
+   */
+  mergeAlias?: string
+  /**
+   * Store the merged data from the chosen aliases, if this is present no need to set merge: true
+   */
+  mergeFrom?: string[]
 }
 
 export type WorkflowArguments<T = any> = {
@@ -53,6 +71,15 @@ export function pipe<T>(
   input: PipelineInput,
   ...functions: [...PipelineHandler[], PipelineHandler<T>]
 ): WorkflowStepHandler {
+  // Apply the aggregator just before the last handler
+  if (
+    (input.merge || input.mergeAlias || input.mergeFrom) &&
+    functions.length
+  ) {
+    const handler = functions.pop()!
+    functions.push(mergeData(input.mergeFrom, input.mergeAlias), handler)
+  }
+
   return async ({
     container,
     payload,