fix(gatsby-source-medusa): adds support for incremental sourcing of medusa data (#1217)

packages/gatsby-source-medusa/src/client.ts

@@ -1,5 +1,4 @@
 import axios, { AxiosPromise, AxiosRequestConfig } from "axios"
-import { Reporter } from "gatsby-cli/lib/reporter/reporter"
 
 function medusaRequest(
   storeURL: string,
@@ -18,34 +17,28 @@ function medusaRequest(
   return client(options)
 }
 
-export const createClient = (
+export const createClient = (options: MedusaPluginOptions): any => {
-  options: MedusaPluginOptions,
-  reporter: Reporter
-): any => {
   const { storeUrl, authToken } = options as any
 
   /**
-   *
    * @param {string} _date used fetch products updated since the specified date
-   * @return {Promise<any[]>}
+   * @return {Promise<any[]>}  products to create nodes from
    */
   async function products(_date?: string): Promise<any[]> {
     let products: any[] = []
     let offset = 0
     let count = 1
     do {
-      await medusaRequest(storeUrl, `/store/products?offset=${offset}`)
+      let path = `/store/products?offset=${offset}`
-        .then(({ data }) => {
+      if (_date) {
-          products = [...products, ...data.products]
+        path += `&updated_at[gt]=${_date}`
-          count = data.count
+      }
-          offset = data.products.length
+
-        })
+      await medusaRequest(storeUrl, path).then(({ data }) => {
-        .catch((error) => {
+        products = [...products, ...data.products]
-          reporter.error(
+        count = data.count
-            `"The following error status was produced while attempting to fetch products: ${error}`
+        offset = data.products.length
-          )
+      })
-          return []
-        })
     } while (products.length < count)
 
     return products
@@ -53,27 +46,25 @@ export const createClient = (
 
   /**
    *
-   * @param {string} date used fetch regions updated since the specified date
+   * @param {string} _date used fetch regions updated since the specified date
-   * @return {Promise<any[]>}
+   * @return {Promise<any[]>} regions to create nodes from
    */
   async function regions(_date?: string): Promise<any[]> {
-    const regions = await medusaRequest(storeUrl, `/store/regions`)
+    let path = `/store/regions`
-      .then(({ data }) => {
+    if (_date) {
-        return data.regions
+      path += `?updated_at[gt]=${_date}`
-      })
+    }
-      .catch((error) => {
+
-        console.warn(`
+    const regions = await medusaRequest(storeUrl, path).then(({ data }) => {
-            "The following error status was produced while attempting to fetch regions: ${error}
+      return data.regions
-      `)
+    })
-        return []
-      })
     return regions
   }
 
   /**
    *
    * @param {string} _date used fetch regions updated since the specified date
-   * @return {Promise<any[]>}
+   * @return {Promise<any[]>} orders to create nodes from
    */
   async function orders(_date?: string): Promise<any[]> {
     const orders = await medusaRequest(storeUrl, `/admin/orders`, {
@@ -95,25 +86,22 @@ export const createClient = (
   /**
    *
    * @param {string} _date used fetch regions updated since the specified date
-   * @return {Promise<any[]>}
+   * @return {Promise<any[]>} collections to create nodes from
    */
   async function collections(_date?: string): Promise<any[]> {
     let collections: any[] = []
     let offset = 0
     let count = 1
     do {
-      await medusaRequest(storeUrl, `/store/collections?offset=${offset}`)
+      let path = `/store/collections?offset=${offset}`
-        .then(({ data }) => {
+      if (_date) {
-          collections = [...collections, ...data.collections]
+        path += `&updated_at[gt]=${_date}`
-          count = data.count
+      }
-          offset = data.collections.length
+      await medusaRequest(storeUrl, path).then(({ data }) => {
-        })
+        collections = [...collections, ...data.collections]
-        .catch((error) => {
+        count = data.count
-          reporter.error(
+        offset = data.collections.length
-            `"The following error status was produced while attempting to fetch products: ${error}`
+      })
-          )
-          return []
-        })
     } while (collections.length < count)
 
     return collections

packages/gatsby-source-medusa/src/gatsby-node.ts

@@ -27,7 +27,7 @@ async function sourceAllNodes(
     createRegionsOperation,
     createOrdersOperation,
     createCollectionsOperation,
-  } = createOperations(pluginOptions, gatsbyApi)
+  } = createOperations(pluginOptions)
 
   const operations = [
     createProductsOperation,
@@ -54,22 +54,17 @@ const medusaNodeTypes = [
   "MedusaCollections",
 ]
 
-export async function sourceUpdatedNodes(
+async function sourceUpdatedNodes(
   gatsbyApi: SourceNodesArgs,
-  pluginOptions: MedusaPluginOptions
+  pluginOptions: MedusaPluginOptions,
+  lastBuildTime: string
 ): Promise<void> {
   const {
     incrementalProductsOperation,
     incrementalRegionsOperation,
     incrementalOrdersOperation,
     incrementalCollectionsOperation,
-  } = createOperations(pluginOptions, gatsbyApi)
+  } = createOperations(pluginOptions)
-
-  const lastBuildTime = new Date(
-    gatsbyApi.store.getState().status.plugins?.[`gatsby-source-medusa`]?.[
-      `lastBuildTime`
-    ]
-  )
 
   for (const nodeType of medusaNodeTypes) {
     gatsbyApi
@@ -94,36 +89,25 @@ export async function sourceUpdatedNodes(
   }
 }
 
+export async function onPostBuild({ cache }: { cache: any }): Promise<void> {
+  await cache.set("timestamp", Date.now())
+}
+
 export async function sourceNodes(
   gatsbyApi: SourceNodesArgs,
   pluginOptions: MedusaPluginOptions
 ): Promise<void> {
-  const pluginStatus =
+  const { cache } = gatsbyApi
-    gatsbyApi.store.getState().status.plugins?.[`gatsby-source-medusa`]
+  const lastBuildTime = await cache.get("timestamp")
-
-  const lastBuildTime = pluginStatus?.[`lastBuildTime`]
 
   if (lastBuildTime !== undefined) {
-    gatsbyApi.reporter.info(
+    await sourceUpdatedNodes(gatsbyApi, pluginOptions, lastBuildTime)
-      `Cache is warm, but incremental builds are currently not supported. Running a clean build.`
-    )
-    await sourceAllNodes(gatsbyApi, pluginOptions)
   } else {
     gatsbyApi.reporter.info(`Cache is cold, running a clean build.`)
     await sourceAllNodes(gatsbyApi, pluginOptions)
   }
 
-  gatsbyApi.reporter.info(`Finished sourcing nodes, caching last build time`)
+  gatsbyApi.reporter.info(`Finished sourcing nodes`)
-  gatsbyApi.actions.setPluginStatus(
-    pluginStatus !== undefined
-      ? {
-          ...pluginStatus,
-          [`lastBuildTime`]: Date.now(),
-        }
-      : {
-          [`lastBuildTime`]: Date.now(),
-        }
-  )
 }
 
 export function createResolvers({ createResolvers }: CreateResolversArgs): any {

packages/gatsby-source-medusa/src/operations.ts

@@ -1,11 +1,7 @@
-import { SourceNodesArgs } from "gatsby"
 import { createClient } from "./client"
 
-export function createOperations(
+export function createOperations(options: MedusaPluginOptions): IOperations {
-  options: MedusaPluginOptions,
+  const client = createClient(options)
-  { reporter }: SourceNodesArgs
-): IOperations {
-  const client = createClient(options, reporter)
 
   function createOperation(
     name: "products" | "collections" | "regions" | "orders",
@@ -22,13 +18,13 @@ export function createOperations(
     createCollectionsOperation: createOperation("collections"),
     createRegionsOperation: createOperation("regions"),
     createOrdersOperation: createOperation("orders"),
-    incrementalProductsOperation: (date: Date): any =>
+    incrementalProductsOperation: (date: string): any =>
-      createOperation("products", date.toISOString()),
+      createOperation("products", date),
-    incrementalCollectionsOperation: (date: Date): any =>
+    incrementalCollectionsOperation: (date: string): any =>
-      createOperation("collections", date.toISOString()),
+      createOperation("collections", date),
-    incrementalRegionsOperation: (date: Date): any =>
+    incrementalRegionsOperation: (date: string): any =>
-      createOperation("regions", date.toISOString()),
+      createOperation("regions", date),
-    incrementalOrdersOperation: (date: Date): any =>
+    incrementalOrdersOperation: (date: string): any =>
-      createOperation("orders", date.toISOString()),
+      createOperation("orders", date),
   }
 }

packages/gatsby-source-medusa/types/interface.d.ts

@@ -22,8 +22,8 @@ interface IOperations {
   createCollectionsOperation: IMedusaOperation
   createRegionsOperation: IMedusaOperation
   createOrdersOperation: IMedusaOperation
-  incrementalProductsOperation: (date: Date) => IMedusaOperation
+  incrementalProductsOperation: (date: string) => IMedusaOperation
-  incrementalCollectionsOperation: (date: Date) => IMedusaOperation
+  incrementalCollectionsOperation: (date: string) => IMedusaOperation
-  incrementalRegionsOperation: (date: Date) => IMedusaOperation
+  incrementalRegionsOperation: (date: string) => IMedusaOperation
-  incrementalOrdersOperation: (date: Date) => IMedusaOperation
+  incrementalOrdersOperation: (date: string) => IMedusaOperation
 }