feat: add Filesystem util and load env util (#7487)

.github/workflows/action.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Assert changed

.github/workflows/docs-freshness-check.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies

.github/workflows/docs-test.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "18"
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies
@@ -75,7 +75,7 @@ jobs:
       # an error state
       - name: Get PR files number
         working-directory: www/utils/packages/scripts
-        run: 'yarn check:pr-files-count ${{ github.ref_name }}'
+        run: "yarn check:pr-files-count ${{ github.ref_name }}"
         id: pr-files
 
       - name: Get Directories to Scan
@@ -90,7 +90,7 @@ jobs:
         with:
           files: ${{ steps.directories.outputs.LIST }}
           fail_on_error: true
-          vale_flags: '--minAlertLevel=error'
+          vale_flags: "--minAlertLevel=error"
           reporter: github-pr-check
           token: ${{ github.token }}
           filter_mode: nofilter
@@ -127,7 +127,7 @@ jobs:
       # an error state
       - name: Get PR files number
         working-directory: www/utils/packages/scripts
-        run: 'yarn check:pr-files-count ${{ github.ref_name }}'
+        run: "yarn check:pr-files-count ${{ github.ref_name }}"
         id: pr-files
 
       - name: Get Directories to Scan
@@ -141,7 +141,7 @@ jobs:
         with:
           files: ${{ steps.directories.outputs.LIST }}
           fail_on_error: true
-          vale_flags: '--minAlertLevel=error'
+          vale_flags: "--minAlertLevel=error"
           reporter: github-pr-check
           token: ${{ github.token }}
           filter_mode: nofilter
@@ -178,7 +178,7 @@ jobs:
       # an error state
       - name: Get PR files number
         working-directory: www/utils/packages/scripts
-        run: 'yarn check:pr-files-count ${{ github.ref_name }}'
+        run: "yarn check:pr-files-count ${{ github.ref_name }}"
         id: pr-files
 
       - name: Get Directories to Scan
@@ -192,7 +192,7 @@ jobs:
         with:
           files: ${{ steps.directories.outputs.LIST }}
           fail_on_error: true
-          vale_flags: '--minAlertLevel=error'
+          vale_flags: "--minAlertLevel=error"
           reporter: github-pr-check
           token: ${{ github.token }}
           filter_mode: nofilter
@@ -229,7 +229,7 @@ jobs:
       # an error state
       - name: Get PR files number
         working-directory: www/utils/packages/scripts
-        run: 'yarn check:pr-files-count ${{ github.ref_name }}'
+        run: "yarn check:pr-files-count ${{ github.ref_name }}"
         id: pr-files
 
       - name: Get Directories to Scan
@@ -244,7 +244,7 @@ jobs:
         with:
           files: ${{ steps.directories.outputs.LIST }}
           fail_on_error: true
-          vale_flags: '--minAlertLevel=error'
+          vale_flags: "--minAlertLevel=error"
           reporter: github-pr-check
           token: ${{ github.token }}
           filter_mode: nofilter
@@ -281,7 +281,7 @@ jobs:
       # an error state
       - name: Get PR files number
         working-directory: www/utils/packages/scripts
-        run: 'yarn check:pr-files-count ${{ github.ref_name }}'
+        run: "yarn check:pr-files-count ${{ github.ref_name }}"
         id: pr-files
 
       - name: Get Directories to Scan
@@ -296,7 +296,7 @@ jobs:
         with:
           files: ${{ steps.directories.outputs.LIST }}
           fail_on_error: true
-          vale_flags: '--minAlertLevel=error'
+          vale_flags: "--minAlertLevel=error"
           reporter: github-pr-check
           token: ${{ github.token }}
           filter_mode: diff_context

.github/workflows/generate-docblocks-reference.yml

@@ -9,9 +9,9 @@ on:
   workflow_dispatch:
     inputs:
       referenceName:
-        description: 'Reference to Generate. Use either `all` or a name of a config file in `www/utils/packages/typedoc-config` such as `product`.'
+        description: "Reference to Generate. Use either `all` or a name of a config file in `www/utils/packages/typedoc-config` such as `product`."
         required: false
-        default: 'all'
+        default: "all"
 
 jobs:
   references:
@@ -31,7 +31,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies

.github/workflows/generate-docblocks.yml

@@ -14,10 +14,10 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
-      - name: Setup Node.js 18
+      - name: Setup Node.js 20
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Install Dependencies
         run: yarn
@@ -70,10 +70,10 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
-      - name: Setup Node.js 18
+      - name: Setup Node.js 20
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Install Dependencies
         run: yarn

.github/workflows/generate-public-references.yml

@@ -3,9 +3,9 @@ on:
   workflow_dispatch:
     inputs:
       referenceName:
-        description: 'Reference to Generate. Use either `all` to generate all references, `api` to generate the API reference, or `ui` to generate UI reference.'
+        description: "Reference to Generate. Use either `all` to generate all references, `api` to generate the API reference, or `ui` to generate UI reference."
         required: false
-        default: 'all'
+        default: "all"
   release:
     types: [published]
 
@@ -28,7 +28,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies
@@ -78,7 +78,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies

.github/workflows/release-notifications.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v3
         with:
-          node-version: "16.10.0"
+          node-version: 20
           cache: "yarn"
 
       - name: Post to Slack channel

.github/workflows/release.yml

@@ -15,10 +15,10 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
-      - name: Setup Node.js 16
+      - name: Setup Node.js 20
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 20
 
       - name: Creating .npmrc
         run: |

.github/workflows/snapshot-this.yml

@@ -79,7 +79,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 20
           cache: "yarn"
 
       - name: Install dependencies

.github/workflows/test-cli-with-database.yml

@@ -45,7 +45,7 @@ jobs:
         uses: ./.github/actions/setup-server
         with:
           cache-extension: "cli-test"
-         node-version: 18
+          node-version: 20
 
       - name: Install Medusa cli
         run: npm i -g @medusajs/medusa-cli@preview

.github/workflows/trigger-release.yml

@@ -5,7 +5,7 @@ on:
     inputs:
       version:
         type: choice
-        default: 'preview'
+        default: "preview"
         description: What tag do you want to release?
         required: true
         options:
@@ -32,10 +32,10 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
-      - name: Setup Node.js 16.x
+      - name: Setup Node.js 20
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 20
 
       - name: Creating .npmrc
         run: |

packages/core/medusa-test-utils/src/medusa-test-runner-utils/bootstrap-app.ts

@@ -1,7 +1,7 @@
 import express from "express"
 import getPort from "get-port"
 import { resolve } from "path"
-import { isObject, promiseAll } from "@medusajs/utils"
+import { isObject, promiseAll, GracefulShutdownServer } from "@medusajs/utils"
 import { MedusaContainer } from "@medusajs/types"
 
 async function bootstrapApp({
@@ -52,7 +52,7 @@ export async function startApp({
   let expressServer
 
   const shutdown = async () => {
-    await promiseAll([expressServer.shutdown(), medusaShutdown()])
+    await promiseAll([expressServer?.shutdown(), medusaShutdown()])
 
     if (typeof global !== "undefined" && global?.gc) {
       global.gc()
@@ -77,10 +77,6 @@ export async function startApp({
       })
 
     // TODO: fix that once we find the appropriate place to put this util
-    const {
-      GracefulShutdownServer,
-    } = require("@medusajs/medusa/dist/utils/graceful-shutdown-server")
-
     expressServer = GracefulShutdownServer.create(server)
   })
 }

packages/core/utils/package.json

@@ -36,6 +36,7 @@
     "@mikro-orm/postgresql": "5.9.7",
     "awilix": "^8.0.1",
     "bignumber.js": "^9.1.2",
+    "dotenv": "^16.4.5",
     "knex": "2.4.2",
     "ulid": "^2.3.0"
   },

packages/core/utils/src/common/__tests__/load-env.spec.ts

@@ -0,0 +1,40 @@
+import { join } from "path"
+import { FileSystem } from "../file-system"
+import { loadEnv } from "../load-env"
+
+const filesystem = new FileSystem(join(__dirname, "tmp"))
+
+describe("loadEnv", function () {
+  afterEach(async () => {
+    await filesystem.cleanup()
+    delete process.env.MEDUSA_VERSION
+    delete process.env.MEDUSA_DEV_VERSION
+    delete process.env.MEDUSA_TEST_VERSION
+    delete process.env.MEDUSA_STAGING_VERSION
+    delete process.env.MEDUSA_PRODUCTION_VERSION
+  })
+
+  it("should load .env file when in unknown environment", async function () {
+    await filesystem.create(".env", "MEDUSA_VERSION=1.0")
+    loadEnv("", filesystem.basePath)
+
+    expect(process.env.MEDUSA_VERSION).toEqual("1.0")
+  })
+
+  it("should load .env file for known environments", async function () {
+    await filesystem.create(".env", "MEDUSA_DEV_VERSION=1.0")
+    await filesystem.create(".env.test", "MEDUSA_TEST_VERSION=1.0")
+    await filesystem.create(".env.staging", "MEDUSA_STAGING_VERSION=1.0")
+    await filesystem.create(".env.production", "MEDUSA_PRODUCTION_VERSION=1.0")
+
+    loadEnv("development", filesystem.basePath)
+    loadEnv("test", filesystem.basePath)
+    loadEnv("staging", filesystem.basePath)
+    loadEnv("production", filesystem.basePath)
+
+    expect(process.env.MEDUSA_DEV_VERSION).toEqual("1.0")
+    expect(process.env.MEDUSA_TEST_VERSION).toEqual("1.0")
+    expect(process.env.MEDUSA_STAGING_VERSION).toEqual("1.0")
+    expect(process.env.MEDUSA_PRODUCTION_VERSION).toEqual("1.0")
+  })
+})

packages/core/utils/src/common/file-system.ts

@@ -0,0 +1,159 @@
+import { dirname, join } from "path"
+import {
+  promises,
+  constants,
+  type Dirent,
+  type RmOptions,
+  type StatOptions,
+  type WriteFileOptions,
+  type MakeDirectoryOptions,
+} from "fs"
+
+const { rm, stat, mkdir, access, readdir, readFile, writeFile } = promises
+
+export type JSONFileOptions = WriteFileOptions & {
+  spaces?: number | string
+  replacer?: (this: any, key: string, value: any) => any
+}
+
+/**
+ * File system abstraction to create and cleanup files during
+ * tests
+ */
+export class FileSystem {
+  constructor(public basePath: string) {}
+
+  private makePath(filePath: string) {
+    return join(this.basePath, filePath)
+  }
+
+  /**
+   * Cleanup directory
+   */
+  async cleanup(options?: RmOptions) {
+    return rm(this.basePath, {
+      recursive: true,
+      maxRetries: 10,
+      force: true,
+      ...options,
+    })
+  }
+
+  /**
+   * Creates a directory inside the root of the filesystem
+   * path. You may use this method to create nested
+   * directories as well.
+   */
+  mkdir(dirPath: string, options?: MakeDirectoryOptions) {
+    return mkdir(this.makePath(dirPath), { recursive: true, ...options })
+  }
+
+  /**
+   * Create a new file
+   */
+  async create(filePath: string, contents: string, options?: WriteFileOptions) {
+    const absolutePath = this.makePath(filePath)
+    await mkdir(dirname(absolutePath), { recursive: true })
+    return writeFile(this.makePath(filePath), contents, options)
+  }
+
+  /**
+   * Remove a file
+   */
+  async remove(filePath: string, options?: RmOptions) {
+    return rm(this.makePath(filePath), {
+      recursive: true,
+      force: true,
+      maxRetries: 2,
+      ...options,
+    })
+  }
+
+  /**
+   * Check if the root of the filesystem exists
+   */
+  async rootExists() {
+    try {
+      await access(this.basePath, constants.F_OK)
+      return true
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return false
+      }
+      throw error
+    }
+  }
+
+  /**
+   * Check if a file exists
+   */
+  async exists(filePath: string) {
+    try {
+      await access(this.makePath(filePath), constants.F_OK)
+      return true
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return false
+      }
+      throw error
+    }
+  }
+
+  /**
+   * Returns file contents
+   */
+  async contents(filePath: string) {
+    return readFile(this.makePath(filePath), "utf-8")
+  }
+
+  /**
+   * Dumps file contents to the stdout
+   */
+  async dump(filePath: string) {
+    console.log("------------------------------------------------------------")
+    console.log(`file path => "${filePath}"`)
+    console.log(`contents => "${await this.contents(filePath)}"`)
+  }
+
+  /**
+   * Returns stats for a file
+   */
+  async stats(filePath: string, options?: StatOptions) {
+    return stat(this.makePath(filePath), options)
+  }
+
+  /**
+   * Recursively reads files from a given directory
+   */
+  readDir(dirPath?: string): Promise<Dirent[]> {
+    const location = dirPath ? this.makePath(dirPath) : this.basePath
+    return readdir(location, {
+      recursive: true,
+      withFileTypes: true,
+    })
+  }
+
+  /**
+   * Create a json file
+   */
+  async createJson(filePath: string, contents: any, options?: JSONFileOptions) {
+    if (options && typeof options === "object") {
+      const { replacer, spaces, ...rest } = options
+      return this.create(
+        filePath,
+        JSON.stringify(contents, replacer, spaces),
+        rest
+      )
+    }
+
+    return this.create(filePath, JSON.stringify(contents), options)
+  }
+
+  /**
+   * Read and parse a json file
+   */
+  async contentsJson(filePath: string) {
+    const contents = await readFile(this.makePath(filePath), "utf-8")
+    return JSON.parse(contents)
+  }
+}

packages/core/utils/src/common/index.ts

@@ -61,3 +61,6 @@ export * from "./to-handle"
 export * from "./validate-handle"
 export * from "./parse-cors-origins"
 export * from "./build-regexp-if-valid"
+export * from "./load-env"
+export * from "./file-system"
+export * from "./graceful-shutdown-server"

packages/core/utils/src/common/load-env.ts

@@ -0,0 +1,24 @@
+import dotenv from "dotenv"
+import { join } from "path"
+const KNOWN_ENVIRONMENTS = ["staging", "production", "test"]
+
+/**
+ * Loads ".env" file based upon the environment in which the
+ * app is running.
+ *
+ * - Loads ".env" file by default.
+ * - Loads ".env.staging" when "environment=staging".
+ * - Loads ".env.production" when "environment=production".
+ * - Loads ".env.test" when "environment=test".
+ *
+ * This method does not return any value and updates the "process.env"
+ * object instead.
+ */
+export function loadEnv(environment: string, envDir: string) {
+  const fileToLoad = KNOWN_ENVIRONMENTS.includes(environment)
+    ? `.env.${environment}`
+    : ".env"
+  try {
+    dotenv.config({ path: join(envDir, fileToLoad) })
+  } catch {}
+}

packages/medusa/src/commands/start-cluster.ts

@@ -3,14 +3,13 @@ import "regenerator-runtime/runtime"
 
 import cluster from "cluster"
 import express from "express"
-import { GracefulShutdownServer } from "../utils"
 import { track } from "medusa-telemetry"
 import { scheduleJob } from "node-schedule"
 import os from "os"
 
 import loaders from "../loaders"
 import Logger from "../loaders/logger"
-import { isPresent } from "@medusajs/utils"
+import { isPresent, GracefulShutdownServer } from "@medusajs/utils"
 
 const EVERY_SIXTH_HOUR = "0 */6 * * *"
 const CRON_SCHEDULE = EVERY_SIXTH_HOUR

packages/medusa/src/commands/start.ts

@@ -2,12 +2,12 @@ import "core-js/stable"
 import "regenerator-runtime/runtime"
 
 import express from "express"
-import { GracefulShutdownServer } from "../utils"
 import { track } from "medusa-telemetry"
 import { scheduleJob } from "node-schedule"
 
 import loaders from "../loaders"
 import Logger from "../loaders/logger"
+import { GracefulShutdownServer } from "@medusajs/utils"
 
 const EVERY_SIXTH_HOUR = "0 */6 * * *"
 const CRON_SCHEDULE = EVERY_SIXTH_HOUR

packages/medusa/src/loaders/__tests__/feature-flags.spec.ts

@@ -1,13 +1,8 @@
-import { mkdirSync, rmSync, writeFileSync } from "fs"
-import { resolve } from "path"
+import { join } from "path"
+import { FileSystem } from "@medusajs/utils"
 
 import loadFeatureFlags from "../feature-flags"
-
-const distTestTargetDirectorPath = resolve(__dirname, "__ff-test__")
-
-const getFolderTestTargetDirectoryPath = (folderName: string): string => {
-  return resolve(distTestTargetDirectorPath, folderName)
-}
+const filesystem = new FileSystem(join(__dirname, "__ff-test__"))
 
 const buildFeatureFlag = (
   key: string,
@@ -28,60 +23,42 @@ const buildFeatureFlag = (
 describe("feature flags", () => {
   const OLD_ENV = { ...process.env }
 
-  beforeEach(() => {
+  beforeEach(async () => {
     jest.resetModules()
     jest.clearAllMocks()
 
     process.env = { ...OLD_ENV }
-
-    rmSync(distTestTargetDirectorPath, { recursive: true, force: true })
-
-    mkdirSync(getFolderTestTargetDirectoryPath("project"), {
-      mode: "777",
-      recursive: true,
+    await filesystem.cleanup()
   })
 
-    mkdirSync(getFolderTestTargetDirectoryPath("flags"), {
-      mode: "777",
-      recursive: true,
-    })
-  })
-
-  afterAll(() => {
+  afterAll(async () => {
     process.env = OLD_ENV
-    rmSync(distTestTargetDirectorPath, { recursive: true, force: true })
+    await filesystem.cleanup()
   })
 
   it("should load the flag from project", async () => {
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-1.js"),
-      buildFeatureFlag("flag-1", true)
-    )
+    await filesystem.create("flags/flag-1.js", buildFeatureFlag("flag-1", true))
 
-    const flags = await loadFeatureFlags(
+    const flags = loadFeatureFlags(
       { featureFlags: { flag_1: false } },
       undefined,
-      getFolderTestTargetDirectoryPath("flags")
+      join(filesystem.basePath, "flags")
     )
 
     expect(flags.isFeatureEnabled("flag_1")).toEqual(false)
   })
 
   it("should load a nested + simple flag from project", async () => {
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "test.js"),
-      buildFeatureFlag("test", false)
-    )
-
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "simpletest.js"),
+    await filesystem.create("flags/test.js", buildFeatureFlag("test", false))
+    await filesystem.create(
+      "flags/simpletest.js",
       buildFeatureFlag("simpletest", false)
     )
 
-    const flags = await loadFeatureFlags(
+    const flags = loadFeatureFlags(
       { featureFlags: { test: { nested: true }, simpletest: true } },
       undefined,
-      getFolderTestTargetDirectoryPath("flags")
+      join(filesystem.basePath, "flags")
     )
 
     expect(flags.isFeatureEnabled({ test: "nested" })).toEqual(true)
@@ -89,15 +66,15 @@ describe("feature flags", () => {
   })
 
   it("should load the default feature flags", async () => {
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-1.js"),
-      buildFeatureFlag("flag-1", true)
+    await filesystem.create(
+      "flags/flag-1.js",
+      buildFeatureFlag("flag-1", false)
     )
 
-    const flags = await loadFeatureFlags(
+    const flags = loadFeatureFlags(
       {},
       undefined,
-      getFolderTestTargetDirectoryPath("flags")
+      join(filesystem.basePath, "flags")
     )
 
     expect(flags.isFeatureEnabled("flag_1")).toEqual(true)
@@ -106,15 +83,14 @@ describe("feature flags", () => {
   it("should load the flag from env", async () => {
     process.env.MEDUSA_FF_FLAG_1 = "false"
 
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-1.js"),
-      buildFeatureFlag("flag-1", true)
+    await filesystem.create(
+      "flags/flag-1.js",
+      buildFeatureFlag("flag-1", false)
     )
-
-    const flags = await loadFeatureFlags(
+    const flags = loadFeatureFlags(
       {},
       undefined,
-      getFolderTestTargetDirectoryPath("flags")
+      join(filesystem.basePath, "flags")
     )
 
     expect(flags.isFeatureEnabled("flag_1")).toEqual(false)
@@ -122,26 +98,23 @@ describe("feature flags", () => {
 
   it("should load mix of flags", async () => {
     process.env.MEDUSA_FF_FLAG_3 = "false"
-
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-1.js"),
-      buildFeatureFlag("flag-1", true)
+    await filesystem.create(
+      "flags/flag-1.js",
+      buildFeatureFlag("flag-1", false)
+    )
+    await filesystem.create(
+      "flags/flag-2.js",
+      buildFeatureFlag("flag-2", false)
+    )
+    await filesystem.create(
+      "flags/flag-3.js",
+      buildFeatureFlag("flag-3", false)
     )
 
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-2.js"),
-      buildFeatureFlag("flag-2", true)
-    )
-
-    writeFileSync(
-      resolve(getFolderTestTargetDirectoryPath("flags"), "flag-3.js"),
-      buildFeatureFlag("flag-3", true)
-    )
-
-    const flags = await loadFeatureFlags(
+    const flags = loadFeatureFlags(
       { featureFlags: { flag_2: false } },
       undefined,
-      getFolderTestTargetDirectoryPath("flags")
+      join(filesystem.basePath, "flags")
     )
 
     expect(flags.isFeatureEnabled("flag_1")).toEqual(true)

packages/medusa/src/utils/index.ts

@@ -3,4 +3,3 @@ export * from "./exception-formatter"
 export * from "./middlewares"
 export * from "./omit-deep"
 export * from "./remove-undefined-properties"
-export * from "./graceful-shutdown-server"

yarn.lock

@@ -6178,6 +6178,7 @@ __metadata:
     awilix: ^8.0.1
     bignumber.js: ^9.1.2
     cross-env: ^5.2.1
+    dotenv: ^16.4.5
     express: ^4.18.2
     jest: ^29.6.3
     knex: 2.4.2