chore: improve ci pipeline (#1764)

* fix: caching deps + add true parallelization to integration tests api

* fix: github action

* chore: upgrade to yarn berry (#1762)

* init migration

* remove: yarn.lock from all pkgs

* fix: build script in pkgs

* adjust yarn config

* fix: yarn.lock and yarnrc.yml

* fix: github actions

* fix: wrong type

* fix (medusa-react): use dts-cli instead of tsdx

* fix: yarn.lock

* fix: yarn v

* fix: prepare script

* add: comment on why we need to downgrade yarn before medusa-dev

* chore: move to Turborepo (#1763)

* increase number of parallel nodes

* fix (medusa-fulfillment-webshipper): build script

* fix: use new version of medusa-dev

* fix: rename cache-bootstrap to cache-deps

.github/actions/cache-bootstrap/action.yml

@@ -1,33 +0,0 @@
-name: cache-bootstrap
-description: Creates a cache with the given extension for lerna packages
-inputs:
-  extension:
-    description: Extension for cache name
-  partial:
-    description: Boolean flag to describe whether or not to run a partial bootstrap when finding cache
-    default: false
-runs:
-  using: composite
-  steps:
-    # for always overriding cache, use: pat-s/always-upload-cache@v2.1.5
-    - uses: actions/cache@v2
-      id: cache
-      with:
-        path: |
-          node_modules
-          */*/node_modules
-        key: ${{ runner.os }}-yarn-${{inputs.extension}}-v8-${{ hashFiles('**/yarn.lock') }}
-      # We want to only bootstrap and install if no cache is found.
-      # Futhermore, we might want to do a partial, hoisted, bootstrap towards
-      # the base branch if it exists, otherwise we choose develop for this.
-
-      # yarn install --frozen-lockfile
-    - run: |
-        if [[ "${{steps.cache.outputs.cache-hit}}" != "true" || "${{inputs.partial}}" != "true" ]]; then
-          yarn install --frozen-lockfile
-          yarn bootstrap --concurrency=2
-        elif [[ "${{inputs.partial}}" = "true" ]]; then
-          [[ ! -z "${GITHUB_BASE_REF}" ]] && ref="${GITHUB_BASE_REF#refs/heads/}" || ref="develop"
-          yarn bootstrap --npm-client=npm --hoist --since "origin/${ref}...HEAD" --concurrency=2 
-        fi
-      shell: bash

.github/actions/cache-deps/action.yml

@@ -0,0 +1,23 @@
+name: cache-deps
+description: Creates a cache with the given extension for monorepo dependencies
+inputs:
+  extension:
+    description: Extension for cache name
+runs:
+  using: composite
+  steps:
+    - uses: actions/cache@v3
+      id: cache
+      with:
+        path: |
+          node_modules
+          */*/node_modules
+        key: ${{ runner.os }}-yarn-${{inputs.extension}}-v8-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-${{inputs.extension}}-v8
+      # We want to only bootstrap and install if no cache is found.
+    - run: |
+        if [[ "${{steps.cache.outputs.cache-hit}}" != "true" ]]; then
+          yarn install --immutable
+        fi
+      shell: bash

.github/actions/setup-server/action.yml

@@ -24,8 +24,8 @@ runs:
         node-version: ${{ inputs.node-version }}
         cache: "yarn"
 
-    - name: Bootstrap packages
-      uses: ./.github/actions/cache-bootstrap
+    - name: Install dependencies
+      uses: ./.github/actions/cache-deps
       with:
         extension: ${{ inputs.cache-extension }}
 

.github/workflows/action.yml

@@ -8,6 +8,9 @@ on:
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.9.1
@@ -28,8 +31,8 @@ jobs:
       - name: Assert changed
         run: ./scripts/assert-changed-files-actions.sh "packages"
 
-      - name: Bootstrap packages
-        uses: ./.github/actions/cache-bootstrap
+      - name: Install dependencies
+        uses: ./.github/actions/cache-deps
         with:
           extension: unit-tests
 
@@ -41,6 +44,9 @@ jobs:
 
   integration-tests-api:
     runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
 
     services:
       redis:
@@ -65,6 +71,10 @@ jobs:
         ports:
           - 5432:5432
 
+    strategy:
+      matrix:
+        node: [0, 1, 2, 3]
+
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.9.1
@@ -82,35 +92,50 @@ jobs:
           node-version: "14"
           cache: "yarn"
 
-      - name: Bootstrap packages
-        uses: ./.github/actions/cache-bootstrap
+      - name: Install dependencies
+        uses: ./.github/actions/cache-deps
         with:
           extension: integration-tests
 
       - name: Build Packages
-        run: lerna run build
+        run: yarn build
 
       - name: Install dev cli
-        run: sudo npm i -g medusa-dev-cli
+        run: sudo npm i -g medusa-dev-cli@latest
 
       - name: Set path to medusa repo
         run: medusa-dev --set-path-to-repo `pwd`
 
-      - name: Force install API
-        run: medusa-dev --force-install
+      - name: Set npm registry
+        run: |
+          yarn config set npmRegistryServer http://localhost:4873
+          echo -e 'unsafeHttpWhitelist:\n - "localhost"' >> .yarnrc.yml
+
+      - name: Force install
+        run: medusa-dev --force-install --external-registry
         working-directory: integration-tests/api
 
       - name: Build integration tests
         run: yarn build
         working-directory: integration-tests/api
 
-      - name: Run API integration tests
-        run: yarn test:integration:api --maxWorkers=50% --silent=false
+      - name: Split tests
+        id: split-tests
+        run: echo "::set-output name=split::$(npx jest --listTests --json | jq -cM '[_nwise(length / 4 | ceil)]')"
+        working-directory: integration-tests/api
+
+      - name: Run integration tests
+        run: echo $SPLIT | jq '.[${{ matrix.node }}] | .[] | @text' | xargs yarn test --maxWorkers=50%
+        working-directory: integration-tests/api
         env:
           DB_PASSWORD: postgres
+          SPLIT: ${{ steps['split-tests'].outputs['split'] }}
 
   integration-tests-plugins:
     runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
 
     services:
       postgres:
@@ -143,13 +168,13 @@ jobs:
           node-version: "14"
           cache: "yarn"
 
-      - name: Bootstrap packages
-        uses: ./.github/actions/cache-bootstrap
+      - name: Install dependencies
+        uses: ./.github/actions/cache-deps
         with:
           extension: integration-tests
 
       - name: Build Packages
-        run: lerna run build
+        run: yarn build
 
       - name: Install dev cli
         run: sudo npm i -g medusa-dev-cli
@@ -157,8 +182,13 @@ jobs:
       - name: Set path to medusa repo
         run: medusa-dev --set-path-to-repo `pwd`
 
+      - name: Set npm registry
+        run: |
+          yarn config set npmRegistryServer http://localhost:4873
+          echo -e 'unsafeHttpWhitelist:\n - "localhost"' >> .yarnrc.yml
+
       - name: Force install
-        run: medusa-dev --force-install
+        run: medusa-dev --force-install --external-registry
         working-directory: integration-tests/plugins
 
       - name: Build plugin integration tests

.github/workflows/generate-reference.yml

@@ -1,8 +1,8 @@
 name: Generate References
-on: 
+on:
   push:
     branches:
-      - 'master'
+      - "master"
     paths:
       - packages/medusa/src/services/**
 jobs:
@@ -24,9 +24,9 @@ jobs:
         with:
           node-version: "14"
           cache: "yarn"
-        
-      - name: Bootstrap packages
-        uses: ./.github/actions/cache-bootstrap
+
+      - name: Install packages
+        uses: ./.github/actions/cache-deps
         with:
           extension: unit-tests
 
@@ -37,7 +37,7 @@ jobs:
         run: yarn generate:services
 
       - uses: stefanzweifel/git-auto-commit-action@v4
-        with: 
+        with:
           commit_message: Automatically Generated Services Reference
           file_pattern: docs/content/*
           skip_dirty_check: false

.github/workflows/snapshot-this.yml

@@ -82,8 +82,8 @@ jobs:
           node-version: 16.x
           cache: "yarn"
 
-      - name: Bootstrap packages
-        uses: ./.github/actions/cache-bootstrap
+      - name: Install dependencies
+        uses: ./.github/actions/cache-deps
         with:
           extension: snapshot-this
 

.github/workflows/test-cli-with-database.yml

@@ -50,7 +50,7 @@ jobs:
           cache-extension: "cli-test"
 
       - name: Install Medusa cli
-        run: yarn global add @medusajs/medusa-cli
+        run: npm i -g @medusajs/medusa-cli
 
       - name: Create Medusa project
         run: |