feat: shard modules and API integration tests (#6775)

what:

- runs module and api integration tests in matrix strategy
  - v28 of jest comes with sharding support, which we can utilize when we upgrade
- splits the tests by number of matrix jobs and run them in parallel
  - This introduced some flakiness in some specs, but couldn't reproduce locally. Those have been skipped for now.
- uses swc/jest for added performance
  - Locally, a chunk took 90 seconds with babel and 30 seconds with swc. 
  - This translated to 2 mins saved per shard in CI, but haven't tested this enough.

.github/workflows/action.yml

@@ -8,6 +8,11 @@ jobs:
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+    outputs:
+      module-matrix: ${{ steps.set-module-matrix.outputs.matrix }}
+      module-chunks: ${{ steps.set-module-chunks.outputs.chunks }}
+      api-matrix: ${{ steps.set-api-matrix.outputs.matrix }}
+      api-chunks: ${{ steps.set-api-chunks.outputs.chunks }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -28,6 +33,28 @@ jobs:
         with:
           extension: pipeline
 
+      - id: set-module-chunks
+        name: Set Module Chunks
+        working-directory: integration-tests/modules
+        run: echo "chunks=$(yarn run jest --listTests --json | jq -cM '[_nwise(length / 3 | ceil)]')" >> $GITHUB_OUTPUT
+
+      - id: set-module-matrix
+        name: Set Module Matrix
+        run: echo "matrix=$(echo $CHUNKS | jq -cM 'to_entries | map(.key)')" >> $GITHUB_OUTPUT
+        env:
+          CHUNKS: ${{ steps.set-module-chunks.outputs.chunks }}
+
+      - id: set-api-chunks
+        name: Set API Chunks
+        working-directory: integration-tests/api
+        run: echo "chunks=$(yarn run jest --listTests --json | jq -cM '[_nwise(length / 3 | ceil)]')" >> $GITHUB_OUTPUT
+
+      - id: set-api-matrix
+        name: Set API Matrix
+        run: echo "matrix=$(echo $CHUNKS | jq -cM 'to_entries | map(.key)')" >> $GITHUB_OUTPUT
+        env:
+          CHUNKS: ${{ steps.set-api-chunks.outputs.chunks }}
+
   unit-tests:
     needs: setup
     runs-on: ubuntu-latest
@@ -126,9 +153,14 @@ jobs:
           DB_PASSWORD: postgres
           DB_USERNAME: postgres
 
-  integration-tests-api:
+  integration-tests-api-matrix:
     needs: setup
+    name: Shard (${{ matrix.chunk }}) API Integration Tests
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: ${{ fromJSON(needs.setup.outputs.api-matrix) }}
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
@@ -181,11 +213,29 @@ jobs:
       - name: Build Packages
         run: yarn build
 
-      - name: Run integration tests
+      - name: Run API integration tests
         run: yarn test:integration:api
         env:
-          DB_PASSWORD: postgres
           DB_USERNAME: postgres
+          DB_PASSWORD: postgres
+          NODE_OPTIONS: "--max_old_space_size=4096"
+          CHUNK: ${{ matrix.chunk }}
+          CHUNKS: ${{ needs.setup.outputs.api-chunks }}
+
+  integration-tests-api:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: integration-tests-api-matrix
+    steps:
+      - run: exit 1
+        if: >-
+          ${{
+               contains(needs.integration-tests-api-matrix.result, 'failure')
+            || contains(needs.integration-tests-api-matrix.result, 'cancelled')
+            || contains(needs.integration-tests-api-matrix.result, 'skipped')
+          }}
+      - run: exit 0
+        if: ${{ contains(needs.integration-tests-api-matrix.result, 'success') }}
 
   integration-tests-plugins:
     needs: setup
@@ -240,8 +290,12 @@ jobs:
           DB_PASSWORD: postgres
           NODE_OPTIONS: "--max_old_space_size=4096"
 
-  integration-tests-modules:
+  integration-tests-modules-matrix:
     needs: setup
+    name: Shard (${{ matrix.chunk }}) Module Integration Tests
+    strategy:
+      matrix:
+        chunk: ${{ fromJSON(needs.setup.outputs.module-matrix) }}
     runs-on: ubuntu-latest
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
@@ -292,6 +346,23 @@ jobs:
           DB_USERNAME: postgres
           DB_PASSWORD: postgres
           NODE_OPTIONS: "--max_old_space_size=4096"
+          CHUNK: ${{ matrix.chunk }}
+          CHUNKS: ${{ needs.setup.outputs.module-chunks }}
+
+  integration-tests-modules:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: integration-tests-modules-matrix
+    steps:
+      - run: exit 1
+        if: >-
+          ${{
+               contains(needs.integration-tests-modules-matrix.result, 'failure')
+            || contains(needs.integration-tests-modules-matrix.result, 'cancelled')
+            || contains(needs.integration-tests-modules-matrix.result, 'skipped')
+          }}
+      - run: exit 0
+        if: ${{ contains(needs.integration-tests-modules-matrix.result, 'success') }}
 
   integration-tests-repositories:
     needs: setup