feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>

packages/modules/index/src/migrations/.snapshot-medusa-index.json

@@ -1,5 +1,7 @@
 {
-  "namespaces": ["public"],
+  "namespaces": [
+    "public"
+  ],
   "name": "public",
   "tables": [
     {
@@ -32,6 +34,16 @@
           "default": "'{}'",
           "mappedType": "json"
         },
+        "staled_at": {
+          "name": "staled_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "length": 6,
+          "mappedType": "datetime"
+        },
         "created_at": {
           "name": "created_at",
           "type": "timestamptz",
@@ -77,36 +89,12 @@
           "unique": false,
           "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_deleted_at\" ON \"index_data\" (deleted_at) WHERE deleted_at IS NULL"
         },
-        {
-          "keyName": "IDX_index_data_gin",
-          "columnNames": [],
-          "composite": false,
-          "constraint": false,
-          "primary": false,
-          "unique": false,
-          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_gin\" ON \"index_data\" USING GIN (data) WHERE deleted_at IS NULL"
-        },
-        {
-          "keyName": "IDX_index_data_id",
-          "columnNames": [],
-          "composite": false,
-          "constraint": false,
-          "primary": false,
-          "unique": false,
-          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_id\" ON \"index_data\" (id) WHERE deleted_at IS NULL"
-        },
-        {
-          "keyName": "IDX_index_data_name",
-          "columnNames": [],
-          "composite": false,
-          "constraint": false,
-          "primary": false,
-          "unique": false,
-          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_name\" ON \"index_data\" (name) WHERE deleted_at IS NULL"
-        },
         {
           "keyName": "index_data_pkey",
-          "columnNames": ["id", "name"],
+          "columnNames": [
+            "id",
+            "name"
+          ],
           "composite": true,
           "constraint": true,
           "primary": true,
@@ -163,7 +151,12 @@
           "primary": false,
           "nullable": false,
           "default": "'pending'",
-          "enumItems": ["pending", "processing", "done", "error"],
+          "enumItems": [
+            "pending",
+            "processing",
+            "done",
+            "error"
+          ],
           "mappedType": "enum"
         },
         "created_at": {
@@ -222,7 +215,9 @@
         },
         {
           "keyName": "index_metadata_pkey",
-          "columnNames": ["id"],
+          "columnNames": [
+            "id"
+          ],
           "composite": false,
           "constraint": true,
           "primary": true,
@@ -289,6 +284,16 @@
           "nullable": false,
           "mappedType": "text"
         },
+        "staled_at": {
+          "name": "staled_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "length": 6,
+          "mappedType": "datetime"
+        },
         "created_at": {
           "name": "created_at",
           "type": "timestamptz",
@@ -325,24 +330,6 @@
       "name": "index_relation",
       "schema": "public",
       "indexes": [
-        {
-          "keyName": "IDX_index_relation_parent_id",
-          "columnNames": [],
-          "composite": false,
-          "constraint": false,
-          "primary": false,
-          "unique": false,
-          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_relation_parent_id\" ON \"index_relation\" (parent_id) WHERE deleted_at IS NULL"
-        },
-        {
-          "keyName": "IDX_index_relation_child_id",
-          "columnNames": [],
-          "composite": false,
-          "constraint": false,
-          "primary": false,
-          "unique": false,
-          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_relation_child_id\" ON \"index_relation\" (child_id) WHERE deleted_at IS NULL"
-        },
         {
           "keyName": "IDX_index_relation_deleted_at",
           "columnNames": [],
@@ -354,7 +341,107 @@
         },
         {
           "keyName": "index_relation_pkey",
-          "columnNames": ["id"],
+          "columnNames": [
+            "id"
+          ],
+          "composite": false,
+          "constraint": true,
+          "primary": true,
+          "unique": true
+        }
+      ],
+      "checks": [],
+      "foreignKeys": {},
+      "nativeEnums": {}
+    },
+    {
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "text"
+        },
+        "entity": {
+          "name": "entity",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "text"
+        },
+        "last_key": {
+          "name": "last_key",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "text"
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "length": 6,
+          "default": "now()",
+          "mappedType": "datetime"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "length": 6,
+          "default": "now()",
+          "mappedType": "datetime"
+        },
+        "deleted_at": {
+          "name": "deleted_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "length": 6,
+          "mappedType": "datetime"
+        }
+      },
+      "name": "index_sync",
+      "schema": "public",
+      "indexes": [
+        {
+          "keyName": "IDX_index_sync_deleted_at",
+          "columnNames": [],
+          "composite": false,
+          "constraint": false,
+          "primary": false,
+          "unique": false,
+          "expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_sync_deleted_at\" ON \"index_sync\" (deleted_at) WHERE deleted_at IS NULL"
+        },
+        {
+          "keyName": "IDX_index_sync_entity",
+          "columnNames": [],
+          "composite": false,
+          "constraint": false,
+          "primary": false,
+          "unique": false,
+          "expression": "CREATE UNIQUE INDEX IF NOT EXISTS \"IDX_index_sync_entity\" ON \"index_sync\" (entity) WHERE deleted_at IS NULL"
+        },
+        {
+          "keyName": "index_sync_pkey",
+          "columnNames": [
+            "id"
+          ],
           "composite": false,
           "constraint": true,
           "primary": true,

packages/modules/index/src/migrations/Migration20231019174230.ts

@@ -5,17 +5,9 @@ export class Migration20231019174230 extends Migration {
     this.addSql(
       `create table "index_data" ("id" text not null, "name" text not null, "data" jsonb not null default '{}', constraint "index_data_pkey" primary key ("id", "name")) PARTITION BY LIST("name");`
     )
-    this.addSql(`create index "IDX_index_data_id" on "index_data" ("id");`)
-    this.addSql(`create index "IDX_index_data_name" on "index_data" ("name");`)
-    this.addSql(
-      `create index "IDX_index_data_gin" on "index_data" using GIN ("data");`
-    )
 
     this.addSql(
       `create table "index_relation" ("id" bigserial, "pivot" text not null, "parent_id" text not null, "parent_name" text not null, "child_id" text not null, "child_name" text not null, constraint "index_relation_pkey" primary key ("id", "pivot")) PARTITION BY LIST("pivot");`
     )
-    this.addSql(
-      `create index "IDX_index_relation_child_id" on "index_relation" ("child_id");`
-    )
   }
 }

packages/modules/index/src/migrations/Migration20250127144442.ts

@@ -0,0 +1,23 @@
+import { Migration } from "@mikro-orm/migrations"
+
+export class Migration20250127144442 extends Migration {
+  override async up(): Promise<void> {
+    this.addSql(
+      `alter table if exists "index_data" add column if not exists "staled_at" timestamptz null;`
+    )
+
+    this.addSql(
+      `alter table if exists "index_relation" add column if not exists "staled_at" timestamptz null;`
+    )
+  }
+
+  override async down(): Promise<void> {
+    this.addSql(
+      `alter table if exists "index_data" drop column if exists "staled_at";`
+    )
+
+    this.addSql(
+      `alter table if exists "index_relation" drop column if exists "staled_at";`
+    )
+  }
+}

packages/modules/index/src/migrations/Migration20250128132404.ts

@@ -0,0 +1,19 @@
+import { Migration } from "@mikro-orm/migrations"
+
+export class Migration20250128132404 extends Migration {
+  override async up(): Promise<void> {
+    this.addSql(
+      `create table if not exists "index_sync" ("id" text not null, "entity" text not null, "last_key" text null, "created_at" timestamptz not null default now(), "updated_at" timestamptz not null default now(), "deleted_at" timestamptz null, constraint "index_sync_pkey" primary key ("id"));`
+    )
+    this.addSql(
+      `CREATE INDEX IF NOT EXISTS "IDX_index_sync_deleted_at" ON "index_sync" (deleted_at) WHERE deleted_at IS NULL;`
+    )
+    this.addSql(
+      `CREATE UNIQUE INDEX IF NOT EXISTS "IDX_index_sync_entity" ON "index_sync" (entity) WHERE deleted_at IS NULL;`
+    )
+  }
+
+  override async down(): Promise<void> {
+    this.addSql(`drop table if exists "index_sync" cascade;`)
+  }
+}