feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>

packages/modules/index/src/models/index-data.ts

@@ -1,25 +1,10 @@
 import { model } from "@medusajs/framework/utils"
 
-const IndexData = model
-  .define("IndexData", {
-    id: model.text().primaryKey(),
-    name: model.text().primaryKey(),
-    data: model.json().default({}),
-  })
-  .indexes([
-    {
-      name: "IDX_index_data_gin",
-      type: "GIN",
-      on: ["data"],
-    },
-    {
-      name: "IDX_index_data_id",
-      on: ["id"],
-    },
-    {
-      name: "IDX_index_data_name",
-      on: ["name"],
-    },
-  ])
+const IndexData = model.define("IndexData", {
+  id: model.text().primaryKey(),
+  name: model.text().primaryKey(),
+  data: model.json().default({}),
+  staled_at: model.dateTime().nullable(),
+})
 
 export default IndexData

packages/modules/index/src/models/index-metadata.ts

@@ -3,7 +3,7 @@ import { IndexMetadataStatus } from "../utils/index-metadata-status"
 
 const IndexMetadata = model
   .define("IndexMetadata", {
-    id: model.id().primaryKey(),
+    id: model.id({ prefix: "idxmeta" }).primaryKey(),
     entity: model.text(),
     fields: model.text(),
     fields_hash: model.text(),

packages/modules/index/src/models/index-relation.ts

@@ -4,9 +4,10 @@ const IndexRelation = model.define("IndexRelation", {
   id: model.autoincrement().primaryKey(),
   pivot: model.text(),
   parent_name: model.text(),
-  parent_id: model.text().index("IDX_index_relation_parent_id"),
+  parent_id: model.text(),
   child_name: model.text(),
-  child_id: model.text().index("IDX_index_relation_child_id"),
+  child_id: model.text(),
+  staled_at: model.dateTime().nullable(),
 })
 
 export default IndexRelation

packages/modules/index/src/models/index-sync.ts

@@ -0,0 +1,17 @@
+import { model } from "@medusajs/framework/utils"
+
+const IndexSync = model
+  .define("IndexSync", {
+    id: model.id({ prefix: "idxsync" }).primaryKey(),
+    entity: model.text(),
+    last_key: model.text().nullable(),
+  })
+  .indexes([
+    {
+      name: "IDX_index_sync_entity",
+      on: ["entity"],
+      unique: true,
+    },
+  ])
+
+export default IndexSync

packages/modules/index/src/models/index.ts

@@ -1,3 +1,4 @@
 export { default as IndexData } from "./index-data"
 export { default as IndexMetadata } from "./index-metadata"
 export { default as IndexRelation } from "./index-relation"
+export { default as IndexSync } from "./index-sync"