chore(): Default caching configuration and gracefull redis error handling (#13663)

* chore(): Default caching configuration and gracefull redis error handling

* Create odd-moons-crash.md

* chore(): Default caching configuration and gracefull redis error handling

* fixes

* address feedback

* revert(): Test utils imit module fix

* reconnect

* reconnect

* reconnect
This commit is contained in:
Adrien de Peretti authored and GitHub committed 2025-10-06 17:57:11 +02:00
1 parent 28d57b7bf8
commit 51859c38a7
9 files changed
+565 -328

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
---
"@medusajs/caching-redis": patch
"@medusajs/utils": patch
---
chore(): Default caching configuration and gracefull redis error handling
@@ -1048,6 +1048,7 @@ describe("defineConfig", function () {
process.env.EXECUTION_CONTEXT = "medusa-cloud" process.env.EXECUTION_CONTEXT = "medusa-cloud"
process.env.REDIS_URL = "redis://localhost:6379" process.env.REDIS_URL = "redis://localhost:6379"
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test" process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
process.env.S3_PREFIX = "test" process.env.S3_PREFIX = "test"
process.env.S3_REGION = "us-east-1" process.env.S3_REGION = "us-east-1"
@@ -1086,6 +1087,21 @@ describe("defineConfig", function () {
}, },
"resolve": "@medusajs/medusa/cache-redis", "resolve": "@medusajs/medusa/cache-redis",
}, },
"caching": {
"options": {
"providers": [
{
"id": "caching-redis",
"is_default": true,
"options": {
"redisUrl": "redis://localhost:6379",
},
"resolve": "@medusajs/medusa/caching-redis",
},
],
},
"resolve": "@medusajs/caching",
},
"cart": { "cart": {
"resolve": "@medusajs/medusa/cart", "resolve": "@medusajs/medusa/cart",
}, },
@@ -1255,6 +1271,7 @@ describe("defineConfig", function () {
process.env.EXECUTION_CONTEXT = "medusa-cloud" process.env.EXECUTION_CONTEXT = "medusa-cloud"
process.env.REDIS_URL = "redis://localhost:6379" process.env.REDIS_URL = "redis://localhost:6379"
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test" process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
process.env.S3_PREFIX = "test" process.env.S3_PREFIX = "test"
process.env.S3_REGION = "us-east-1" process.env.S3_REGION = "us-east-1"
@@ -1294,6 +1311,21 @@ describe("defineConfig", function () {
}, },
"resolve": "@medusajs/medusa/cache-redis", "resolve": "@medusajs/medusa/cache-redis",
}, },
"caching": {
"options": {
"providers": [
{
"id": "caching-redis",
"is_default": true,
"options": {
"redisUrl": "redis://localhost:6379",
},
"resolve": "@medusajs/medusa/caching-redis",
},
],
},
"resolve": "@medusajs/caching",
},
"cart": { "cart": {
"resolve": "@medusajs/medusa/cart", "resolve": "@medusajs/medusa/cart",
}, },
@@ -1473,6 +1505,7 @@ describe("defineConfig", function () {
process.env.EXECUTION_CONTEXT = "medusa-cloud" process.env.EXECUTION_CONTEXT = "medusa-cloud"
process.env.REDIS_URL = "redis://localhost:6379" process.env.REDIS_URL = "redis://localhost:6379"
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test" process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
process.env.S3_PREFIX = "test" process.env.S3_PREFIX = "test"
process.env.S3_REGION = "us-east-1" process.env.S3_REGION = "us-east-1"
@@ -1518,6 +1551,21 @@ describe("defineConfig", function () {
}, },
"resolve": "@medusajs/medusa/cache-redis", "resolve": "@medusajs/medusa/cache-redis",
}, },
"caching": {
"options": {
"providers": [
{
"id": "caching-redis",
"is_default": true,
"options": {
"redisUrl": "redis://localhost:6379",
},
"resolve": "@medusajs/medusa/caching-redis",
},
],
},
"resolve": "@medusajs/caching",
},
"cart": { "cart": {
"resolve": "@medusajs/medusa/cart", "resolve": "@medusajs/medusa/cart",
}, },
@@ -303,6 +303,24 @@ function resolveModules(
}, },
] ]
if (process.env.CACHE_REDIS_URL) {
cloudModules.push({
resolve: MODULE_PACKAGE_NAMES[Modules.CACHING],
options: {
providers: [
{
id: "caching-redis",
resolve: "@medusajs/medusa/caching-redis",
is_default: true,
options: {
redisUrl: process.env.CACHE_REDIS_URL,
},
},
],
},
})
}
/** /**
* The default set of modules to always use. The end user can swap * The default set of modules to always use. The end user can swap
* the modules by providing an alternate implementation via their * the modules by providing an alternate implementation via their
@@ -64,12 +64,12 @@ export async function initModules({
async function shutdown() { async function shutdown() {
const promises: Promise<void>[] = [] const promises: Promise<void>[] = []
promises.push(medusaApp.onApplicationPrepareShutdown())
promises.push(medusaApp.onApplicationShutdown())
if (shouldDestroyConnectionAutomatically) { if (shouldDestroyConnectionAutomatically) {
promises.push((sharedPgConnection as any).context?.destroy()) promises.push((sharedPgConnection as any).context?.destroy())
promises.push((sharedPgConnection as any).destroy()) promises.push((sharedPgConnection as any).destroy())
promises.push(medusaApp.onApplicationPrepareShutdown())
promises.push(medusaApp.onApplicationShutdown())
} else { } else {
if (!preventConnectionDestroyWarning) { if (!preventConnectionDestroyWarning) {
logger.info( logger.info(
@@ -4,7 +4,7 @@ import type {
ModulesSdkTypes, ModulesSdkTypes,
} from "@medusajs/framework/types" } from "@medusajs/framework/types"
import { RedisCacheModuleOptions } from "@types" import { RedisCacheModuleOptions } from "@types"
import Redis from "ioredis" import Redis, { type RedisOptions } from "ioredis"
export default async ( export default async (
{ {
@@ -23,36 +23,56 @@ export default async (
const moduleOptions = (options ?? const moduleOptions = (options ??
moduleDeclaration?.options ?? moduleDeclaration?.options ??
{}) as RedisCacheModuleOptions & { {}) as RedisCacheModuleOptions
redisUrl?: string
}
if (!moduleOptions.redisUrl) { const { redisUrl, ...redisOptions_ } = moduleOptions
if (!redisUrl) {
throw new Error("[caching-redis] redisUrl is required") throw new Error("[caching-redis] redisUrl is required")
} }
let redisClient: Redis let redisClient: Redis
try { const redisOptions: RedisOptions = {
redisClient = new Redis(moduleOptions.redisUrl!, { connectTimeout: 10000,
connectTimeout: 10000, commandTimeout: 5000,
lazyConnect: true, lazyConnect: true,
retryDelayOnFailover: 100, maxRetriesPerRequest: 3,
connectionName: "medusa-cache-redis", enableOfflineQueue: true,
...moduleOptions, connectionName: "medusa-cache-redis",
}) ...redisOptions_,
}
// Test connection redisClient = new Redis(redisUrl!, redisOptions)
await redisClient.ping()
// Handle connection errors gracefully
redisClient.on("error", (error) => {
logger_.warn(`Redis cache connection error: ${error.message}`)
})
redisClient.on("connect", () => {
logger_.info("Redis cache connection established successfully") logger_.info("Redis cache connection established successfully")
})
redisClient.on("ready", () => {
logger_.info("Redis cache is ready to accept commands")
})
redisClient.on("close", () => {
logger_.warn("Redis cache connection closed")
})
redisClient.on("reconnecting", () => {
logger_.info("Redis cache attempting to reconnect...")
})
try {
// Test connection with timeout
await redisClient.ping()
logger_.info("Redis cache connection test successful")
} catch (error) { } catch (error) {
logger_.error(`Failed to connect to Redis cache: ${error.message}`) logger_.warn(
redisClient = new Redis(moduleOptions.redisUrl!, { `Redis cache connection test failed: ${error.message}, but continuing with lazy connection`
connectTimeout: 10000, )
lazyConnect: true,
retryDelayOnFailover: 100,
...moduleOptions,
})
} }
container.register({ container.register({
@@ -62,5 +82,8 @@ export default async (
prefix: { prefix: {
resolve: () => moduleOptions.prefix ?? "mc:", resolve: () => moduleOptions.prefix ?? "mc:",
}, },
logger: {
resolve: () => logger_,
},
}) })
} }
@@ -1,3 +1,4 @@
import { Logger } from "@medusajs/framework/types"
import { RedisCacheModuleOptions } from "@types" import { RedisCacheModuleOptions } from "@types"
import { Redis } from "ioredis" import { Redis } from "ioredis"
import { createGunzip, createGzip } from "zlib" import { createGunzip, createGzip } from "zlib"
@@ -10,13 +11,20 @@ export class RedisCachingProvider {
protected defaultTTL: number protected defaultTTL: number
protected compressionThreshold: number protected compressionThreshold: number
protected hasher: (key: string) => string protected hasher: (key: string) => string
protected logger: Logger
constructor( constructor(
{ {
redisClient, redisClient,
logger,
prefix, prefix,
hasher, hasher,
}: { redisClient: Redis; prefix: string; hasher: (key: string) => string }, }: {
redisClient: Redis
prefix: string
hasher: (key: string) => string
logger: Logger
},
options?: RedisCacheModuleOptions options?: RedisCacheModuleOptions
) { ) {
this.redisClient = redisClient this.redisClient = redisClient
@@ -24,6 +32,29 @@ export class RedisCachingProvider {
this.defaultTTL = options?.ttl ?? 3600 // 1 hour default this.defaultTTL = options?.ttl ?? 3600 // 1 hour default
this.compressionThreshold = options?.compressionThreshold ?? 2048 // 2KB default this.compressionThreshold = options?.compressionThreshold ?? 2048 // 2KB default
this.hasher = hasher this.hasher = hasher
this.logger = logger
}
private isConnectionError(error: any): boolean {
return (
error.code === "ECONNREFUSED" ||
error.code === "ENOTFOUND" ||
error.code === "ETIMEDOUT" ||
error.code === "ECONNRESET" ||
error.code === "EPIPE" ||
error.message?.includes("Connection is closed") ||
error.message?.includes("connect ECONNREFUSED") ||
error.message?.includes("connect ETIMEDOUT") ||
error.message?.includes("Command timed out") ||
error.message?.includes("Maximum number of retries exceeded") ||
["connecting", "reconnecting", "disconnecting", "wait", "end"].includes(
this.redisClient.status
)
)
}
private isConnectionHealthy(): boolean {
return this.redisClient.status === "ready"
} }
#getKeyName(key: string): string { #getKeyName(key: string): string {
@@ -252,66 +283,92 @@ export class RedisCachingProvider {
} }
async get({ key, tags }: { key?: string; tags?: string[] }): Promise<any> { async get({ key, tags }: { key?: string; tags?: string[] }): Promise<any> {
if (key) { if (!this.isConnectionHealthy()) {
const keyName = this.#getKeyName(key) return null
const buffer = await this.redisClient.hgetBuffer(keyName, "data") }
if (!buffer) {
return null
}
const finalData = await this.#decompressData(buffer) if (key) {
return JSON.parse(finalData) try {
const keyName = this.#getKeyName(key)
const buffer = await this.redisClient.hgetBuffer(keyName, "data")
if (!buffer) {
return null
}
const finalData = await this.#decompressData(buffer)
return JSON.parse(finalData)
} catch (error) {
if (this.isConnectionError(error)) {
this.logger.warn(
"Redis connection error during get operation, returning null to trigger fallback to original data source"
)
return null
}
throw error
}
} }
if (tags?.length) { if (tags?.length) {
// Get all keys associated with the tags try {
const pipeline = this.redisClient.pipeline() // Get all keys associated with the tags
tags.forEach((tag) => { const pipeline = this.redisClient.pipeline()
const tagKey = this.#getTagKey(tag) tags.forEach((tag) => {
pipeline.smembers(tagKey) const tagKey = this.#getTagKey(tag)
}) pipeline.smembers(tagKey)
})
const tagResults = await pipeline.exec() const tagResults = await pipeline.exec()
const allKeys = new Set<string>() const allKeys = new Set<string>()
tagResults?.forEach((result, index) => { tagResults?.forEach((result, index) => {
if (result && result[1]) { if (result && result[1]) {
;(result[1] as string[]).forEach((key) => allKeys.add(key)) ;(result[1] as string[]).forEach((key) => allKeys.add(key))
}
})
if (allKeys.size === 0) {
return []
} }
})
if (allKeys.size === 0) { // Get all hash data for the keys
return [] const valuePipeline = this.redisClient.pipeline()
} Array.from(allKeys).forEach((key) => {
valuePipeline.hgetBuffer(key, "data")
})
// Get all hash data for the keys const valueResults = await valuePipeline.exec()
const valuePipeline = this.redisClient.pipeline() const results: any[] = []
Array.from(allKeys).forEach((key) => {
valuePipeline.hgetBuffer(key, "data")
})
const valueResults = await valuePipeline.exec() const decompressionPromises = (valueResults || []).map(
const results: any[] = [] async (result) => {
if (result && result[1]) {
const decompressionPromises = (valueResults || []).map(async (result) => { const buffer = result[1] as Buffer
if (result && result[1]) { try {
const buffer = result[1] as Buffer const finalData = await this.#decompressData(buffer)
try { return JSON.parse(finalData)
const finalData = await this.#decompressData(buffer) } catch (e) {
return JSON.parse(finalData) // If JSON parsing fails, skip this entry (corrupted data)
} catch (e) { this.logger.warn(`Skipping corrupted cache entry: ${e.message}`)
// If JSON parsing fails, skip this entry (corrupted data) return null
console.warn(`Skipping corrupted cache entry: ${e.message}`) }
}
return null return null
} }
)
const decompressionResults = await Promise.all(decompressionPromises)
results.push(...decompressionResults.filter(Boolean))
return results
} catch (error) {
if (this.isConnectionError(error)) {
this.logger.warn(
"Redis connection error during get operation, returning empty array to trigger fallback to original data source"
)
return null
} }
return null throw error
}) }
const decompressionResults = await Promise.all(decompressionPromises)
results.push(...decompressionResults.filter(Boolean))
return results
} }
return null return null
@@ -332,53 +389,63 @@ export class RedisCachingProvider {
autoInvalidate?: boolean autoInvalidate?: boolean
} }
}): Promise<void> { }): Promise<void> {
const keyName = this.#getKeyName(key) try {
const serializedData = JSON.stringify(data) const keyName = this.#getKeyName(key)
const effectiveTTL = ttl ?? this.defaultTTL const serializedData = JSON.stringify(data)
const effectiveTTL = ttl ?? this.defaultTTL
const finalData = await this.#compressData(serializedData) const finalData = await this.#compressData(serializedData)
let tagIds: number[] = [] let tagIds: number[] = []
if (tags?.length) { if (tags?.length) {
tagIds = await this.#internTags(tags) tagIds = await this.#internTags(tags)
}
const setPipeline = this.redisClient.pipeline()
// Main data with conditional operations
setPipeline.hsetnx(keyName, "data", finalData)
if (options && Object.keys(options).length) {
setPipeline.hset(keyName, "options", JSON.stringify(options))
}
if (effectiveTTL) {
setPipeline.expire(keyName, effectiveTTL)
}
// Store tag IDs if present
if (tags?.length && tagIds.length) {
const tagsKey = this.#getTagsKey(key)
const buffer = Buffer.alloc(tagIds.length * 4)
tagIds.forEach((id, index) => {
buffer.writeUInt32LE(id, index * 4)
})
if (effectiveTTL) {
setPipeline.set(tagsKey, buffer, "EX", effectiveTTL + 60, "NX")
} else {
setPipeline.setnx(tagsKey, buffer)
} }
// Add tag operations to the same pipeline const setPipeline = this.redisClient.pipeline()
tags.forEach((tag) => {
const tagKey = this.#getTagKey(tag)
setPipeline.sadd(tagKey, keyName)
if (effectiveTTL) {
setPipeline.expire(tagKey, effectiveTTL + 60)
}
})
}
await setPipeline.exec() // Main data with conditional operations
setPipeline.hsetnx(keyName, "data", finalData)
if (options && Object.keys(options).length) {
setPipeline.hset(keyName, "options", JSON.stringify(options))
}
if (effectiveTTL) {
setPipeline.expire(keyName, effectiveTTL)
}
// Store tag IDs if present
if (tags?.length && tagIds.length) {
const tagsKey = this.#getTagsKey(key)
const buffer = Buffer.alloc(tagIds.length * 4)
tagIds.forEach((id, index) => {
buffer.writeUInt32LE(id, index * 4)
})
if (effectiveTTL) {
setPipeline.set(tagsKey, buffer, "EX", effectiveTTL + 60, "NX")
} else {
setPipeline.setnx(tagsKey, buffer)
}
// Add tag operations to the same pipeline
tags.forEach((tag) => {
const tagKey = this.#getTagKey(tag)
setPipeline.sadd(tagKey, keyName)
if (effectiveTTL) {
setPipeline.expire(tagKey, effectiveTTL + 60)
}
})
}
await setPipeline.exec()
} catch (error) {
if (this.isConnectionError(error)) {
this.logger.warn(
"Redis connection error during set operation, relying on IORedis retry mechanism"
)
return
}
throw error
}
} }
async clear({ async clear({
@@ -392,208 +459,96 @@ export class RedisCachingProvider {
autoInvalidate?: boolean autoInvalidate?: boolean
} }
}): Promise<void> { }): Promise<void> {
if (key) { try {
const keyName = this.#getKeyName(key) if (key) {
const tagsKey = this.#getTagsKey(key) const keyName = this.#getKeyName(key)
const tagsKey = this.#getTagsKey(key)
const clearPipeline = this.redisClient.pipeline() const clearPipeline = this.redisClient.pipeline()
// Get tags for cleanup and delete main key in same pipeline // Get tags for cleanup and delete main key in same pipeline
clearPipeline.getBuffer(tagsKey) clearPipeline.getBuffer(tagsKey)
clearPipeline.unlink(keyName) clearPipeline.unlink(keyName)
const results = await clearPipeline.exec() const results = await clearPipeline.exec()
const tagsBuffer = results?.[0]?.[1] as Buffer const tagsBuffer = results?.[0]?.[1] as Buffer
if (tagsBuffer?.length) { if (tagsBuffer?.length) {
try { try {
// Binary format: array of 32-bit integers // Binary format: array of 32-bit integers
const tagIds: number[] = [] const tagIds: number[] = []
for (let i = 0; i < tagsBuffer.length; i += 4) { for (let i = 0; i < tagsBuffer.length; i += 4) {
tagIds.push(tagsBuffer.readUInt32LE(i)) tagIds.push(tagsBuffer.readUInt32LE(i))
}
if (tagIds.length) {
const entryTags = await this.#resolveTagIds(tagIds)
const tagCleanupPipeline = this.redisClient.pipeline()
entryTags.forEach((tag) => {
const tagKey = this.#getTagKey(tag, { isHashed: true })
tagCleanupPipeline.srem(tagKey, keyName)
})
tagCleanupPipeline.unlink(tagsKey)
await tagCleanupPipeline.exec()
// Decrement reference counts and cleanup unused tags
await this.#decrementTagRefs(tagIds)
}
} catch (e) {
// noop - corrupted tag data, skip cleanup
} }
if (tagIds.length) {
const entryTags = await this.#resolveTagIds(tagIds)
const tagCleanupPipeline = this.redisClient.pipeline()
entryTags.forEach((tag) => {
const tagKey = this.#getTagKey(tag, { isHashed: true })
tagCleanupPipeline.srem(tagKey, keyName)
})
tagCleanupPipeline.unlink(tagsKey)
await tagCleanupPipeline.exec()
// Decrement reference counts and cleanup unused tags
await this.#decrementTagRefs(tagIds)
}
} catch (e) {
// noop - corrupted tag data, skip cleanup
} }
}
return
}
if (tags?.length) {
// Handle wildcard tag to clear all cache data
if (tags.includes("*")) {
await this.flush()
return return
} }
// Get all keys associated with the tags if (tags?.length) {
const pipeline = this.redisClient.pipeline() // Handle wildcard tag to clear all cache data
tags.forEach((tag) => { if (tags.includes("*")) {
const tagKey = this.#getTagKey(tag) await this.flush()
pipeline.smembers(tagKey)
})
const tagResults = await pipeline.exec()
const allKeys = new Set<string>()
tagResults?.forEach((result) => {
if (result && result[1]) {
;(result[1] as string[]).forEach((key) => allKeys.add(key))
}
})
if (allKeys.size) {
// If no options provided (user explicit call), clear everything
if (!options) {
const deletePipeline = this.redisClient.pipeline()
// Delete main keys and options
Array.from(allKeys).forEach((key) => {
deletePipeline.unlink(key)
})
// Clean up tag references for each key
const tagDataPromises = Array.from(allKeys).map(async (key) => {
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
const tagsData = await this.redisClient.getBuffer(tagsKey)
return { key, tagsKey, tagsData }
})
const tagResults = await Promise.all(tagDataPromises)
// Build single pipeline for all tag cleanup operations
const tagCleanupPipeline = this.redisClient.pipeline()
const cleanupPromises = tagResults.map(
async ({ key, tagsKey, tagsData }) => {
if (tagsData) {
try {
// Binary format: array of 32-bit integers
const tagIds: number[] = []
for (let i = 0; i < tagsData.length; i += 4) {
tagIds.push(tagsData.readUInt32LE(i))
}
if (tagIds.length) {
const entryTags = await this.#resolveTagIds(tagIds)
entryTags.forEach((tag) => {
const tagKey = this.#getTagKey(tag, { isHashed: true })
tagCleanupPipeline.srem(tagKey, key)
})
tagCleanupPipeline.unlink(tagsKey)
// Decrement reference counts and cleanup unused tags
await this.#decrementTagRefs(tagIds)
}
} catch (e) {
// noop
}
}
}
)
await Promise.all(cleanupPromises)
await tagCleanupPipeline.exec()
await deletePipeline.exec()
// Clean up empty tag sets
const allTagKeys = await this.redisClient.keys(
`${this.keyNamePrefix}tag:*`
)
if (allTagKeys.length) {
const cardinalityPipeline = this.redisClient.pipeline()
allTagKeys.forEach((tagKey) => {
cardinalityPipeline.scard(tagKey)
})
const cardinalityResults = await cardinalityPipeline.exec()
// Delete empty tag keys
const emptyTagPipeline = this.redisClient.pipeline()
cardinalityResults?.forEach((result, index) => {
if (result && result[1] === 0) {
emptyTagPipeline.unlink(allTagKeys[index])
}
})
await emptyTagPipeline.exec()
}
return return
} }
// If autoInvalidate is true (strategy call), only clear entries with autoInvalidate=true (default) // Get all keys associated with the tags
if (options.autoInvalidate === true) { const pipeline = this.redisClient.pipeline()
const optionsPipeline = this.redisClient.pipeline() tags.forEach((tag) => {
const tagKey = this.#getTagKey(tag)
pipeline.smembers(tagKey)
})
Array.from(allKeys).forEach((key) => { const tagResults = await pipeline.exec()
optionsPipeline.hget(key, "options")
})
const optionsResults = await optionsPipeline.exec() const allKeys = new Set<string>()
const keysToDelete: string[] = []
Array.from(allKeys).forEach((key, index) => { tagResults?.forEach((result) => {
const optionsResult = optionsResults?.[index] if (result && result[1]) {
;(result[1] as string[]).forEach((key) => allKeys.add(key))
}
})
if (optionsResult && optionsResult[1]) { if (allKeys.size) {
try { // If no options provided (user explicit call), clear everything
const entryOptions = JSON.parse(optionsResult[1] as string) if (!options) {
// Delete if entry has autoInvalidate=true or no setting (default true)
const shouldAutoInvalidate = entryOptions.autoInvalidate ?? true
if (shouldAutoInvalidate) {
keysToDelete.push(key)
}
} catch (e) {
// If can't parse options, assume it's safe to delete (default true)
keysToDelete.push(key)
}
} else {
// No options stored, default to true
keysToDelete.push(key)
}
})
if (keysToDelete.length) {
const deletePipeline = this.redisClient.pipeline() const deletePipeline = this.redisClient.pipeline()
keysToDelete.forEach((key) => { // Delete main keys and options
Array.from(allKeys).forEach((key) => {
deletePipeline.unlink(key) deletePipeline.unlink(key)
}) })
// Clean up tag references for each key to delete // Clean up tag references for each key
const tagDataPromises = keysToDelete.map(async (key) => { const tagDataPromises = Array.from(allKeys).map(async (key) => {
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "") const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
const tagsKey = this.#getTagsKey(keyWithoutPrefix) const tagsKey = this.#getTagsKey(keyWithoutPrefix)
const tagsData = await this.redisClient.getBuffer(tagsKey) const tagsData = await this.redisClient.getBuffer(tagsKey)
return { key, tagsKey, tagsData } return { key, tagsKey, tagsData }
}) })
// Wait for all tag data fetches
const tagResults = await Promise.all(tagDataPromises) const tagResults = await Promise.all(tagDataPromises)
// Build single pipeline for all tag cleanup operations // Build single pipeline for all tag cleanup operations
const tagCleanupPipeline = this.redisClient.pipeline() const tagCleanupPipeline = this.redisClient.pipeline()
const cleanupPromises = tagResults.map( const cleanupPromises = tagResults.map(
async ({ key, tagsKey, tagsData }) => { async ({ key, tagsKey, tagsData }) => {
if (tagsData) { if (tagsData) {
@@ -610,7 +565,7 @@ export class RedisCachingProvider {
const tagKey = this.#getTagKey(tag, { isHashed: true }) const tagKey = this.#getTagKey(tag, { isHashed: true })
tagCleanupPipeline.srem(tagKey, key) tagCleanupPipeline.srem(tagKey, key)
}) })
tagCleanupPipeline.unlink(tagsKey) // Delete the tags key tagCleanupPipeline.unlink(tagsKey)
// Decrement reference counts and cleanup unused tags // Decrement reference counts and cleanup unused tags
await this.#decrementTagRefs(tagIds) await this.#decrementTagRefs(tagIds)
@@ -624,7 +579,6 @@ export class RedisCachingProvider {
await Promise.all(cleanupPromises) await Promise.all(cleanupPromises)
await tagCleanupPipeline.exec() await tagCleanupPipeline.exec()
await deletePipeline.exec() await deletePipeline.exec()
// Clean up empty tag sets // Clean up empty tag sets
@@ -632,52 +586,188 @@ export class RedisCachingProvider {
`${this.keyNamePrefix}tag:*` `${this.keyNamePrefix}tag:*`
) )
if (allTagKeys.length) { if (allTagKeys.length) {
const cleanupPipeline = this.redisClient.pipeline() const cardinalityPipeline = this.redisClient.pipeline()
allTagKeys.forEach((tagKey) => { allTagKeys.forEach((tagKey) => {
cleanupPipeline.scard(tagKey) cardinalityPipeline.scard(tagKey)
}) })
const cardinalityResults = await cleanupPipeline.exec() const cardinalityResults = await cardinalityPipeline.exec()
// Delete tag keys that are now empty // Delete empty tag keys
const emptyTagDeletePipeline = this.redisClient.pipeline() const emptyTagPipeline = this.redisClient.pipeline()
cardinalityResults?.forEach((result, index) => { cardinalityResults?.forEach((result, index) => {
if (result && result[1] === 0) { if (result && result[1] === 0) {
emptyTagDeletePipeline.unlink(allTagKeys[index]) emptyTagPipeline.unlink(allTagKeys[index])
} }
}) })
await emptyTagDeletePipeline.exec() await emptyTagPipeline.exec()
} }
return return
} }
// If autoInvalidate is true (strategy call), only clear entries with autoInvalidate=true (default)
if (options.autoInvalidate === true) {
const optionsPipeline = this.redisClient.pipeline()
Array.from(allKeys).forEach((key) => {
optionsPipeline.hget(key, "options")
})
const optionsResults = await optionsPipeline.exec()
const keysToDelete: string[] = []
Array.from(allKeys).forEach((key, index) => {
const optionsResult = optionsResults?.[index]
if (optionsResult && optionsResult[1]) {
try {
const entryOptions = JSON.parse(optionsResult[1] as string)
// Delete if entry has autoInvalidate=true or no setting (default true)
const shouldAutoInvalidate =
entryOptions.autoInvalidate ?? true
if (shouldAutoInvalidate) {
keysToDelete.push(key)
}
} catch (e) {
// If can't parse options, assume it's safe to delete (default true)
keysToDelete.push(key)
}
} else {
// No options stored, default to true
keysToDelete.push(key)
}
})
if (keysToDelete.length) {
const deletePipeline = this.redisClient.pipeline()
keysToDelete.forEach((key) => {
deletePipeline.unlink(key)
})
// Clean up tag references for each key to delete
const tagDataPromises = keysToDelete.map(async (key) => {
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
const tagsData = await this.redisClient.getBuffer(tagsKey)
return { key, tagsKey, tagsData }
})
// Wait for all tag data fetches
const tagResults = await Promise.all(tagDataPromises)
// Build single pipeline for all tag cleanup operations
const tagCleanupPipeline = this.redisClient.pipeline()
const cleanupPromises = tagResults.map(
async ({ key, tagsKey, tagsData }) => {
if (tagsData) {
try {
// Binary format: array of 32-bit integers
const tagIds: number[] = []
for (let i = 0; i < tagsData.length; i += 4) {
tagIds.push(tagsData.readUInt32LE(i))
}
if (tagIds.length) {
const entryTags = await this.#resolveTagIds(tagIds)
entryTags.forEach((tag) => {
const tagKey = this.#getTagKey(tag, {
isHashed: true,
})
tagCleanupPipeline.srem(tagKey, key)
})
tagCleanupPipeline.unlink(tagsKey) // Delete the tags key
// Decrement reference counts and cleanup unused tags
await this.#decrementTagRefs(tagIds)
}
} catch (e) {
// noop
}
}
}
)
await Promise.all(cleanupPromises)
await tagCleanupPipeline.exec()
await deletePipeline.exec()
// Clean up empty tag sets
const allTagKeys = await this.redisClient.keys(
`${this.keyNamePrefix}tag:*`
)
if (allTagKeys.length) {
const cleanupPipeline = this.redisClient.pipeline()
allTagKeys.forEach((tagKey) => {
cleanupPipeline.scard(tagKey)
})
const cardinalityResults = await cleanupPipeline.exec()
// Delete tag keys that are now empty
const emptyTagDeletePipeline = this.redisClient.pipeline()
cardinalityResults?.forEach((result, index) => {
if (result && result[1] === 0) {
emptyTagDeletePipeline.unlink(allTagKeys[index])
}
})
await emptyTagDeletePipeline.exec()
}
return
}
}
} }
} }
} catch (error) {
if (this.isConnectionError(error)) {
this.logger.warn(
"Redis connection error during clear operation, relying on IORedis retry mechanism"
)
return
}
throw error
} }
} }
async flush(): Promise<void> { async flush(): Promise<void> {
// Use SCAN to find ALL keys with our prefix and delete them try {
// This includes main cache keys, tag keys (tag:*), and tags keys (tags:*) // Use SCAN to find ALL keys with our prefix and delete them
const pattern = `${this.keyNamePrefix}*` // This includes main cache keys, tag keys (tag:*), and tags keys (tags:*)
let cursor = "0" const pattern = `${this.keyNamePrefix}*`
let cursor = "0"
do { do {
const result = await this.redisClient.scan( const result = await this.redisClient.scan(
cursor, cursor,
"MATCH", "MATCH",
pattern, pattern,
"COUNT", "COUNT",
1000 1000
) )
cursor = result[0] cursor = result[0]
const keys = result[1] const keys = result[1]
if (keys.length) { if (keys.length) {
await this.redisClient.unlink(...keys) await this.redisClient.unlink(...keys)
}
} while (cursor !== "0")
} catch (error) {
if (this.isConnectionError(error)) {
this.logger.warn(
"Redis connection error during flush operation, relying on IORedis retry mechanism"
)
return
} }
} while (cursor !== "0") throw error
}
} }
} }
@@ -1,20 +1,13 @@
export interface RedisCacheModuleOptions { import { RedisOptions } from "ioredis"
export interface RedisCacheModuleOptions extends RedisOptions {
/**
* Redis connection string
*/
redisUrl?: string
/** /**
* TTL in milliseconds * TTL in milliseconds
*/ */
ttl?: number ttl?: number
/**
* Connection timeout in milliseconds
*/
connectTimeout?: number
/**
* Lazyload connections
*/
lazyConnect?: boolean
/**
* Connection retries
*/
retryDelayOnFailover?: number
/** /**
* Key prefix for all cache keys * Key prefix for all cache keys
*/ */
@@ -42,11 +42,11 @@ export default async (
maxRetriesPerRequest: null, maxRetriesPerRequest: null,
}) })
logger?.info( logger?.info(
`Connection to Redis in module 'workflow-engine-redis' established` `[Workflow-engine-redis] Connection to Redis in module 'workflow-engine-redis' established`
) )
} catch (err) { } catch (err) {
logger?.error( logger?.error(
`An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}` `[Workflow-engine-redis] An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}`
) )
} }
@@ -54,11 +54,11 @@ export default async (
redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options) redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options)
redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options) redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options)
logger?.info( logger?.info(
`Connection to Redis PubSub in module 'workflow-engine-redis' established` `[Workflow-engine-redis] Connection to Redis PubSub in module 'workflow-engine-redis' established`
) )
} catch (err) { } catch (err) {
logger?.error( logger?.error(
`An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}` `[Workflow-engine-redis] An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}`
) )
} }
@@ -128,6 +128,7 @@ export class RedisDistributedTransactionStorage
} }
async onApplicationStart() { async onApplicationStart() {
await this.ensureRedisConnection()
const allowedJobs = [ const allowedJobs = [
JobType.RETRY, JobType.RETRY,
JobType.STEP_TIMEOUT, JobType.STEP_TIMEOUT,
@@ -212,6 +213,64 @@ export class RedisDistributedTransactionStorage
this.workflowOrchestratorService_ = workflowOrchestratorService this.workflowOrchestratorService_ = workflowOrchestratorService
} }
private async ensureRedisConnection(): Promise<void> {
const reconnectTasks: Promise<void>[] = []
if (this.redisClient.status !== "ready") {
this.logger_.warn(
`[Workflow-engine-redis] Redis connection is not ready (status: ${this.redisClient.status}). Attempting to reconnect...`
)
reconnectTasks.push(
this.redisClient
.connect()
.then(() => {
this.logger_.info(
"[Workflow-engine-redis] Redis connection reestablished successfully"
)
})
.catch((error) => {
this.logger_.error(
"[Workflow-engine-redis] Failed to reconnect to Redis",
error
)
throw new MedusaError(
MedusaError.Types.DB_ERROR,
`Redis connection failed: ${error.message}`
)
})
)
}
if (this.redisWorkerConnection.status !== "ready") {
this.logger_.warn(
`[Workflow-engine-redis] Redis worker connection is not ready (status: ${this.redisWorkerConnection.status}). Attempting to reconnect...`
)
reconnectTasks.push(
this.redisWorkerConnection
.connect()
.then(() => {
this.logger_.info(
"[Workflow-engine-redis] Redis worker connection reestablished successfully"
)
})
.catch((error) => {
this.logger_.error(
"[Workflow-engine-redis] Failed to reconnect to Redis worker connection",
error
)
throw new MedusaError(
MedusaError.Types.DB_ERROR,
`Redis worker connection failed: ${error.message}`
)
})
)
}
if (reconnectTasks.length > 0) {
await promiseAll(reconnectTasks)
}
}
private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) { private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isFinished = [ const isFinished = [