chore(): Default caching configuration and gracefull redis error handling (#13663)
* chore(): Default caching configuration and gracefull redis error handling * Create odd-moons-crash.md * chore(): Default caching configuration and gracefull redis error handling * fixes * address feedback * revert(): Test utils imit module fix * reconnect * reconnect * reconnect
This commit is contained in:
1 parent
28d57b7bf8
commit
51859c38a7
9 files changed
+565
-328
No files matched your search
@@ -0,0 +1,6 @@
|
|||||||
|
---
|
||||||
|
"@medusajs/caching-redis": patch
|
||||||
|
"@medusajs/utils": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
chore(): Default caching configuration and gracefull redis error handling
|
||||||
@@ -1048,6 +1048,7 @@ describe("defineConfig", function () {
|
|||||||
|
|
||||||
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
||||||
process.env.REDIS_URL = "redis://localhost:6379"
|
process.env.REDIS_URL = "redis://localhost:6379"
|
||||||
|
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
|
||||||
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
||||||
process.env.S3_PREFIX = "test"
|
process.env.S3_PREFIX = "test"
|
||||||
process.env.S3_REGION = "us-east-1"
|
process.env.S3_REGION = "us-east-1"
|
||||||
@@ -1086,6 +1087,21 @@ describe("defineConfig", function () {
|
|||||||
},
|
},
|
||||||
"resolve": "@medusajs/medusa/cache-redis",
|
"resolve": "@medusajs/medusa/cache-redis",
|
||||||
},
|
},
|
||||||
|
"caching": {
|
||||||
|
"options": {
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"id": "caching-redis",
|
||||||
|
"is_default": true,
|
||||||
|
"options": {
|
||||||
|
"redisUrl": "redis://localhost:6379",
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/medusa/caching-redis",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/caching",
|
||||||
|
},
|
||||||
"cart": {
|
"cart": {
|
||||||
"resolve": "@medusajs/medusa/cart",
|
"resolve": "@medusajs/medusa/cart",
|
||||||
},
|
},
|
||||||
@@ -1255,6 +1271,7 @@ describe("defineConfig", function () {
|
|||||||
|
|
||||||
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
||||||
process.env.REDIS_URL = "redis://localhost:6379"
|
process.env.REDIS_URL = "redis://localhost:6379"
|
||||||
|
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
|
||||||
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
||||||
process.env.S3_PREFIX = "test"
|
process.env.S3_PREFIX = "test"
|
||||||
process.env.S3_REGION = "us-east-1"
|
process.env.S3_REGION = "us-east-1"
|
||||||
@@ -1294,6 +1311,21 @@ describe("defineConfig", function () {
|
|||||||
},
|
},
|
||||||
"resolve": "@medusajs/medusa/cache-redis",
|
"resolve": "@medusajs/medusa/cache-redis",
|
||||||
},
|
},
|
||||||
|
"caching": {
|
||||||
|
"options": {
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"id": "caching-redis",
|
||||||
|
"is_default": true,
|
||||||
|
"options": {
|
||||||
|
"redisUrl": "redis://localhost:6379",
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/medusa/caching-redis",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/caching",
|
||||||
|
},
|
||||||
"cart": {
|
"cart": {
|
||||||
"resolve": "@medusajs/medusa/cart",
|
"resolve": "@medusajs/medusa/cart",
|
||||||
},
|
},
|
||||||
@@ -1473,6 +1505,7 @@ describe("defineConfig", function () {
|
|||||||
|
|
||||||
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
process.env.EXECUTION_CONTEXT = "medusa-cloud"
|
||||||
process.env.REDIS_URL = "redis://localhost:6379"
|
process.env.REDIS_URL = "redis://localhost:6379"
|
||||||
|
process.env.CACHE_REDIS_URL = "redis://localhost:6379"
|
||||||
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
process.env.S3_FILE_URL = "https://s3.amazonaws.com/medusa-cloud-test"
|
||||||
process.env.S3_PREFIX = "test"
|
process.env.S3_PREFIX = "test"
|
||||||
process.env.S3_REGION = "us-east-1"
|
process.env.S3_REGION = "us-east-1"
|
||||||
@@ -1518,6 +1551,21 @@ describe("defineConfig", function () {
|
|||||||
},
|
},
|
||||||
"resolve": "@medusajs/medusa/cache-redis",
|
"resolve": "@medusajs/medusa/cache-redis",
|
||||||
},
|
},
|
||||||
|
"caching": {
|
||||||
|
"options": {
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"id": "caching-redis",
|
||||||
|
"is_default": true,
|
||||||
|
"options": {
|
||||||
|
"redisUrl": "redis://localhost:6379",
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/medusa/caching-redis",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"resolve": "@medusajs/caching",
|
||||||
|
},
|
||||||
"cart": {
|
"cart": {
|
||||||
"resolve": "@medusajs/medusa/cart",
|
"resolve": "@medusajs/medusa/cart",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -303,6 +303,24 @@ function resolveModules(
|
|||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if (process.env.CACHE_REDIS_URL) {
|
||||||
|
cloudModules.push({
|
||||||
|
resolve: MODULE_PACKAGE_NAMES[Modules.CACHING],
|
||||||
|
options: {
|
||||||
|
providers: [
|
||||||
|
{
|
||||||
|
id: "caching-redis",
|
||||||
|
resolve: "@medusajs/medusa/caching-redis",
|
||||||
|
is_default: true,
|
||||||
|
options: {
|
||||||
|
redisUrl: process.env.CACHE_REDIS_URL,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The default set of modules to always use. The end user can swap
|
* The default set of modules to always use. The end user can swap
|
||||||
* the modules by providing an alternate implementation via their
|
* the modules by providing an alternate implementation via their
|
||||||
|
|||||||
@@ -64,12 +64,12 @@ export async function initModules({
|
|||||||
|
|
||||||
async function shutdown() {
|
async function shutdown() {
|
||||||
const promises: Promise<void>[] = []
|
const promises: Promise<void>[] = []
|
||||||
promises.push(medusaApp.onApplicationPrepareShutdown())
|
|
||||||
promises.push(medusaApp.onApplicationShutdown())
|
|
||||||
|
|
||||||
if (shouldDestroyConnectionAutomatically) {
|
if (shouldDestroyConnectionAutomatically) {
|
||||||
promises.push((sharedPgConnection as any).context?.destroy())
|
promises.push((sharedPgConnection as any).context?.destroy())
|
||||||
promises.push((sharedPgConnection as any).destroy())
|
promises.push((sharedPgConnection as any).destroy())
|
||||||
|
promises.push(medusaApp.onApplicationPrepareShutdown())
|
||||||
|
promises.push(medusaApp.onApplicationShutdown())
|
||||||
} else {
|
} else {
|
||||||
if (!preventConnectionDestroyWarning) {
|
if (!preventConnectionDestroyWarning) {
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import type {
|
|||||||
ModulesSdkTypes,
|
ModulesSdkTypes,
|
||||||
} from "@medusajs/framework/types"
|
} from "@medusajs/framework/types"
|
||||||
import { RedisCacheModuleOptions } from "@types"
|
import { RedisCacheModuleOptions } from "@types"
|
||||||
import Redis from "ioredis"
|
import Redis, { type RedisOptions } from "ioredis"
|
||||||
|
|
||||||
export default async (
|
export default async (
|
||||||
{
|
{
|
||||||
@@ -23,36 +23,56 @@ export default async (
|
|||||||
|
|
||||||
const moduleOptions = (options ??
|
const moduleOptions = (options ??
|
||||||
moduleDeclaration?.options ??
|
moduleDeclaration?.options ??
|
||||||
{}) as RedisCacheModuleOptions & {
|
{}) as RedisCacheModuleOptions
|
||||||
redisUrl?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!moduleOptions.redisUrl) {
|
const { redisUrl, ...redisOptions_ } = moduleOptions
|
||||||
|
if (!redisUrl) {
|
||||||
throw new Error("[caching-redis] redisUrl is required")
|
throw new Error("[caching-redis] redisUrl is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
let redisClient: Redis
|
let redisClient: Redis
|
||||||
|
|
||||||
try {
|
const redisOptions: RedisOptions = {
|
||||||
redisClient = new Redis(moduleOptions.redisUrl!, {
|
connectTimeout: 10000,
|
||||||
connectTimeout: 10000,
|
commandTimeout: 5000,
|
||||||
lazyConnect: true,
|
lazyConnect: true,
|
||||||
retryDelayOnFailover: 100,
|
maxRetriesPerRequest: 3,
|
||||||
connectionName: "medusa-cache-redis",
|
enableOfflineQueue: true,
|
||||||
...moduleOptions,
|
connectionName: "medusa-cache-redis",
|
||||||
})
|
...redisOptions_,
|
||||||
|
}
|
||||||
|
|
||||||
// Test connection
|
redisClient = new Redis(redisUrl!, redisOptions)
|
||||||
await redisClient.ping()
|
|
||||||
|
// Handle connection errors gracefully
|
||||||
|
redisClient.on("error", (error) => {
|
||||||
|
logger_.warn(`Redis cache connection error: ${error.message}`)
|
||||||
|
})
|
||||||
|
|
||||||
|
redisClient.on("connect", () => {
|
||||||
logger_.info("Redis cache connection established successfully")
|
logger_.info("Redis cache connection established successfully")
|
||||||
|
})
|
||||||
|
|
||||||
|
redisClient.on("ready", () => {
|
||||||
|
logger_.info("Redis cache is ready to accept commands")
|
||||||
|
})
|
||||||
|
|
||||||
|
redisClient.on("close", () => {
|
||||||
|
logger_.warn("Redis cache connection closed")
|
||||||
|
})
|
||||||
|
|
||||||
|
redisClient.on("reconnecting", () => {
|
||||||
|
logger_.info("Redis cache attempting to reconnect...")
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test connection with timeout
|
||||||
|
await redisClient.ping()
|
||||||
|
logger_.info("Redis cache connection test successful")
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger_.error(`Failed to connect to Redis cache: ${error.message}`)
|
logger_.warn(
|
||||||
redisClient = new Redis(moduleOptions.redisUrl!, {
|
`Redis cache connection test failed: ${error.message}, but continuing with lazy connection`
|
||||||
connectTimeout: 10000,
|
)
|
||||||
lazyConnect: true,
|
|
||||||
retryDelayOnFailover: 100,
|
|
||||||
...moduleOptions,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
container.register({
|
container.register({
|
||||||
@@ -62,5 +82,8 @@ export default async (
|
|||||||
prefix: {
|
prefix: {
|
||||||
resolve: () => moduleOptions.prefix ?? "mc:",
|
resolve: () => moduleOptions.prefix ?? "mc:",
|
||||||
},
|
},
|
||||||
|
logger: {
|
||||||
|
resolve: () => logger_,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { Logger } from "@medusajs/framework/types"
|
||||||
import { RedisCacheModuleOptions } from "@types"
|
import { RedisCacheModuleOptions } from "@types"
|
||||||
import { Redis } from "ioredis"
|
import { Redis } from "ioredis"
|
||||||
import { createGunzip, createGzip } from "zlib"
|
import { createGunzip, createGzip } from "zlib"
|
||||||
@@ -10,13 +11,20 @@ export class RedisCachingProvider {
|
|||||||
protected defaultTTL: number
|
protected defaultTTL: number
|
||||||
protected compressionThreshold: number
|
protected compressionThreshold: number
|
||||||
protected hasher: (key: string) => string
|
protected hasher: (key: string) => string
|
||||||
|
protected logger: Logger
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
{
|
{
|
||||||
redisClient,
|
redisClient,
|
||||||
|
logger,
|
||||||
prefix,
|
prefix,
|
||||||
hasher,
|
hasher,
|
||||||
}: { redisClient: Redis; prefix: string; hasher: (key: string) => string },
|
}: {
|
||||||
|
redisClient: Redis
|
||||||
|
prefix: string
|
||||||
|
hasher: (key: string) => string
|
||||||
|
logger: Logger
|
||||||
|
},
|
||||||
options?: RedisCacheModuleOptions
|
options?: RedisCacheModuleOptions
|
||||||
) {
|
) {
|
||||||
this.redisClient = redisClient
|
this.redisClient = redisClient
|
||||||
@@ -24,6 +32,29 @@ export class RedisCachingProvider {
|
|||||||
this.defaultTTL = options?.ttl ?? 3600 // 1 hour default
|
this.defaultTTL = options?.ttl ?? 3600 // 1 hour default
|
||||||
this.compressionThreshold = options?.compressionThreshold ?? 2048 // 2KB default
|
this.compressionThreshold = options?.compressionThreshold ?? 2048 // 2KB default
|
||||||
this.hasher = hasher
|
this.hasher = hasher
|
||||||
|
this.logger = logger
|
||||||
|
}
|
||||||
|
|
||||||
|
private isConnectionError(error: any): boolean {
|
||||||
|
return (
|
||||||
|
error.code === "ECONNREFUSED" ||
|
||||||
|
error.code === "ENOTFOUND" ||
|
||||||
|
error.code === "ETIMEDOUT" ||
|
||||||
|
error.code === "ECONNRESET" ||
|
||||||
|
error.code === "EPIPE" ||
|
||||||
|
error.message?.includes("Connection is closed") ||
|
||||||
|
error.message?.includes("connect ECONNREFUSED") ||
|
||||||
|
error.message?.includes("connect ETIMEDOUT") ||
|
||||||
|
error.message?.includes("Command timed out") ||
|
||||||
|
error.message?.includes("Maximum number of retries exceeded") ||
|
||||||
|
["connecting", "reconnecting", "disconnecting", "wait", "end"].includes(
|
||||||
|
this.redisClient.status
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private isConnectionHealthy(): boolean {
|
||||||
|
return this.redisClient.status === "ready"
|
||||||
}
|
}
|
||||||
|
|
||||||
#getKeyName(key: string): string {
|
#getKeyName(key: string): string {
|
||||||
@@ -252,66 +283,92 @@ export class RedisCachingProvider {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async get({ key, tags }: { key?: string; tags?: string[] }): Promise<any> {
|
async get({ key, tags }: { key?: string; tags?: string[] }): Promise<any> {
|
||||||
if (key) {
|
if (!this.isConnectionHealthy()) {
|
||||||
const keyName = this.#getKeyName(key)
|
return null
|
||||||
const buffer = await this.redisClient.hgetBuffer(keyName, "data")
|
}
|
||||||
if (!buffer) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
const finalData = await this.#decompressData(buffer)
|
if (key) {
|
||||||
return JSON.parse(finalData)
|
try {
|
||||||
|
const keyName = this.#getKeyName(key)
|
||||||
|
const buffer = await this.redisClient.hgetBuffer(keyName, "data")
|
||||||
|
if (!buffer) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalData = await this.#decompressData(buffer)
|
||||||
|
return JSON.parse(finalData)
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isConnectionError(error)) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Redis connection error during get operation, returning null to trigger fallback to original data source"
|
||||||
|
)
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (tags?.length) {
|
if (tags?.length) {
|
||||||
// Get all keys associated with the tags
|
try {
|
||||||
const pipeline = this.redisClient.pipeline()
|
// Get all keys associated with the tags
|
||||||
tags.forEach((tag) => {
|
const pipeline = this.redisClient.pipeline()
|
||||||
const tagKey = this.#getTagKey(tag)
|
tags.forEach((tag) => {
|
||||||
pipeline.smembers(tagKey)
|
const tagKey = this.#getTagKey(tag)
|
||||||
})
|
pipeline.smembers(tagKey)
|
||||||
|
})
|
||||||
|
|
||||||
const tagResults = await pipeline.exec()
|
const tagResults = await pipeline.exec()
|
||||||
const allKeys = new Set<string>()
|
const allKeys = new Set<string>()
|
||||||
|
|
||||||
tagResults?.forEach((result, index) => {
|
tagResults?.forEach((result, index) => {
|
||||||
if (result && result[1]) {
|
if (result && result[1]) {
|
||||||
;(result[1] as string[]).forEach((key) => allKeys.add(key))
|
;(result[1] as string[]).forEach((key) => allKeys.add(key))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if (allKeys.size === 0) {
|
||||||
|
return []
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
|
||||||
if (allKeys.size === 0) {
|
// Get all hash data for the keys
|
||||||
return []
|
const valuePipeline = this.redisClient.pipeline()
|
||||||
}
|
Array.from(allKeys).forEach((key) => {
|
||||||
|
valuePipeline.hgetBuffer(key, "data")
|
||||||
|
})
|
||||||
|
|
||||||
// Get all hash data for the keys
|
const valueResults = await valuePipeline.exec()
|
||||||
const valuePipeline = this.redisClient.pipeline()
|
const results: any[] = []
|
||||||
Array.from(allKeys).forEach((key) => {
|
|
||||||
valuePipeline.hgetBuffer(key, "data")
|
|
||||||
})
|
|
||||||
|
|
||||||
const valueResults = await valuePipeline.exec()
|
const decompressionPromises = (valueResults || []).map(
|
||||||
const results: any[] = []
|
async (result) => {
|
||||||
|
if (result && result[1]) {
|
||||||
const decompressionPromises = (valueResults || []).map(async (result) => {
|
const buffer = result[1] as Buffer
|
||||||
if (result && result[1]) {
|
try {
|
||||||
const buffer = result[1] as Buffer
|
const finalData = await this.#decompressData(buffer)
|
||||||
try {
|
return JSON.parse(finalData)
|
||||||
const finalData = await this.#decompressData(buffer)
|
} catch (e) {
|
||||||
return JSON.parse(finalData)
|
// If JSON parsing fails, skip this entry (corrupted data)
|
||||||
} catch (e) {
|
this.logger.warn(`Skipping corrupted cache entry: ${e.message}`)
|
||||||
// If JSON parsing fails, skip this entry (corrupted data)
|
return null
|
||||||
console.warn(`Skipping corrupted cache entry: ${e.message}`)
|
}
|
||||||
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const decompressionResults = await Promise.all(decompressionPromises)
|
||||||
|
results.push(...decompressionResults.filter(Boolean))
|
||||||
|
|
||||||
|
return results
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isConnectionError(error)) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Redis connection error during get operation, returning empty array to trigger fallback to original data source"
|
||||||
|
)
|
||||||
|
return null
|
||||||
}
|
}
|
||||||
return null
|
throw error
|
||||||
})
|
}
|
||||||
|
|
||||||
const decompressionResults = await Promise.all(decompressionPromises)
|
|
||||||
results.push(...decompressionResults.filter(Boolean))
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return null
|
return null
|
||||||
@@ -332,53 +389,63 @@ export class RedisCachingProvider {
|
|||||||
autoInvalidate?: boolean
|
autoInvalidate?: boolean
|
||||||
}
|
}
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
const keyName = this.#getKeyName(key)
|
try {
|
||||||
const serializedData = JSON.stringify(data)
|
const keyName = this.#getKeyName(key)
|
||||||
const effectiveTTL = ttl ?? this.defaultTTL
|
const serializedData = JSON.stringify(data)
|
||||||
|
const effectiveTTL = ttl ?? this.defaultTTL
|
||||||
|
|
||||||
const finalData = await this.#compressData(serializedData)
|
const finalData = await this.#compressData(serializedData)
|
||||||
|
|
||||||
let tagIds: number[] = []
|
let tagIds: number[] = []
|
||||||
if (tags?.length) {
|
if (tags?.length) {
|
||||||
tagIds = await this.#internTags(tags)
|
tagIds = await this.#internTags(tags)
|
||||||
}
|
|
||||||
|
|
||||||
const setPipeline = this.redisClient.pipeline()
|
|
||||||
|
|
||||||
// Main data with conditional operations
|
|
||||||
setPipeline.hsetnx(keyName, "data", finalData)
|
|
||||||
if (options && Object.keys(options).length) {
|
|
||||||
setPipeline.hset(keyName, "options", JSON.stringify(options))
|
|
||||||
}
|
|
||||||
if (effectiveTTL) {
|
|
||||||
setPipeline.expire(keyName, effectiveTTL)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store tag IDs if present
|
|
||||||
if (tags?.length && tagIds.length) {
|
|
||||||
const tagsKey = this.#getTagsKey(key)
|
|
||||||
const buffer = Buffer.alloc(tagIds.length * 4)
|
|
||||||
tagIds.forEach((id, index) => {
|
|
||||||
buffer.writeUInt32LE(id, index * 4)
|
|
||||||
})
|
|
||||||
|
|
||||||
if (effectiveTTL) {
|
|
||||||
setPipeline.set(tagsKey, buffer, "EX", effectiveTTL + 60, "NX")
|
|
||||||
} else {
|
|
||||||
setPipeline.setnx(tagsKey, buffer)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add tag operations to the same pipeline
|
const setPipeline = this.redisClient.pipeline()
|
||||||
tags.forEach((tag) => {
|
|
||||||
const tagKey = this.#getTagKey(tag)
|
|
||||||
setPipeline.sadd(tagKey, keyName)
|
|
||||||
if (effectiveTTL) {
|
|
||||||
setPipeline.expire(tagKey, effectiveTTL + 60)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
await setPipeline.exec()
|
// Main data with conditional operations
|
||||||
|
setPipeline.hsetnx(keyName, "data", finalData)
|
||||||
|
if (options && Object.keys(options).length) {
|
||||||
|
setPipeline.hset(keyName, "options", JSON.stringify(options))
|
||||||
|
}
|
||||||
|
if (effectiveTTL) {
|
||||||
|
setPipeline.expire(keyName, effectiveTTL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store tag IDs if present
|
||||||
|
if (tags?.length && tagIds.length) {
|
||||||
|
const tagsKey = this.#getTagsKey(key)
|
||||||
|
const buffer = Buffer.alloc(tagIds.length * 4)
|
||||||
|
tagIds.forEach((id, index) => {
|
||||||
|
buffer.writeUInt32LE(id, index * 4)
|
||||||
|
})
|
||||||
|
|
||||||
|
if (effectiveTTL) {
|
||||||
|
setPipeline.set(tagsKey, buffer, "EX", effectiveTTL + 60, "NX")
|
||||||
|
} else {
|
||||||
|
setPipeline.setnx(tagsKey, buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add tag operations to the same pipeline
|
||||||
|
tags.forEach((tag) => {
|
||||||
|
const tagKey = this.#getTagKey(tag)
|
||||||
|
setPipeline.sadd(tagKey, keyName)
|
||||||
|
if (effectiveTTL) {
|
||||||
|
setPipeline.expire(tagKey, effectiveTTL + 60)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
await setPipeline.exec()
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isConnectionError(error)) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Redis connection error during set operation, relying on IORedis retry mechanism"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async clear({
|
async clear({
|
||||||
@@ -392,208 +459,96 @@ export class RedisCachingProvider {
|
|||||||
autoInvalidate?: boolean
|
autoInvalidate?: boolean
|
||||||
}
|
}
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (key) {
|
try {
|
||||||
const keyName = this.#getKeyName(key)
|
if (key) {
|
||||||
const tagsKey = this.#getTagsKey(key)
|
const keyName = this.#getKeyName(key)
|
||||||
|
const tagsKey = this.#getTagsKey(key)
|
||||||
|
|
||||||
const clearPipeline = this.redisClient.pipeline()
|
const clearPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
// Get tags for cleanup and delete main key in same pipeline
|
// Get tags for cleanup and delete main key in same pipeline
|
||||||
clearPipeline.getBuffer(tagsKey)
|
clearPipeline.getBuffer(tagsKey)
|
||||||
clearPipeline.unlink(keyName)
|
clearPipeline.unlink(keyName)
|
||||||
|
|
||||||
const results = await clearPipeline.exec()
|
const results = await clearPipeline.exec()
|
||||||
const tagsBuffer = results?.[0]?.[1] as Buffer
|
const tagsBuffer = results?.[0]?.[1] as Buffer
|
||||||
|
|
||||||
if (tagsBuffer?.length) {
|
if (tagsBuffer?.length) {
|
||||||
try {
|
try {
|
||||||
// Binary format: array of 32-bit integers
|
// Binary format: array of 32-bit integers
|
||||||
const tagIds: number[] = []
|
const tagIds: number[] = []
|
||||||
for (let i = 0; i < tagsBuffer.length; i += 4) {
|
for (let i = 0; i < tagsBuffer.length; i += 4) {
|
||||||
tagIds.push(tagsBuffer.readUInt32LE(i))
|
tagIds.push(tagsBuffer.readUInt32LE(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tagIds.length) {
|
||||||
|
const entryTags = await this.#resolveTagIds(tagIds)
|
||||||
|
|
||||||
|
const tagCleanupPipeline = this.redisClient.pipeline()
|
||||||
|
entryTags.forEach((tag) => {
|
||||||
|
const tagKey = this.#getTagKey(tag, { isHashed: true })
|
||||||
|
tagCleanupPipeline.srem(tagKey, keyName)
|
||||||
|
})
|
||||||
|
tagCleanupPipeline.unlink(tagsKey)
|
||||||
|
await tagCleanupPipeline.exec()
|
||||||
|
|
||||||
|
// Decrement reference counts and cleanup unused tags
|
||||||
|
await this.#decrementTagRefs(tagIds)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// noop - corrupted tag data, skip cleanup
|
||||||
}
|
}
|
||||||
|
|
||||||
if (tagIds.length) {
|
|
||||||
const entryTags = await this.#resolveTagIds(tagIds)
|
|
||||||
|
|
||||||
const tagCleanupPipeline = this.redisClient.pipeline()
|
|
||||||
entryTags.forEach((tag) => {
|
|
||||||
const tagKey = this.#getTagKey(tag, { isHashed: true })
|
|
||||||
tagCleanupPipeline.srem(tagKey, keyName)
|
|
||||||
})
|
|
||||||
tagCleanupPipeline.unlink(tagsKey)
|
|
||||||
await tagCleanupPipeline.exec()
|
|
||||||
|
|
||||||
// Decrement reference counts and cleanup unused tags
|
|
||||||
await this.#decrementTagRefs(tagIds)
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
// noop - corrupted tag data, skip cleanup
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tags?.length) {
|
|
||||||
// Handle wildcard tag to clear all cache data
|
|
||||||
if (tags.includes("*")) {
|
|
||||||
await this.flush()
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get all keys associated with the tags
|
if (tags?.length) {
|
||||||
const pipeline = this.redisClient.pipeline()
|
// Handle wildcard tag to clear all cache data
|
||||||
tags.forEach((tag) => {
|
if (tags.includes("*")) {
|
||||||
const tagKey = this.#getTagKey(tag)
|
await this.flush()
|
||||||
pipeline.smembers(tagKey)
|
|
||||||
})
|
|
||||||
|
|
||||||
const tagResults = await pipeline.exec()
|
|
||||||
|
|
||||||
const allKeys = new Set<string>()
|
|
||||||
|
|
||||||
tagResults?.forEach((result) => {
|
|
||||||
if (result && result[1]) {
|
|
||||||
;(result[1] as string[]).forEach((key) => allKeys.add(key))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if (allKeys.size) {
|
|
||||||
// If no options provided (user explicit call), clear everything
|
|
||||||
if (!options) {
|
|
||||||
const deletePipeline = this.redisClient.pipeline()
|
|
||||||
|
|
||||||
// Delete main keys and options
|
|
||||||
Array.from(allKeys).forEach((key) => {
|
|
||||||
deletePipeline.unlink(key)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Clean up tag references for each key
|
|
||||||
const tagDataPromises = Array.from(allKeys).map(async (key) => {
|
|
||||||
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
|
|
||||||
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
|
|
||||||
const tagsData = await this.redisClient.getBuffer(tagsKey)
|
|
||||||
return { key, tagsKey, tagsData }
|
|
||||||
})
|
|
||||||
|
|
||||||
const tagResults = await Promise.all(tagDataPromises)
|
|
||||||
|
|
||||||
// Build single pipeline for all tag cleanup operations
|
|
||||||
const tagCleanupPipeline = this.redisClient.pipeline()
|
|
||||||
const cleanupPromises = tagResults.map(
|
|
||||||
async ({ key, tagsKey, tagsData }) => {
|
|
||||||
if (tagsData) {
|
|
||||||
try {
|
|
||||||
// Binary format: array of 32-bit integers
|
|
||||||
const tagIds: number[] = []
|
|
||||||
for (let i = 0; i < tagsData.length; i += 4) {
|
|
||||||
tagIds.push(tagsData.readUInt32LE(i))
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tagIds.length) {
|
|
||||||
const entryTags = await this.#resolveTagIds(tagIds)
|
|
||||||
entryTags.forEach((tag) => {
|
|
||||||
const tagKey = this.#getTagKey(tag, { isHashed: true })
|
|
||||||
tagCleanupPipeline.srem(tagKey, key)
|
|
||||||
})
|
|
||||||
tagCleanupPipeline.unlink(tagsKey)
|
|
||||||
|
|
||||||
// Decrement reference counts and cleanup unused tags
|
|
||||||
await this.#decrementTagRefs(tagIds)
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
// noop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
await Promise.all(cleanupPromises)
|
|
||||||
await tagCleanupPipeline.exec()
|
|
||||||
await deletePipeline.exec()
|
|
||||||
|
|
||||||
// Clean up empty tag sets
|
|
||||||
const allTagKeys = await this.redisClient.keys(
|
|
||||||
`${this.keyNamePrefix}tag:*`
|
|
||||||
)
|
|
||||||
if (allTagKeys.length) {
|
|
||||||
const cardinalityPipeline = this.redisClient.pipeline()
|
|
||||||
allTagKeys.forEach((tagKey) => {
|
|
||||||
cardinalityPipeline.scard(tagKey)
|
|
||||||
})
|
|
||||||
|
|
||||||
const cardinalityResults = await cardinalityPipeline.exec()
|
|
||||||
|
|
||||||
// Delete empty tag keys
|
|
||||||
const emptyTagPipeline = this.redisClient.pipeline()
|
|
||||||
cardinalityResults?.forEach((result, index) => {
|
|
||||||
if (result && result[1] === 0) {
|
|
||||||
emptyTagPipeline.unlink(allTagKeys[index])
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await emptyTagPipeline.exec()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// If autoInvalidate is true (strategy call), only clear entries with autoInvalidate=true (default)
|
// Get all keys associated with the tags
|
||||||
if (options.autoInvalidate === true) {
|
const pipeline = this.redisClient.pipeline()
|
||||||
const optionsPipeline = this.redisClient.pipeline()
|
tags.forEach((tag) => {
|
||||||
|
const tagKey = this.#getTagKey(tag)
|
||||||
|
pipeline.smembers(tagKey)
|
||||||
|
})
|
||||||
|
|
||||||
Array.from(allKeys).forEach((key) => {
|
const tagResults = await pipeline.exec()
|
||||||
optionsPipeline.hget(key, "options")
|
|
||||||
})
|
|
||||||
|
|
||||||
const optionsResults = await optionsPipeline.exec()
|
const allKeys = new Set<string>()
|
||||||
const keysToDelete: string[] = []
|
|
||||||
|
|
||||||
Array.from(allKeys).forEach((key, index) => {
|
tagResults?.forEach((result) => {
|
||||||
const optionsResult = optionsResults?.[index]
|
if (result && result[1]) {
|
||||||
|
;(result[1] as string[]).forEach((key) => allKeys.add(key))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
if (optionsResult && optionsResult[1]) {
|
if (allKeys.size) {
|
||||||
try {
|
// If no options provided (user explicit call), clear everything
|
||||||
const entryOptions = JSON.parse(optionsResult[1] as string)
|
if (!options) {
|
||||||
|
|
||||||
// Delete if entry has autoInvalidate=true or no setting (default true)
|
|
||||||
const shouldAutoInvalidate = entryOptions.autoInvalidate ?? true
|
|
||||||
|
|
||||||
if (shouldAutoInvalidate) {
|
|
||||||
keysToDelete.push(key)
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
// If can't parse options, assume it's safe to delete (default true)
|
|
||||||
keysToDelete.push(key)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// No options stored, default to true
|
|
||||||
keysToDelete.push(key)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if (keysToDelete.length) {
|
|
||||||
const deletePipeline = this.redisClient.pipeline()
|
const deletePipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
keysToDelete.forEach((key) => {
|
// Delete main keys and options
|
||||||
|
Array.from(allKeys).forEach((key) => {
|
||||||
deletePipeline.unlink(key)
|
deletePipeline.unlink(key)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Clean up tag references for each key to delete
|
// Clean up tag references for each key
|
||||||
const tagDataPromises = keysToDelete.map(async (key) => {
|
const tagDataPromises = Array.from(allKeys).map(async (key) => {
|
||||||
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
|
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
|
||||||
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
|
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
|
||||||
const tagsData = await this.redisClient.getBuffer(tagsKey)
|
const tagsData = await this.redisClient.getBuffer(tagsKey)
|
||||||
return { key, tagsKey, tagsData }
|
return { key, tagsKey, tagsData }
|
||||||
})
|
})
|
||||||
|
|
||||||
// Wait for all tag data fetches
|
|
||||||
const tagResults = await Promise.all(tagDataPromises)
|
const tagResults = await Promise.all(tagDataPromises)
|
||||||
|
|
||||||
// Build single pipeline for all tag cleanup operations
|
// Build single pipeline for all tag cleanup operations
|
||||||
const tagCleanupPipeline = this.redisClient.pipeline()
|
const tagCleanupPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
const cleanupPromises = tagResults.map(
|
const cleanupPromises = tagResults.map(
|
||||||
async ({ key, tagsKey, tagsData }) => {
|
async ({ key, tagsKey, tagsData }) => {
|
||||||
if (tagsData) {
|
if (tagsData) {
|
||||||
@@ -610,7 +565,7 @@ export class RedisCachingProvider {
|
|||||||
const tagKey = this.#getTagKey(tag, { isHashed: true })
|
const tagKey = this.#getTagKey(tag, { isHashed: true })
|
||||||
tagCleanupPipeline.srem(tagKey, key)
|
tagCleanupPipeline.srem(tagKey, key)
|
||||||
})
|
})
|
||||||
tagCleanupPipeline.unlink(tagsKey) // Delete the tags key
|
tagCleanupPipeline.unlink(tagsKey)
|
||||||
|
|
||||||
// Decrement reference counts and cleanup unused tags
|
// Decrement reference counts and cleanup unused tags
|
||||||
await this.#decrementTagRefs(tagIds)
|
await this.#decrementTagRefs(tagIds)
|
||||||
@@ -624,7 +579,6 @@ export class RedisCachingProvider {
|
|||||||
|
|
||||||
await Promise.all(cleanupPromises)
|
await Promise.all(cleanupPromises)
|
||||||
await tagCleanupPipeline.exec()
|
await tagCleanupPipeline.exec()
|
||||||
|
|
||||||
await deletePipeline.exec()
|
await deletePipeline.exec()
|
||||||
|
|
||||||
// Clean up empty tag sets
|
// Clean up empty tag sets
|
||||||
@@ -632,52 +586,188 @@ export class RedisCachingProvider {
|
|||||||
`${this.keyNamePrefix}tag:*`
|
`${this.keyNamePrefix}tag:*`
|
||||||
)
|
)
|
||||||
if (allTagKeys.length) {
|
if (allTagKeys.length) {
|
||||||
const cleanupPipeline = this.redisClient.pipeline()
|
const cardinalityPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
allTagKeys.forEach((tagKey) => {
|
allTagKeys.forEach((tagKey) => {
|
||||||
cleanupPipeline.scard(tagKey)
|
cardinalityPipeline.scard(tagKey)
|
||||||
})
|
})
|
||||||
|
|
||||||
const cardinalityResults = await cleanupPipeline.exec()
|
const cardinalityResults = await cardinalityPipeline.exec()
|
||||||
|
|
||||||
// Delete tag keys that are now empty
|
// Delete empty tag keys
|
||||||
const emptyTagDeletePipeline = this.redisClient.pipeline()
|
const emptyTagPipeline = this.redisClient.pipeline()
|
||||||
cardinalityResults?.forEach((result, index) => {
|
cardinalityResults?.forEach((result, index) => {
|
||||||
if (result && result[1] === 0) {
|
if (result && result[1] === 0) {
|
||||||
emptyTagDeletePipeline.unlink(allTagKeys[index])
|
emptyTagPipeline.unlink(allTagKeys[index])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
await emptyTagDeletePipeline.exec()
|
await emptyTagPipeline.exec()
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If autoInvalidate is true (strategy call), only clear entries with autoInvalidate=true (default)
|
||||||
|
if (options.autoInvalidate === true) {
|
||||||
|
const optionsPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
|
Array.from(allKeys).forEach((key) => {
|
||||||
|
optionsPipeline.hget(key, "options")
|
||||||
|
})
|
||||||
|
|
||||||
|
const optionsResults = await optionsPipeline.exec()
|
||||||
|
const keysToDelete: string[] = []
|
||||||
|
|
||||||
|
Array.from(allKeys).forEach((key, index) => {
|
||||||
|
const optionsResult = optionsResults?.[index]
|
||||||
|
|
||||||
|
if (optionsResult && optionsResult[1]) {
|
||||||
|
try {
|
||||||
|
const entryOptions = JSON.parse(optionsResult[1] as string)
|
||||||
|
|
||||||
|
// Delete if entry has autoInvalidate=true or no setting (default true)
|
||||||
|
const shouldAutoInvalidate =
|
||||||
|
entryOptions.autoInvalidate ?? true
|
||||||
|
|
||||||
|
if (shouldAutoInvalidate) {
|
||||||
|
keysToDelete.push(key)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// If can't parse options, assume it's safe to delete (default true)
|
||||||
|
keysToDelete.push(key)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No options stored, default to true
|
||||||
|
keysToDelete.push(key)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if (keysToDelete.length) {
|
||||||
|
const deletePipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
|
keysToDelete.forEach((key) => {
|
||||||
|
deletePipeline.unlink(key)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Clean up tag references for each key to delete
|
||||||
|
const tagDataPromises = keysToDelete.map(async (key) => {
|
||||||
|
const keyWithoutPrefix = key.replace(this.keyNamePrefix, "")
|
||||||
|
const tagsKey = this.#getTagsKey(keyWithoutPrefix)
|
||||||
|
const tagsData = await this.redisClient.getBuffer(tagsKey)
|
||||||
|
return { key, tagsKey, tagsData }
|
||||||
|
})
|
||||||
|
|
||||||
|
// Wait for all tag data fetches
|
||||||
|
const tagResults = await Promise.all(tagDataPromises)
|
||||||
|
|
||||||
|
// Build single pipeline for all tag cleanup operations
|
||||||
|
const tagCleanupPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
|
const cleanupPromises = tagResults.map(
|
||||||
|
async ({ key, tagsKey, tagsData }) => {
|
||||||
|
if (tagsData) {
|
||||||
|
try {
|
||||||
|
// Binary format: array of 32-bit integers
|
||||||
|
const tagIds: number[] = []
|
||||||
|
for (let i = 0; i < tagsData.length; i += 4) {
|
||||||
|
tagIds.push(tagsData.readUInt32LE(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tagIds.length) {
|
||||||
|
const entryTags = await this.#resolveTagIds(tagIds)
|
||||||
|
entryTags.forEach((tag) => {
|
||||||
|
const tagKey = this.#getTagKey(tag, {
|
||||||
|
isHashed: true,
|
||||||
|
})
|
||||||
|
tagCleanupPipeline.srem(tagKey, key)
|
||||||
|
})
|
||||||
|
tagCleanupPipeline.unlink(tagsKey) // Delete the tags key
|
||||||
|
|
||||||
|
// Decrement reference counts and cleanup unused tags
|
||||||
|
await this.#decrementTagRefs(tagIds)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// noop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
await Promise.all(cleanupPromises)
|
||||||
|
await tagCleanupPipeline.exec()
|
||||||
|
|
||||||
|
await deletePipeline.exec()
|
||||||
|
|
||||||
|
// Clean up empty tag sets
|
||||||
|
const allTagKeys = await this.redisClient.keys(
|
||||||
|
`${this.keyNamePrefix}tag:*`
|
||||||
|
)
|
||||||
|
if (allTagKeys.length) {
|
||||||
|
const cleanupPipeline = this.redisClient.pipeline()
|
||||||
|
|
||||||
|
allTagKeys.forEach((tagKey) => {
|
||||||
|
cleanupPipeline.scard(tagKey)
|
||||||
|
})
|
||||||
|
|
||||||
|
const cardinalityResults = await cleanupPipeline.exec()
|
||||||
|
|
||||||
|
// Delete tag keys that are now empty
|
||||||
|
const emptyTagDeletePipeline = this.redisClient.pipeline()
|
||||||
|
cardinalityResults?.forEach((result, index) => {
|
||||||
|
if (result && result[1] === 0) {
|
||||||
|
emptyTagDeletePipeline.unlink(allTagKeys[index])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
await emptyTagDeletePipeline.exec()
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isConnectionError(error)) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Redis connection error during clear operation, relying on IORedis retry mechanism"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
throw error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async flush(): Promise<void> {
|
async flush(): Promise<void> {
|
||||||
// Use SCAN to find ALL keys with our prefix and delete them
|
try {
|
||||||
// This includes main cache keys, tag keys (tag:*), and tags keys (tags:*)
|
// Use SCAN to find ALL keys with our prefix and delete them
|
||||||
const pattern = `${this.keyNamePrefix}*`
|
// This includes main cache keys, tag keys (tag:*), and tags keys (tags:*)
|
||||||
let cursor = "0"
|
const pattern = `${this.keyNamePrefix}*`
|
||||||
|
let cursor = "0"
|
||||||
|
|
||||||
do {
|
do {
|
||||||
const result = await this.redisClient.scan(
|
const result = await this.redisClient.scan(
|
||||||
cursor,
|
cursor,
|
||||||
"MATCH",
|
"MATCH",
|
||||||
pattern,
|
pattern,
|
||||||
"COUNT",
|
"COUNT",
|
||||||
1000
|
1000
|
||||||
)
|
)
|
||||||
cursor = result[0]
|
cursor = result[0]
|
||||||
const keys = result[1]
|
const keys = result[1]
|
||||||
|
|
||||||
if (keys.length) {
|
if (keys.length) {
|
||||||
await this.redisClient.unlink(...keys)
|
await this.redisClient.unlink(...keys)
|
||||||
|
}
|
||||||
|
} while (cursor !== "0")
|
||||||
|
} catch (error) {
|
||||||
|
if (this.isConnectionError(error)) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Redis connection error during flush operation, relying on IORedis retry mechanism"
|
||||||
|
)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
} while (cursor !== "0")
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,20 +1,13 @@
|
|||||||
export interface RedisCacheModuleOptions {
|
import { RedisOptions } from "ioredis"
|
||||||
|
export interface RedisCacheModuleOptions extends RedisOptions {
|
||||||
|
/**
|
||||||
|
* Redis connection string
|
||||||
|
*/
|
||||||
|
redisUrl?: string
|
||||||
/**
|
/**
|
||||||
* TTL in milliseconds
|
* TTL in milliseconds
|
||||||
*/
|
*/
|
||||||
ttl?: number
|
ttl?: number
|
||||||
/**
|
|
||||||
* Connection timeout in milliseconds
|
|
||||||
*/
|
|
||||||
connectTimeout?: number
|
|
||||||
/**
|
|
||||||
* Lazyload connections
|
|
||||||
*/
|
|
||||||
lazyConnect?: boolean
|
|
||||||
/**
|
|
||||||
* Connection retries
|
|
||||||
*/
|
|
||||||
retryDelayOnFailover?: number
|
|
||||||
/**
|
/**
|
||||||
* Key prefix for all cache keys
|
* Key prefix for all cache keys
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -42,11 +42,11 @@ export default async (
|
|||||||
maxRetriesPerRequest: null,
|
maxRetriesPerRequest: null,
|
||||||
})
|
})
|
||||||
logger?.info(
|
logger?.info(
|
||||||
`Connection to Redis in module 'workflow-engine-redis' established`
|
`[Workflow-engine-redis] Connection to Redis in module 'workflow-engine-redis' established`
|
||||||
)
|
)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger?.error(
|
logger?.error(
|
||||||
`An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}`
|
`[Workflow-engine-redis] An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,11 +54,11 @@ export default async (
|
|||||||
redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options)
|
redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options)
|
||||||
redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options)
|
redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options)
|
||||||
logger?.info(
|
logger?.info(
|
||||||
`Connection to Redis PubSub in module 'workflow-engine-redis' established`
|
`[Workflow-engine-redis] Connection to Redis PubSub in module 'workflow-engine-redis' established`
|
||||||
)
|
)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger?.error(
|
logger?.error(
|
||||||
`An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}`
|
`[Workflow-engine-redis] An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ export class RedisDistributedTransactionStorage
|
|||||||
}
|
}
|
||||||
|
|
||||||
async onApplicationStart() {
|
async onApplicationStart() {
|
||||||
|
await this.ensureRedisConnection()
|
||||||
const allowedJobs = [
|
const allowedJobs = [
|
||||||
JobType.RETRY,
|
JobType.RETRY,
|
||||||
JobType.STEP_TIMEOUT,
|
JobType.STEP_TIMEOUT,
|
||||||
@@ -212,6 +213,64 @@ export class RedisDistributedTransactionStorage
|
|||||||
this.workflowOrchestratorService_ = workflowOrchestratorService
|
this.workflowOrchestratorService_ = workflowOrchestratorService
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async ensureRedisConnection(): Promise<void> {
|
||||||
|
const reconnectTasks: Promise<void>[] = []
|
||||||
|
|
||||||
|
if (this.redisClient.status !== "ready") {
|
||||||
|
this.logger_.warn(
|
||||||
|
`[Workflow-engine-redis] Redis connection is not ready (status: ${this.redisClient.status}). Attempting to reconnect...`
|
||||||
|
)
|
||||||
|
reconnectTasks.push(
|
||||||
|
this.redisClient
|
||||||
|
.connect()
|
||||||
|
.then(() => {
|
||||||
|
this.logger_.info(
|
||||||
|
"[Workflow-engine-redis] Redis connection reestablished successfully"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
this.logger_.error(
|
||||||
|
"[Workflow-engine-redis] Failed to reconnect to Redis",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
throw new MedusaError(
|
||||||
|
MedusaError.Types.DB_ERROR,
|
||||||
|
`Redis connection failed: ${error.message}`
|
||||||
|
)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.redisWorkerConnection.status !== "ready") {
|
||||||
|
this.logger_.warn(
|
||||||
|
`[Workflow-engine-redis] Redis worker connection is not ready (status: ${this.redisWorkerConnection.status}). Attempting to reconnect...`
|
||||||
|
)
|
||||||
|
reconnectTasks.push(
|
||||||
|
this.redisWorkerConnection
|
||||||
|
.connect()
|
||||||
|
.then(() => {
|
||||||
|
this.logger_.info(
|
||||||
|
"[Workflow-engine-redis] Redis worker connection reestablished successfully"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
this.logger_.error(
|
||||||
|
"[Workflow-engine-redis] Failed to reconnect to Redis worker connection",
|
||||||
|
error
|
||||||
|
)
|
||||||
|
throw new MedusaError(
|
||||||
|
MedusaError.Types.DB_ERROR,
|
||||||
|
`Redis worker connection failed: ${error.message}`
|
||||||
|
)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (reconnectTasks.length > 0) {
|
||||||
|
await promiseAll(reconnectTasks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
|
private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
|
||||||
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
|
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
|
||||||
const isFinished = [
|
const isFinished = [
|
||||||
|
|||||||
Reference in new issue
Block a user