docs: add generator for llms-full.txt (#11323)

* initial * improvements * finished implementation * transform links to index.html.md links * fix for resources
2025-02-05 16:34:39 +02:00
parent da25980d24
commit 5f7ff7f9f0
17 changed files with 31023 additions and 17 deletions
@@ -0,0 +1,115 @@
+import { getCleanMd, GetCleanMdOptions } from "docs-utils"
+import { fdir } from "fdir"
+import { writeFile } from "fs/promises"
+import path from "path"
+import {
+  apiRefLlmsGenerator,
+  CustomLlmsGenerator,
+  jsSdkLlmsGenerator,
+  stepsLlmsGenerator,
+  workflowsLlmsGenerator,
+} from "./utils/custom-llms-generators.js"
+
+type FileExt = "md" | "yaml"
+
+type Options = {
+  outputPath: string
+  scanDirs: {
+    dir: string
+    options?: Omit<GetCleanMdOptions, "file" | "type">
+    allowedFilesPatterns?: RegExp[]
+    generator?: {
+      name: "workflows" | "steps" | "jsSdk" | "apiRef"
+      options: Record<string, unknown>
+    }
+    ext?: FileExt
+  }[]
+  introText?: string
+  plugins?: GetCleanMdOptions["plugins"]
+}
+
+const generators: Record<string, CustomLlmsGenerator<any>> = {
+  workflows: workflowsLlmsGenerator,
+  steps: stepsLlmsGenerator,
+  jsSdk: jsSdkLlmsGenerator,
+  apiRef: apiRefLlmsGenerator,
+}
+
+const isExtAllowed = (fileName: string, allowedExt: FileExt) => {
+  switch (allowedExt) {
+    case "md":
+      return fileName.endsWith(".md") || fileName.endsWith(".mdx")
+    case "yaml":
+      return fileName.endsWith(".yaml") || fileName.endsWith(".yml")
+  }
+}
+
+const getContentFromDir = async ({
+  dir,
+  options = {},
+  allowedFilesPatterns,
+  generator,
+  ext = "md",
+}: Options["scanDirs"][0]): Promise<string> => {
+  const files = await new fdir()
+    .withFullPaths()
+    .filter((file) => {
+      const baseName = path.basename(file)
+
+      return isExtAllowed(baseName, ext) && !baseName.startsWith("_")
+    })
+    .filter(
+      (file) =>
+        !allowedFilesPatterns?.length ||
+        allowedFilesPatterns.some((pattern) => file.match(pattern))
+    )
+    .crawl(dir)
+    .withPromise()
+
+  const content: string[] =
+    generator?.name && generators[generator?.name]
+      ? [await generators[generator?.name](files, generator.options)]
+      : []
+
+  if (content.length) {
+    return await getCleanMd({
+      file: content.join("\n\n"),
+      ...options,
+      type: "content",
+    })
+  }
+
+  for (const file of files) {
+    content.push(
+      await getCleanMd({
+        file,
+        ...options,
+      })
+    )
+  }
+
+  return content.join("\n\n")
+}
+
+export const generateLlmsFull = async ({
+  outputPath,
+  scanDirs,
+  introText = "",
+  plugins,
+}: Options) => {
+  const text: string[] = [introText]
+
+  for (const scanDir of scanDirs) {
+    text.push(
+      await getContentFromDir({
+        ...scanDir,
+        options: {
+          plugins,
+          ...scanDir.options,
+        },
+      })
+    )
+  }
+
+  await writeFile(outputPath, text.join("\n\n"))
+}
@@ -1,4 +1,5 @@
 export * from "./generate-edited-dates.js"
+export * from "./generate-llms-full.js"
 export * from "./generate-sidebar.js"
 export * from "./retrieve-mdx-pages.js"

@@ -0,0 +1,127 @@
+import { findPageTitle, oasFileToPath } from "docs-utils"
+import { readFile } from "fs/promises"
+import path from "path"
+import pkg from "slugify"
+import YAML from "yaml"
+
+const slugify = pkg.default
+
+const monorepoRoot = path.resolve(process.cwd(), "..", "..", "..")
+const referencesRoot = path.join(monorepoRoot, "www", "apps", "resources")
+const slugsPath = path.join(
+  monorepoRoot,
+  "www",
+  "apps",
+  "resources",
+  "generated",
+  "slug-changes.mjs"
+)
+const slugsFileContentPrefix = `export const slugChanges = `
+
+export type CustomLlmsGenerator<T = Record<string, unknown>> = (
+  files: string[],
+  options?: T
+) => Promise<string>
+
+type CommonOptions = {
+  baseUrl?: string
+}
+
+export const workflowsLlmsGenerator: CustomLlmsGenerator<
+  CommonOptions
+> = async (files, options) => {
+  return generateListForReferenceFiles({
+    files,
+    title: "Workflows",
+    itemContent: (title, fileSlug) =>
+      `[${title.replace(/ - .+/, "")}](${fileSlug})`,
+    options,
+  })
+}
+
+export const stepsLlmsGenerator: CustomLlmsGenerator<CommonOptions> = async (
+  files,
+  options
+) => {
+  return generateListForReferenceFiles({
+    files,
+    title: "Steps",
+    itemContent: (title, fileSlug) =>
+      `[${title.replace(/ - .+/, "")}](${fileSlug})`,
+    options,
+  })
+}
+
+export const jsSdkLlmsGenerator: CustomLlmsGenerator<
+  CommonOptions & {
+    type: "Admin" | "Store" | "Auth"
+  }
+> = async (files, options) => {
+  return generateListForReferenceFiles({
+    files,
+    title: `JS SDK ${options?.type}`,
+    itemContent: (title, fileSlug) =>
+      `[${title.replace(/ - .+/, "")}](${fileSlug})`,
+    options,
+  })
+}
+
+export const apiRefLlmsGenerator: CustomLlmsGenerator<
+  CommonOptions & {
+    type: "Admin" | "Store"
+  }
+> = async (files, options) => {
+  let content = `## ${options?.type} API Reference\n\n`
+
+  for (const file of files) {
+    const baseName = path.basename(file)
+    const fileYaml = YAML.parse(await readFile(file, "utf-8"))
+
+    const oasPath = oasFileToPath(baseName)
+
+    Object.entries(fileYaml).forEach(
+      ([httpMethod, operation]: [string, any]) => {
+        const hash = `${slugify(operation.tags[0])}_${slugify(operation.operationId)}`
+
+        content += `- [${httpMethod.toUpperCase()} ${oasPath}](${options?.baseUrl}#${hash})\n`
+      }
+    )
+  }
+
+  return content
+}
+
+/**
+ * Helpers
+ */
+export const generateListForReferenceFiles = async ({
+  files,
+  title,
+  itemContent,
+  options,
+}: {
+  files: string[]
+  title: string
+  itemContent: (title: string, fileSlug: string) => string
+  options?: CommonOptions
+}) => {
+  const slugChanges = JSON.parse(
+    (await readFile(slugsPath, "utf-8")).replace(slugsFileContentPrefix, "")
+  )
+  let content = `## ${title}\n\n`
+  for (const file of files) {
+    const relativeFilePath = file.replace(monorepoRoot, "")
+    const fileSlug = `${options?.baseUrl}${
+      slugChanges.find(
+        (slugChange: any) => slugChange.filePath === relativeFilePath
+      )?.newSlug ||
+      file.replace(referencesRoot, "").replace(/\/page\.mdx?$/, "")
+    }/index.html.md`
+
+    const itemTitle = (findPageTitle(file) || "").replace(/ - .+/, "")
+
+    content += `- ${itemContent(itemTitle, fileSlug)}\n`
+  }
+
+  return content
+}