docs: add generator for llms-full.txt (#11323)

* initial

* improvements

* finished implementation

* transform links to index.html.md links

* fix for resources

www/packages/docs-utils/src/get-clean-md.ts

@@ -145,6 +145,22 @@ const removeFrontmatterPlugin = (): Transformer => {
   }
 }
 
+const changeLinksPlugin = (): Transformer => {
+  return async (tree) => {
+    const { visit } = await import("unist-util-visit")
+
+    visit(tree as UnistTree, ["link"], (node: UnistNode) => {
+      if (
+        node.type === "link" &&
+        node.url?.startsWith("https://docs.medusajs.com") &&
+        !node.url.endsWith("index.html.md")
+      ) {
+        node.url += `/index.html.md`
+      }
+    })
+  }
+}
+
 const getParsedAsString = (file: VFile): string => {
   let content = file.toString().replaceAll(/^([\s]*)\* /gm, "$1- ")
   const frontmatter = file.data.matter as FrontMatter | undefined
@@ -156,21 +172,23 @@ const getParsedAsString = (file: VFile): string => {
   return content
 }
 
-type Options = {
-  filePath: string
+export type GetCleanMdOptions = {
+  file: string
   plugins?: {
     before?: Plugin[]
     after?: Plugin[]
   }
   parserOptions?: ParserPluginOptions
+  type?: "file" | "content"
 }
 
 export const getCleanMd = async ({
-  filePath,
+  file,
   plugins,
   parserOptions,
-}: Options): Promise<string> => {
-  if (!filePath.endsWith(".md") && !filePath.endsWith(".mdx")) {
+  type = "file",
+}: GetCleanMdOptions): Promise<string> => {
+  if (type === "file" && !file.endsWith(".md") && !file.endsWith(".mdx")) {
     return ""
   }
   const unifier = unified()
@@ -196,7 +214,10 @@ export const getCleanMd = async ({
     unifier.use(...(Array.isArray(plugin) ? plugin : [plugin]))
   })
 
-  const parsed = await unifier.process(await read(filePath))
+  unifier.use(changeLinksPlugin)
+
+  const content = type === "file" ? await read(file) : file
+  const parsed = await unifier.process(content)
 
   return getParsedAsString(parsed)
 }

www/packages/docs-utils/src/index.ts

@@ -5,3 +5,4 @@ export * from "./get-clean-md.js"
 export * from "./get-file-slug-sync.js"
 export * from "./get-file-slug.js"
 export * from "./get-front-matter.js"
+export * from "./oas-file-to-path.js"

www/packages/docs-utils/src/oas-file-to-path.ts

@@ -0,0 +1,5 @@
+export function oasFileToPath(fileName: string): string {
+  return `/${fileName
+    .replaceAll(/(?<!\{[^}]*)_(?![^{]*\})/g, "/")
+    .replace(/\.[A-Za-z]+$/, "")}`
+}