docs: serve markdown to agents (#13915)

www/apps/book/app/md-content/[[...slug]]/route.ts

@@ -1,5 +1,5 @@
 import { getCleanMd } from "docs-utils"
-import { existsSync } from "fs"
+import { existsSync, readFileSync } from "fs"
 import { unstable_cache } from "next/cache"
 import { notFound } from "next/navigation"
 import { NextRequest, NextResponse } from "next/server"
@@ -16,7 +16,21 @@ type Params = {
 }
 
 export async function GET(req: NextRequest, { params }: Params) {
-  const { slug } = await params
+  const { slug = ["/"] } = await params
+
+  if (slug[0] === "/") {
+    const llmsFile = readFileSync(
+      path.join(process.cwd(), "public", "llms.txt"),
+      "utf-8"
+    )
+
+    return new NextResponse(llmsFile, {
+      headers: {
+        "Content-Type": "text/markdown",
+      },
+      status: 200,
+    })
+  }
 
   // keep this so that Vercel keeps the files in deployment
   const basePath = path.join(process.cwd(), "app")

www/apps/book/middleware.ts

@@ -1,13 +0,0 @@
-import { NextResponse } from "next/server"
-import type { NextRequest } from "next/server"
-
-export function middleware(request: NextRequest) {
-  const path = request.nextUrl.pathname.replace("/index.html.md", "")
-  return NextResponse.rewrite(
-    new URL(`/md-content${path.replace("/index.html.md", "")}`, request.url)
-  )
-}
-
-export const config = {
-  matcher: "/((?!resources|api|ui|user-guide|cloud).*)index.html.md",
-}

www/apps/book/next.config.mjs

@@ -115,6 +115,35 @@ const nextConfig = {
   transpilePackages: ["docs-ui"],
   async rewrites() {
     return {
+      beforeFiles: [
+        {
+          source:
+            "/:path((?!resources|api|ui|user-guide|cloud).*)index.html.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "Accept",
+              value: "text/markdown",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "Accept",
+              value: "text/plain",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+      ],
       fallback: [
         {
           source: "/resources",

www/apps/cloud/middleware.ts

@@ -1,15 +0,0 @@
-import { NextResponse } from "next/server"
-import type { NextRequest } from "next/server"
-
-export function middleware(request: NextRequest) {
-  return NextResponse.rewrite(
-    new URL(
-      `${request.nextUrl.basePath}/md-content${request.nextUrl.pathname.replace("/index.html.md", "")}`,
-      request.url
-    )
-  )
-}
-
-export const config = {
-  matcher: "/:path*/index.html.md",
-}

www/apps/cloud/next.config.mjs

@@ -152,6 +152,38 @@ const nextConfig = {
       permanent: true,
     },
   ],
+  rewrites: async () => {
+    return {
+      beforeFiles: [
+        {
+          source: "/:path*/index.html.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/markdown",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          destination: "/md-content/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/plain",
+            },
+          ],
+        },
+      ],
+    }
+  },
 }
 
 const withBundleAnalyzer = bundleAnalyzer({

www/apps/resources/middleware.ts

@@ -1,15 +0,0 @@
-import { NextResponse } from "next/server"
-import type { NextRequest } from "next/server"
-
-export function middleware(request: NextRequest) {
-  return NextResponse.rewrite(
-    new URL(
-      `${request.nextUrl.basePath}/md-content${request.nextUrl.pathname.replace("/index.html.md", "")}`,
-      request.url
-    )
-  )
-}
-
-export const config = {
-  matcher: "/:path*/index.html.md",
-}

www/apps/resources/next.config.mjs

@@ -286,6 +286,38 @@ const nextConfig = {
   experimental: {
     optimizePackageImports: ["@medusajs/icons", "@medusajs/ui", "elkjs"],
   },
+  rewrites: async () => {
+    return {
+      beforeFiles: [
+        {
+          source: "/:path*/index.html.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/markdown",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          destination: "/md-content/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/plain",
+            },
+          ],
+        },
+      ],
+    }
+  },
 }
 
 const withBundleAnalyzer = bundleAnalyzer({

www/apps/ui/middleware.ts

@@ -1,15 +0,0 @@
-import { NextResponse } from "next/server"
-import type { NextRequest } from "next/server"
-
-export function middleware(request: NextRequest) {
-  return NextResponse.rewrite(
-    new URL(
-      `${request.nextUrl.basePath}/md-content${request.nextUrl.pathname.replace("/index.html.md", "")}`,
-      request.url
-    )
-  )
-}
-
-export const config = {
-  matcher: "/:path*/index.html.md",
-}

www/apps/ui/next.config.mjs

@@ -141,6 +141,38 @@ const nextConfig = {
   experimental: {
     optimizePackageImports: ["@medusajs/icons", "@medusajs/ui"],
   },
+  rewrites: async () => {
+    return {
+      beforeFiles: [
+        {
+          source: "/:path*/index.html.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/markdown",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          destination: "/md-content/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/plain",
+            },
+          ],
+        },
+      ],
+    }
+  },
 }
 
 const withBundleAnalyzer = bundleAnalyzer({

www/apps/user-guide/middleware.ts

@@ -1,15 +0,0 @@
-import { NextResponse } from "next/server"
-import type { NextRequest } from "next/server"
-
-export function middleware(request: NextRequest) {
-  return NextResponse.rewrite(
-    new URL(
-      `${request.nextUrl.basePath}/md-content${request.nextUrl.pathname.replace("/index.html.md", "")}`,
-      request.url
-    )
-  )
-}
-
-export const config = {
-  matcher: "/:path*/index.html.md",
-}

www/apps/user-guide/next.config.mjs

@@ -129,6 +129,38 @@ const nextConfig = {
   experimental: {
     optimizePackageImports: ["@medusajs/icons", "@medusajs/ui"],
   },
+  rewrites: async () => {
+    return {
+      beforeFiles: [
+        {
+          source: "/:path*/index.html.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/markdown",
+            },
+          ],
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*",
+          destination: "/md-content/:path*",
+          has: [
+            {
+              type: "header",
+              key: "accept",
+              value: "text/plain",
+            },
+          ],
+        },
+      ],
+    }
+  },
 }
 
 const withBundleAnalyzer = bundleAnalyzer({