docs: add tags package to generate tags (#10666)

* initial changes

* finalize implementation

* run generator on prep

* remove tags package from book

* optimization attempt

* test transpile

* test transpile

* rename utils

* rename directory

* add vercel ignore

* add tags to book

www/packages/build-scripts/package.json

@@ -24,13 +24,15 @@
   ],
   "scripts": {
     "build": "yarn clean && tsc",
-    "clean": "rimraf dist"
+    "clean": "rimraf dist",
+    "watch": "tsc --watch"
   },
   "dependencies": {
     "remark-rehype-plugins": "*"
   },
   "devDependencies": {
     "@types/node": "^20.11.20",
+    "docs-utils": "*",
     "rimraf": "^5.0.5",
     "tsconfig": "*",
     "types": "*",

www/packages/build-scripts/src/index.ts

@@ -2,8 +2,6 @@ export * from "./generate-edited-dates.js"
 export * from "./generate-sidebar.js"
 export * from "./retrieve-mdx-pages.js"
 
-export * from "./utils/find-metadata-title.js"
-export * from "./utils/find-page-heading.js"
 export * from "./utils/get-core-flows-ref-sidebar-children.js"
 export * from "./utils/get-sidebar-item-link.js"
 export * from "./utils/sidebar-attach-href-common-options.js"

www/packages/build-scripts/src/retrieve-mdx-pages.ts

@@ -1,6 +1,6 @@
 import { readdirSync } from "fs"
 import path from "path"
-import { getFileSlugSyncUtil } from "remark-rehype-plugins"
+import { getFileSlugSync } from "../../docs-utils/dist/index.js"
 
 type Options = {
   basePath: string
@@ -24,7 +24,7 @@ export function retrieveMdxPages({ basePath }: Options): string[] {
         continue
       }
 
-      const slug = getFileSlugSyncUtil(filePath)
+      const slug = getFileSlugSync(filePath)
 
       urls.push(
         slug ||

www/packages/build-scripts/src/utils/find-metadata-title.ts

@@ -1,7 +0,0 @@
-const REGEX = /export const metadata = {[\s\S]*title: `(?<title>.*)`/
-
-export default function findMetadataTitle(content: string): string | undefined {
-  const headingMatch = REGEX.exec(content)
-
-  return headingMatch?.groups?.title
-}

www/packages/build-scripts/src/utils/find-page-heading.ts

@@ -1,7 +0,0 @@
-const HEADING_REGEX = /# (?<title>.*)/
-
-export default function findPageHeading(content: string): string | undefined {
-  const headingMatch = HEADING_REGEX.exec(content)
-
-  return headingMatch?.groups?.title
-}

www/packages/build-scripts/src/utils/get-sidebar-item-link.ts

@@ -1,8 +1,5 @@
-import { getFrontMatterUtil } from "remark-rehype-plugins"
+import { getFrontMatter, findPageTitle } from "../../../docs-utils/dist/index.js"
 import { ItemsToAdd, sidebarAttachHrefCommonOptions } from "../index.js"
-import { readFileSync } from "fs"
-import findMetadataTitle from "./find-metadata-title.js"
-import findPageHeading from "./find-page-heading.js"
 import { InteractiveSidebarItem } from "types"
 
 export async function getSidebarItemLink({
@@ -14,26 +11,18 @@ export async function getSidebarItemLink({
   basePath: string
   fileBasename: string
 }): Promise<ItemsToAdd | undefined> {
-  const frontmatter = await getFrontMatterUtil(filePath)
+  const frontmatter = await getFrontMatter(filePath)
   if (frontmatter.sidebar_autogenerate_exclude) {
     return
   }
 
-  const fileContent = frontmatter.sidebar_label
-    ? ""
-    : readFileSync(filePath, "utf-8")
-
   const newItem = sidebarAttachHrefCommonOptions([
     {
       type: "link",
       path:
         frontmatter.slug ||
         filePath.replace(basePath, "").replace(`/${fileBasename}`, ""),
-      title:
-        frontmatter.sidebar_label ||
-        findMetadataTitle(fileContent) ||
-        findPageHeading(fileContent) ||
-        "",
+      title: frontmatter.sidebar_label || findPageTitle(filePath) || "",
     },
   ])[0] as InteractiveSidebarItem
 

www/packages/build-scripts/tsconfig.json

@@ -6,6 +6,7 @@
     "tsBuildInfoFile": "./dist/.tsbuildinfo-client",
     "noEmit": false,
     "lib": ["es2022"],
+    "target": "es2022",
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "declaration": true,

www/packages/docs-utils/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "docs-utils",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "yarn clean && tsc",
+    "clean": "rimraf dist",
+    "watch": "tsc --watch"
+  },
+  "dependencies": {
+    "remark-frontmatter": "^5.0.0",
+    "remark-parse": "^11.0.0",
+    "remark-stringify": "^11.0.0",
+    "to-vfile": "^8.0.0",
+    "unified": "^11.0.4",
+    "vfile-matter": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.20",
+    "rimraf": "^5.0.5",
+    "tsconfig": "*",
+    "types": "*",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=18.17.0"
+  }
+}

www/packages/docs-utils/src/find-title.ts

@@ -0,0 +1,23 @@
+import { readFileSync } from "fs"
+
+const REGEX = /export const metadata = {[\s\S]*title: `(?<title>.*)`/
+
+export function findMetadataTitle(content: string): string | undefined {
+  const headingMatch = REGEX.exec(content)
+
+  return headingMatch?.groups?.title
+}
+
+const HEADING_REGEX = /# (?<title>.*)/
+
+export function findPageHeading(content: string): string | undefined {
+  const headingMatch = HEADING_REGEX.exec(content)
+
+  return headingMatch?.groups?.title
+}
+
+export function findPageTitle(filePath: string): string | undefined {
+  const content = readFileSync(filePath, "utf-8")
+
+  return findMetadataTitle(content) || findPageHeading(content)
+}

www/packages/docs-utils/src/get-file-slug.ts

@@ -1,12 +1,12 @@
-import { getFrontMatterUtil } from "./get-front-matter.js"
 import { matter } from "vfile-matter"
 import { readSync } from "to-vfile"
-import { FrontMatter } from "../types/index.js"
+import { FrontMatter } from "types"
+import { getFrontMatter } from "./get-front-matter.js"
 
-export async function getFileSlugUtil(
+export async function getFileSlug(
   filePath: string
 ): Promise<string | undefined> {
-  const fileFrontmatter = await getFrontMatterUtil(filePath)
+  const fileFrontmatter = await getFrontMatter(filePath)
 
   if (fileFrontmatter.slug) {
     // add to slugs array
@@ -14,7 +14,7 @@ export async function getFileSlugUtil(
   }
 }
 
-export function getFileSlugSyncUtil(filePath: string): string | undefined {
+export function getFileSlugSync(filePath: string): string | undefined {
   const content = readSync(filePath)
 
   matter(content)

www/packages/docs-utils/src/get-front-matter.ts

@@ -2,13 +2,11 @@ import remarkFrontmatter from "remark-frontmatter"
 import remarkParse from "remark-parse"
 import remarkStringify from "remark-stringify"
 import { unified } from "unified"
-import { read } from "to-vfile"
+import { read, readSync } from "to-vfile"
 import { matter } from "vfile-matter"
-import { FrontMatter } from "../types/index.js"
+import { FrontMatter } from "types"
 
-export async function getFrontMatterUtil(
-  filePath: string
-): Promise<FrontMatter> {
+export async function getFrontMatter(filePath: string): Promise<FrontMatter> {
   return (
     await unified()
       .use(remarkParse)
@@ -22,3 +20,11 @@ export async function getFrontMatterUtil(
       .process(await read(filePath))
   ).data.matter as FrontMatter
 }
+
+export function getFrontMatterSync(filePath: string): FrontMatter {
+  const content = readSync(filePath)
+
+  matter(content)
+
+  return content.data.matter as FrontMatter
+}

www/packages/docs-utils/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./find-title.js"
+export * from "./get-file-slug.js"
+export * from "./get-front-matter.js"

www/packages/docs-utils/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": ["tsconfig/base.json"],
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo-client",
+    "noEmit": false,
+    "lib": ["es2022"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "target": "ESNext",
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src"]
+}

www/packages/remark-rehype-plugins/package.json

@@ -24,10 +24,12 @@
   ],
   "scripts": {
     "build": "yarn clean && tsc",
-    "clean": "rimraf dist"
+    "clean": "rimraf dist",
+    "watch": "tsc --watch"
   },
   "dependencies": {
     "@cloudinary/url-gen": "^1.17.0",
+    "docs-utils": "*",
     "remark-frontmatter": "^5.0.0",
     "remark-parse": "^11.0.0",
     "remark-stringify": "^11.0.0",

www/packages/remark-rehype-plugins/src/index.ts

@@ -9,5 +9,3 @@ export * from "./type-list-link-fixer.js"
 export * from "./workflow-diagram-link-fixer.js"
 
 export * from "./utils/fix-link.js"
-export * from "./utils/get-file-slug.js"
-export * from "./utils/get-front-matter.js"

www/packages/remark-rehype-plugins/src/types/index.ts

@@ -98,15 +98,6 @@ export declare type CloudinaryConfig = {
   roundCorners?: number
 }
 
-export declare type FrontMatter = {
-  slug?: string
-  sidebar_label?: string
-  sidebar_group?: string
-  sidebar_group_main?: boolean
-  sidebar_position?: number
-  sidebar_autogenerate_exclude?: boolean
-}
-
 export declare type CrossProjectLinksOptions = {
   baseUrl: string
   projectUrls?: {

www/packages/remark-rehype-plugins/src/utils/fix-link.ts

@@ -1,5 +1,5 @@
 import path from "path"
-import { getFileSlugSyncUtil } from "./get-file-slug.js"
+import { getFileSlugSync } from "../../../docs-utils/dist/index.js"
 
 export type FixLinkOptions = {
   currentPageFilePath: string
@@ -20,7 +20,7 @@ export function fixLinkUtil({
   fullLinkedFilePath = fullLinkedFilePath.replace(hash, "")
   // get absolute path of the URL
   const linkedFilePath = fullLinkedFilePath.replace(basePath, "")
-  const linkedFileSlug = getFileSlugSyncUtil(fullLinkedFilePath)
+  const linkedFileSlug = getFileSlugSync(fullLinkedFilePath)
 
   const newLink =
     linkedFileSlug ||

www/packages/tags/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "tags",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "yarn clean && yarn generate:tags && tsc",
+    "generate:tags": "node --loader ts-node/esm src/scripts/generate-tags.ts",
+    "clean": "rimraf dist",
+    "watch": "tsc --watch"
+  },
+  "dependencies": {
+    "docs-utils": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.20",
+    "rimraf": "^5.0.5",
+    "ts-node": "^10.9.1",
+    "tsconfig": "*",
+    "types": "*",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=18.17.0"
+  }
+}

www/packages/tags/src/index.ts

@@ -0,0 +1 @@
+export * from "./utils/index.js"

www/packages/tags/src/scripts/generate-tags.ts

@@ -0,0 +1,3 @@
+import { generateTags } from "../utils/generate-tags.js"
+
+void generateTags()

www/packages/tags/src/utils/generate-tags.ts

@@ -0,0 +1,119 @@
+import { statSync } from "fs"
+import { mkdir, readdir, rm, writeFile } from "fs/promises"
+import path from "path"
+import type { Tags } from "types"
+import { findPageTitle, getFrontMatterSync } from "docs-utils"
+
+type ConfigItem = {
+  path: string
+  contentPaths: string[]
+}
+
+const config: ConfigItem[] = [
+  {
+    path: path.resolve("..", "..", "apps", "book"),
+    contentPaths: ["app"],
+  },
+  {
+    path: path.resolve("..", "..", "apps", "resources"),
+    contentPaths: ["app", "references"],
+  },
+  {
+    path: path.resolve("..", "..", "apps", "ui"),
+    contentPaths: [path.join("src", "content", "docs")],
+  },
+  {
+    path: path.resolve("..", "..", "apps", "user-guide"),
+    contentPaths: ["app"],
+  },
+]
+
+function normalizePageTitle(title: string): string {
+  // remove variables from title
+  return title.replaceAll(/\$\{.+\}/g, "").trim()
+}
+
+function tagNameToFileName(tagName: string): string {
+  return `${tagName.toLowerCase().replaceAll(" ", "-")}.ts`
+}
+
+function tagNameToVarName(tagName: string): string {
+  return tagName
+    .toLowerCase()
+    .replaceAll(/\s([a-zA-Z\d])/g, (captured) => captured.toUpperCase().trim())
+}
+
+export async function generateTags(basePath?: string) {
+  basePath = basePath || path.resolve()
+  const tags: Tags = {}
+  async function getTags(item: ConfigItem) {
+    async function scanDirectory(dirPath: string) {
+      const files = await readdir(dirPath)
+
+      for (const file of files) {
+        const fullPath = path.join(dirPath, file)
+        if (!file.endsWith(".mdx") || file.startsWith("_")) {
+          if (statSync(fullPath).isDirectory()) {
+            await scanDirectory(fullPath)
+          }
+          continue
+        }
+
+        const frontmatter = getFrontMatterSync(fullPath)
+        const fileBasename = path.basename(file)
+
+        frontmatter.tags?.forEach((tag) => {
+          if (!Object.hasOwn(tags, tag)) {
+            tags[tag] = []
+          }
+
+          tags[tag].push({
+            title: normalizePageTitle(
+              frontmatter.sidebar_label || findPageTitle(fullPath) || ""
+            ),
+            path:
+              frontmatter.slug ||
+              fullPath.replace(item.path, "").replace(`/${fileBasename}`, ""),
+          })
+        })
+      }
+    }
+
+    for (const contentPath of item.contentPaths) {
+      const basePath = path.join(item.path, contentPath)
+
+      await scanDirectory(basePath)
+    }
+  }
+
+  await Promise.all(
+    config.map(async (item) => {
+      await getTags(item)
+    })
+  )
+
+  const tagsDir = path.join(basePath, "src", "tags")
+  // clear existing tags
+  await rm(tagsDir, {
+    recursive: true,
+    force: true,
+  })
+  await mkdir(tagsDir)
+  // write tags
+  const files: string[] = []
+  await Promise.all(
+    Object.keys(tags).map(async (tagName) => {
+      const fileName = tagNameToFileName(tagName)
+      const varName = tagNameToVarName(tagName)
+
+      const content = `export const ${varName} = ${JSON.stringify(tags[tagName], null, 2)}`
+
+      await writeFile(path.join(tagsDir, fileName), content)
+      files.push(fileName.replace(/\.ts$/, ".js"))
+    })
+  )
+
+  // write index.ts
+  const indexContent = files.map((file) => `export * from "./${file}"\n`)
+  await writeFile(path.join(tagsDir, "index.ts"), indexContent)
+}

www/packages/tags/src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from "./generate-tags.js"
+export * from "./tags.js"

www/packages/tags/src/utils/tags.ts

@@ -0,0 +1,13 @@
+import { Tag, Tags } from "types"
+import * as tags from "../tags/index.js"
+
+export const getTagItems = (tagName: string): Tag | undefined => {
+  if (!Object.hasOwn(tags, tagName)) {
+    return
+  }
+  return tags[tagName as keyof typeof tags]
+}
+
+export const getAllTags = (): Tags => {
+  return tags
+}

www/packages/tags/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "extends": "tsconfig/base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo-client",
+    "noEmit": false,
+    "lib": ["es2022"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "target": "ESNext",
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src"],
+  "ts-node": {
+    "esm": true,
+    "experimentalSpecifierResolution": "node",
+    "transpileOnly": true
+  }
+}

www/packages/types/package.json

@@ -28,7 +28,8 @@
   ],
   "scripts": {
     "build": "yarn clean && tsc",
-    "clean": "rimraf dist"
+    "clean": "rimraf dist",
+    "watch": "tsc --watch"
   },
   "devDependencies": {
     "@types/node": "^20.11.20",

www/packages/types/src/frontmatter.ts

@@ -0,0 +1,9 @@
+export declare type FrontMatter = {
+  slug?: string
+  sidebar_label?: string
+  sidebar_group?: string
+  sidebar_group_main?: boolean
+  sidebar_position?: number
+  sidebar_autogenerate_exclude?: boolean
+  tags?: string[]
+}

www/packages/types/src/index.ts

@@ -1,10 +1,12 @@
 export * from "./api-testing.js"
 export * from "./build-scripts.js"
 export * from "./config.js"
+export * from "./frontmatter.js"
 export * from "./general.js"
 export * from "./menu.js"
 export * from "./navigation.js"
 export * from "./navigation-dropdown.js"
 export * from "./sidebar.js"
+export * from "./tags.js"
 export * from "./toc.js"
 export * from "./workflow.js"

www/packages/types/src/tags.ts

@@ -0,0 +1,8 @@
+export type Tag = {
+  title: string
+  path: string
+}[]
+
+export type Tags = {
+  [k: string]: Tag
+}