docs: add util to generate clean markdown for a file (#11303)

This commit is contained in:
Shahed Nasser
2025-02-04 16:44:17 +02:00
committed by GitHub
parent 3c51709daf
commit 604f46f8bc
23 changed files with 832 additions and 37 deletions
+2
View File
@@ -28,7 +28,9 @@
"watch": "tsc --watch"
},
"dependencies": {
"@mdx-js/mdx": "^3.1.0",
"remark-frontmatter": "^5.0.0",
"remark-mdx": "^3.1.0",
"remark-parse": "^11.0.0",
"remark-stringify": "^11.0.0",
"to-vfile": "^8.0.0",
@@ -0,0 +1,70 @@
/* eslint-disable no-case-declarations */
import {
ArrayExpression,
Estree,
Expression,
ExpressionJsVar,
ExpressionJsVarLiteral,
LiteralExpression,
ObjectExpression,
} from "types"
export function estreeToJs(estree: Estree) {
// TODO improve on this utility. Currently it's implemented to work
// for specific use cases as we don't have a lot of info on other
// use cases.
if (
!estree.body?.length ||
estree.body[0].type !== "ExpressionStatement" ||
!estree.body[0].expression
) {
return
}
return expressionToJs(estree.body[0].expression)
}
function expressionToJs(
expression: Expression
): ExpressionJsVar | ExpressionJsVar[] | undefined {
switch (expression.type) {
case "ArrayExpression":
const arrVar: ExpressionJsVar[] = []
;(expression as ArrayExpression).elements.forEach((elm) => {
const elmJsVar = expressionToJs(elm)
if (!elmJsVar) {
return
}
if (Array.isArray(elmJsVar)) {
arrVar.push(...elmJsVar)
} else {
arrVar.push(elmJsVar)
}
})
return arrVar
case "ObjectExpression":
const objVar: ExpressionJsVar = {}
;(expression as ObjectExpression).properties.forEach((property) => {
const keyName = property.key.name ?? property.key.value
if (!keyName) {
return
}
const jsVal = expressionToJs(property.value)
if (!jsVal) {
return
}
objVar[keyName] = jsVal
})
return objVar
case "Literal":
return {
original: expression,
data: (expression as LiteralExpression).value,
} as ExpressionJsVarLiteral
case "JSXElement":
// ignore JSXElements
return
}
}
@@ -0,0 +1,21 @@
import { ExpressionJsVarLiteral, ExpressionJsVarObj } from "types"
export function isExpressionJsVarLiteral(
expression: unknown
): expression is ExpressionJsVarLiteral {
return (
typeof expression === "object" &&
expression !== null &&
Object.hasOwn(expression, "original")
)
}
export function isExpressionJsVarObj(
expression: unknown
): expression is ExpressionJsVarObj {
return (
typeof expression === "object" &&
expression !== null &&
!Object.hasOwn(expression, "original")
)
}
+136
View File
@@ -0,0 +1,136 @@
import remarkMdx from "remark-mdx"
import remarkParse from "remark-parse"
import remarkStringify from "remark-stringify"
import { read } from "to-vfile"
import { UnistNode, UnistNodeWithData, UnistTree } from "types"
import { Plugin, Transformer, unified } from "unified"
import { SKIP } from "unist-util-visit"
import type { VFile } from "vfile"
import {
parseCard,
parseCardList,
parseCodeTabs,
parseDetails,
parseNote,
parsePrerequisites,
parseSourceCodeLink,
parseTable,
parseTabs,
parseTypeList,
parseWorkflowDiagram,
} from "./utils/parse-elms.js"
const parseComponentsPlugin = (): Transformer => {
return async (tree) => {
const { visit } = await import("unist-util-visit")
let pageTitle = ""
visit(
tree as UnistTree,
["mdxJsxFlowElement", "element", "mdxjsEsm", "heading"],
(node: UnistNode, index, parent) => {
if (typeof index !== "number" || !parent) {
return
}
if (
node.type === "mdxjsEsm" &&
node.value?.startsWith("export const metadata = ") &&
node.data &&
"estree" in node.data
) {
const regexMatch = /title: (?<title>.+),?/.exec(node.value)
if (regexMatch?.groups?.title) {
pageTitle = regexMatch.groups.title
.replace(/,$/, "")
.replaceAll(/\$\{.+\}/g, "")
.replaceAll(/^['"`]/g, "")
.replaceAll(/['"`]$/g, "")
.trim()
}
}
if (node.type === "heading") {
if (
node.depth === 1 &&
node.children?.length &&
node.children[0].value === "metadata.title"
) {
node.children[0] = {
type: "text",
value: pageTitle,
}
}
return
}
if (
node.type === "mdxjsEsm" ||
node.name === "Feedback" ||
node.name === "ChildDocs" ||
node.name === "DetailsList"
) {
parent?.children.splice(index, 1)
return [SKIP, index]
}
switch (node.name) {
case "Card":
return parseCard(node, index, parent)
case "CardList":
return parseCardList(node as UnistNodeWithData, index, parent)
case "CodeTabs":
return parseCodeTabs(node as UnistNodeWithData, index, parent)
case "Details":
return parseDetails(node as UnistNodeWithData, index, parent)
case "Note":
return parseNote(node, index, parent)
case "Prerequisites":
return parsePrerequisites(node as UnistNodeWithData, index, parent)
case "SourceCodeLink":
return parseSourceCodeLink(node as UnistNodeWithData, index, parent)
case "Table":
return parseTable(node as UnistNodeWithData, index, parent)
case "Tabs":
return parseTabs(node as UnistNodeWithData, index, parent)
case "TypeList":
return parseTypeList(node as UnistNodeWithData, index, parent)
case "WorkflowDiagram":
return parseWorkflowDiagram(
node as UnistNodeWithData,
index,
parent
)
}
}
)
}
}
const getParsedAsString = (file: VFile): string => {
return file.toString().replaceAll(/^([\s]*)\* /gm, "$1- ")
}
export const getCleanMd = async (
filePath: string,
plugins?: {
before?: Plugin[]
after?: Plugin[]
}
): Promise<string> => {
if (!filePath.endsWith(".md") && !filePath.endsWith(".mdx")) {
return ""
}
const unifier = unified().use(remarkParse).use(remarkMdx).use(remarkStringify)
plugins?.before?.forEach((plugin) => {
unifier.use(...(Array.isArray(plugin) ? plugin : [plugin]))
})
unifier.use(parseComponentsPlugin)
plugins?.after?.forEach((plugin) => {
unifier.use(...(Array.isArray(plugin) ? plugin : [plugin]))
})
const parsed = await unifier.process(await read(filePath))
return getParsedAsString(parsed)
}
+3
View File
@@ -1,4 +1,7 @@
export * from "./estree-to-js.js"
export * from "./expression-is-utils.js"
export * from "./find-title.js"
export * from "./get-clean-md.js"
export * from "./get-file-slug-sync.js"
export * from "./get-file-slug.js"
export * from "./get-front-matter.js"
@@ -0,0 +1,597 @@
import { ExpressionJsVar, UnistNode, UnistNodeWithData, UnistTree } from "types"
import { SKIP, VisitorResult } from "unist-util-visit"
import { estreeToJs } from "../estree-to-js.js"
import {
isExpressionJsVarLiteral,
isExpressionJsVarObj,
} from "../expression-is-utils.js"
export const parseCard = (
node: UnistNode,
index: number,
parent: UnistTree
): VisitorResult => {
let title: string | undefined,
text: string | undefined,
href: string | undefined
node.attributes?.some((attr) => {
if (title && text && href) {
return true
}
if (attr.name === "title") {
title = attr.value as string
} else if (attr.name === "text") {
text = attr.value as string
} else if (attr.name === "href") {
href = attr.value as string
}
return false
})
if (!title || !href) {
return
}
parent?.children.splice(index, 1, {
type: "paragraph",
children: [
{
type: "link",
url: href,
children: [
{
type: "text",
value: title,
},
],
},
{
type: "text",
value: `: ${text}`,
},
],
})
return [SKIP, index]
}
export const parseCardList = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const items = node.attributes?.find((attr) => attr.name === "items")
if (!items || typeof items.value === "string" || !items.value.data?.estree) {
return
}
const itemsJsVar = estreeToJs(items.value.data.estree)
if (!itemsJsVar || !Array.isArray(itemsJsVar)) {
return
}
const listItems = itemsJsVar
.map((item) => {
if (
!isExpressionJsVarObj(item) ||
!("text" in item) ||
!("link" in item) ||
!isExpressionJsVarLiteral(item.text) ||
!isExpressionJsVarLiteral(item.link)
) {
return null
}
return {
type: "listItem",
children: [
{
type: "paragraph",
children: [
{
type: "link",
url: `#${item.link.data}`,
children: [
{
type: "text",
value: item.text.data,
},
],
},
],
},
],
}
})
.filter(Boolean) as UnistNode[]
parent?.children.splice(index, 1, {
type: "list",
ordered: false,
spread: false,
children: listItems,
})
return [SKIP, index]
}
export const parseCodeTabs = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const tabs = node.children?.filter((child) => child.name === "CodeTab")
if (!tabs) {
return
}
const children: UnistNode[] = []
tabs.forEach((tab) => {
const label = tab.attributes?.find((attr) => attr.name === "label")
const code = tab.children?.find((child) => child.type === "code")
if (!label || !code) {
return
}
children.push({
type: "mdxJsxFlowElement",
name: "details",
children: [
{
type: "mdxJsxFlowElement",
name: "summary",
children: [
{
type: "text",
value: (label.value as string) || "summary",
},
],
},
code,
],
})
})
parent?.children.splice(index, 1, ...children)
return [SKIP, index]
}
export const parseDetails = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const summary = node.attributes?.find(
(attr) => attr.name === "summaryContent"
)
parent?.children.splice(index, 1, {
type: "mdxJsxFlowElement",
name: "details",
children: [
{
type: "mdxJsxFlowElement",
name: "summary",
children: [
{
type: "text",
value: (summary?.value as string) || "Details",
},
],
},
...(node.children || []),
],
})
return [SKIP, index]
}
export const parseNote = (
node: UnistNode,
index: number,
parent: UnistTree
): VisitorResult => {
parent.children?.splice(index, 1, ...(node.children || []))
return [SKIP, index]
}
export const parsePrerequisites = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const items = node.attributes?.find((attr) => attr.name === "items")
if (!items || typeof items.value === "string" || !items.value.data?.estree) {
return
}
const itemsJsVar = estreeToJs(items.value.data.estree)
if (!itemsJsVar || !Array.isArray(itemsJsVar)) {
return
}
const listItems = itemsJsVar
.map((item) => {
if (
!isExpressionJsVarObj(item) ||
!("text" in item) ||
!("link" in item) ||
!isExpressionJsVarLiteral(item.text) ||
!isExpressionJsVarLiteral(item.link)
) {
return null
}
return {
type: "listItem",
children: [
{
type: "paragraph",
children: [
{
type: "link",
url: `${item.link.data}`,
children: [
{
type: "text",
value: item.text.data,
},
],
},
],
},
],
}
})
.filter(Boolean) as UnistNode[]
parent?.children.splice(
index,
1,
{
type: "heading",
depth: 3,
children: [
{
type: "text",
value: "Prerequisites",
},
],
},
{
type: "list",
ordered: false,
spread: false,
children: listItems,
}
)
return [SKIP, index]
}
export const parseSourceCodeLink = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const link = node.attributes?.find((attr) => attr.name === "link")
if (!link) {
return
}
parent?.children.splice(index, 1, {
type: "paragraph",
children: [
{
type: "link",
url: link?.value as string,
children: [
{
type: "text",
value: "Source code",
},
],
},
],
})
return [SKIP, index]
}
export const parseTable = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const headerNode = node.children?.find(
(child) => child.name === "Table.Header"
)
const bodyNode = node.children?.find((child) => child.name === "Table.Body")
let nodeText = ``
let headerCellsCount = 0
headerNode?.children?.forEach((headerRow, rowIndex) => {
if (rowIndex > 0) {
nodeText += `\n`
}
const childCells =
headerRow.children?.length === 1 &&
headerRow.children[0].type === "paragraph"
? headerRow.children[0].children
: headerRow.children
headerCellsCount = childCells?.length || 0
childCells?.forEach((headerCell) => {
if (headerCell.name !== "Table.HeaderCell") {
return
}
nodeText += `|`
nodeText += formatNodeText(getTextNode(headerCell))
})
nodeText += `|`
})
nodeText += `\n|${new Array(headerCellsCount).fill(`---`).join(`|`)}|`
bodyNode?.children?.forEach((bodyRow) => {
nodeText += `\n`
bodyRow.children?.forEach((bodyCell) => {
nodeText += `|`
nodeText += formatNodeText(getTextNode(bodyCell))
})
nodeText += `|`
})
parent.children?.splice(index, 1, {
type: "paragraph",
children: [
{
type: "text",
value: nodeText,
},
],
})
}
export const parseTabs = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
if ((node.children?.length || 0) < 2) {
return
}
const tabs: UnistNode[] = []
node.children![0].children?.forEach((tabList) => {
tabList.children?.forEach((tabTrigger, index) => {
const tabContentNode = node.children?.[1].children?.[index]
const tabLabel = formatNodeText(getTextNode(tabTrigger))
const tabContent = tabContentNode?.children || []
if (!tabLabel || !tabContent) {
return
}
tabs.push({
type: "mdxJsxFlowElement",
name: "details",
children: [
{
type: "mdxJsxFlowElement",
name: "summary",
children: [
{
type: "text",
value: tabLabel,
},
],
},
...tabContent,
],
})
})
})
parent.children?.splice(index, 1, ...tabs)
return [SKIP, index]
}
export const parseTypeList = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const types = node.attributes?.find((attr) => attr.name === "types")
if (!types || typeof types.value === "string" || !types.value.data?.estree) {
return
}
const typesJsVar = estreeToJs(types.value.data.estree)
if (!typesJsVar || !Array.isArray(typesJsVar)) {
return
}
const generateTypeListItems = (
typesJsVar: ExpressionJsVar[]
): UnistNode[] => {
const listItems: UnistNode[] = []
typesJsVar.forEach((item) => {
if (!isExpressionJsVarObj(item)) {
return
}
const typeName = isExpressionJsVarLiteral(item.name) ? item.name.data : ""
const itemType = isExpressionJsVarLiteral(item.type) ? item.type.data : ""
const itemDescription = isExpressionJsVarLiteral(item.description)
? item.description.data
: ""
if (!typeName || !itemType) {
return
}
const itemListChildren = item.children || []
const children: UnistNode[] = [
{
type: "paragraph",
children: [
{
type: "text",
value: `${typeName}: (${itemType}) ${itemDescription}`,
},
],
},
]
if (Array.isArray(itemListChildren) && itemListChildren.length) {
children.push(...generateTypeListItems(itemListChildren))
}
listItems.push({
type: "listItem",
children,
})
})
return listItems
}
const listItems = generateTypeListItems(typesJsVar)
parent?.children.splice(index, 1, {
type: "list",
ordered: false,
spread: false,
children: listItems,
})
return [SKIP, index]
}
export const parseWorkflowDiagram = (
node: UnistNodeWithData,
index: number,
parent: UnistTree
): VisitorResult => {
const worflowItems = node.attributes?.find((attr) => attr.name === "workflow")
if (
!worflowItems ||
typeof worflowItems.value === "string" ||
!worflowItems.value.data?.estree
) {
return
}
const workflowJsVar = estreeToJs(worflowItems.value.data.estree)
if (
!isExpressionJsVarObj(workflowJsVar) ||
!("steps" in workflowJsVar) ||
!Array.isArray(workflowJsVar.steps)
) {
return
}
const generateWorkflowItems = (
jsVarItems: ExpressionJsVar[]
): UnistNode[] => {
const listItems: UnistNode[] = []
jsVarItems.forEach((item) => {
if (!isExpressionJsVarObj(item)) {
return
}
const stepName = isExpressionJsVarLiteral(item.name) ? item.name.data : ""
const stepDescription = isExpressionJsVarLiteral(item.description)
? item.description.data
: ""
const stepLink = isExpressionJsVarLiteral(item.link)
? (item.link.data as string)
: `#${stepName}`
if (!stepName) {
return
}
const stepChildren = item.steps || []
const children: UnistNode[] = [
{
type: "paragraph",
children: [
{
type: "link",
url: stepLink,
children: [
{
type: "text",
value: `${stepName}`,
},
],
},
{
type: "text",
value: `: ${stepDescription}`,
},
],
},
]
if (Array.isArray(stepChildren) && stepChildren.length) {
children.push(...generateWorkflowItems(stepChildren))
}
listItems.push({
type: "listItem",
children,
})
})
return listItems
}
const listItems = generateWorkflowItems(workflowJsVar.steps)
parent?.children.splice(index, 1, {
type: "list",
ordered: false,
spread: false,
children: listItems,
})
return [SKIP, index]
}
/**
* Helpers
*/
const getTextNode = (node: UnistNode): UnistNode | undefined => {
let textNode: UnistNode | undefined
node.children?.some((child) => {
if (textNode) {
return true
}
if (child.type === "text") {
textNode = child
} else if (child.type === "paragraph") {
textNode = getTextNode(child)
} else if (child.value) {
textNode = child
}
return false
})
return textNode
}
const formatNodeText = (node?: UnistNode): string => {
if (!node) {
return ""
}
if (node.type === "inlineCode") {
return `\`${node.value}\``
} else if (node.type === "code") {
return `\`\`\`${"lang" in node ? node.lang : "ts"}\n${node.value}\n\`\`\``
} else if (node.type === "link") {
return `[${node.children?.[0].value}](${node.url})`
}
return node.value || ""
}