docs: add tags package to generate tags (#10666)

* initial changes

* finalize implementation

* run generator on prep

* remove tags package from book

* optimization attempt

* test transpile

* test transpile

* rename utils

* rename directory

* add vercel ignore

* add tags to book
This commit is contained in:
Shahed Nasser
2024-12-19 13:15:42 +02:00
committed by GitHub
parent 3f4d574748
commit 16ae192456
41 changed files with 424 additions and 62 deletions

1
www/.vercelignore Normal file
View File

@@ -0,0 +1 @@
utils

View File

@@ -38,6 +38,7 @@
"eslint-plugin-prettier": "^5.2.1",
"eslint-plugin-react-hooks": "^5.0.0",
"postcss": "^8",
"tags": "*",
"tailwind": "*",
"tailwindcss": "^3.3.0",
"tsconfig": "*",

View File

@@ -1,7 +1,10 @@
import { generateEditedDates } from "build-scripts"
import path from "path"
import { generateTags } from "tags"
async function main() {
await generateEditedDates()
await generateTags(path.resolve("..", "..", "packages", "tags"))
}
void main()

View File

@@ -0,0 +1 @@
../../utils

View File

@@ -6,6 +6,7 @@ import {
typeListLinkFixerPlugin,
workflowDiagramLinkFixerPlugin,
} from "remark-rehype-plugins"
import bundleAnalyzer from "@next/bundle-analyzer"
import mdxPluginOptions from "./mdx-options.mjs"
import path from "node:path"
@@ -137,4 +138,8 @@ const nextConfig = {
// },
}
export default withMDX(nextConfig)
const withBundleAnalyzer = bundleAnalyzer({
enabled: process.env.ANALYZE === "true",
})
export default withMDX(withBundleAnalyzer(nextConfig))

View File

@@ -27,17 +27,20 @@
"remark-frontmatter": "^5.0.0"
},
"devDependencies": {
"@next/bundle-analyzer": "^15.1.1",
"@types/mdx": "^2.0.13",
"@types/node": "^20",
"@types/react": "npm:types-react@rc",
"@types/react-dom": "npm:types-react@rc",
"autoprefixer": "^10.0.1",
"build-scripts": "*",
"docs-utils": "*",
"eslint": "^9.13.0",
"eslint-plugin-prettier": "^5.2.1",
"eslint-plugin-react-hooks": "^5.0.0",
"postcss": "^8",
"remark-rehype-plugins": "*",
"tags": "*",
"tailwind": "*",
"tailwindcss": "^3.3.0",
"ts-node": "^10.9.2",

View File

@@ -1,13 +1,16 @@
import { generateEditedDates, generateSidebar } from "build-scripts"
import { generateTags } from "tags"
import { main as generateSlugChanges } from "./generate-slug-changes.mjs"
import { main as generateFilesMap } from "./generate-files-map.mjs"
import { sidebar } from "../sidebar.mjs"
import path from "path"
async function main() {
await generateSidebar(sidebar)
await generateSlugChanges()
await generateFilesMap()
await generateEditedDates()
await generateTags(path.resolve("..", "..", "packages", "tags"))
}
void main()

View File

@@ -1,6 +1,6 @@
import { statSync, readdirSync } from "fs"
import path from "path"
import { getFileSlugUtil } from "remark-rehype-plugins"
import { getFileSlug } from "../../../packages/docs-utils/dist"
const monoRepoPath = path.resolve("..", "..", "..")
@@ -42,7 +42,7 @@ export default async function getSlugs(options = {}) {
continue
}
const newSlug = await getFileSlugUtil(filePath)
const newSlug = await getFileSlug(filePath)
if (newSlug) {
slugs.push({

View File

@@ -29,6 +29,7 @@ export default [
"**/public",
"**/.eslintrc.js",
"**/generated",
"packages/tags/src/tags"
],
}, ...compat.extends(
"eslint:recommended",

View File

@@ -16,7 +16,8 @@
"start": "turbo run start:monorepo",
"dev": "turbo run dev:monorepo",
"lint": "turbo run lint",
"lint:content": "turbo run lint:content"
"lint:content": "turbo run lint:content",
"watch": "turbo run watch"
},
"dependencies": {
"autoprefixer": "10.4.14",

View File

@@ -24,13 +24,15 @@
],
"scripts": {
"build": "yarn clean && tsc",
"clean": "rimraf dist"
"clean": "rimraf dist",
"watch": "tsc --watch"
},
"dependencies": {
"remark-rehype-plugins": "*"
},
"devDependencies": {
"@types/node": "^20.11.20",
"docs-utils": "*",
"rimraf": "^5.0.5",
"tsconfig": "*",
"types": "*",

View File

@@ -2,8 +2,6 @@ export * from "./generate-edited-dates.js"
export * from "./generate-sidebar.js"
export * from "./retrieve-mdx-pages.js"
export * from "./utils/find-metadata-title.js"
export * from "./utils/find-page-heading.js"
export * from "./utils/get-core-flows-ref-sidebar-children.js"
export * from "./utils/get-sidebar-item-link.js"
export * from "./utils/sidebar-attach-href-common-options.js"

View File

@@ -1,6 +1,6 @@
import { readdirSync } from "fs"
import path from "path"
import { getFileSlugSyncUtil } from "remark-rehype-plugins"
import { getFileSlugSync } from "../../docs-utils/dist/index.js"
type Options = {
basePath: string
@@ -24,7 +24,7 @@ export function retrieveMdxPages({ basePath }: Options): string[] {
continue
}
const slug = getFileSlugSyncUtil(filePath)
const slug = getFileSlugSync(filePath)
urls.push(
slug ||

View File

@@ -1,7 +0,0 @@
const REGEX = /export const metadata = {[\s\S]*title: `(?<title>.*)`/
export default function findMetadataTitle(content: string): string | undefined {
const headingMatch = REGEX.exec(content)
return headingMatch?.groups?.title
}

View File

@@ -1,7 +0,0 @@
const HEADING_REGEX = /# (?<title>.*)/
export default function findPageHeading(content: string): string | undefined {
const headingMatch = HEADING_REGEX.exec(content)
return headingMatch?.groups?.title
}

View File

@@ -1,8 +1,5 @@
import { getFrontMatterUtil } from "remark-rehype-plugins"
import { getFrontMatter, findPageTitle } from "../../../docs-utils/dist/index.js"
import { ItemsToAdd, sidebarAttachHrefCommonOptions } from "../index.js"
import { readFileSync } from "fs"
import findMetadataTitle from "./find-metadata-title.js"
import findPageHeading from "./find-page-heading.js"
import { InteractiveSidebarItem } from "types"
export async function getSidebarItemLink({
@@ -14,26 +11,18 @@ export async function getSidebarItemLink({
basePath: string
fileBasename: string
}): Promise<ItemsToAdd | undefined> {
const frontmatter = await getFrontMatterUtil(filePath)
const frontmatter = await getFrontMatter(filePath)
if (frontmatter.sidebar_autogenerate_exclude) {
return
}
const fileContent = frontmatter.sidebar_label
? ""
: readFileSync(filePath, "utf-8")
const newItem = sidebarAttachHrefCommonOptions([
{
type: "link",
path:
frontmatter.slug ||
filePath.replace(basePath, "").replace(`/${fileBasename}`, ""),
title:
frontmatter.sidebar_label ||
findMetadataTitle(fileContent) ||
findPageHeading(fileContent) ||
"",
title: frontmatter.sidebar_label || findPageTitle(filePath) || "",
},
])[0] as InteractiveSidebarItem

View File

@@ -6,6 +6,7 @@
"tsBuildInfoFile": "./dist/.tsbuildinfo-client",
"noEmit": false,
"lib": ["es2022"],
"target": "es2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"declaration": true,

View File

@@ -0,0 +1,48 @@
{
"name": "docs-utils",
"version": "0.0.0",
"private": true,
"license": "MIT",
"publishConfig": {
"access": "public"
},
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"module": "./dist/index.js",
"exports": {
".": {
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
}
},
"sideEffects": false,
"files": [
"dist/**"
],
"scripts": {
"build": "yarn clean && tsc",
"clean": "rimraf dist",
"watch": "tsc --watch"
},
"dependencies": {
"remark-frontmatter": "^5.0.0",
"remark-parse": "^11.0.0",
"remark-stringify": "^11.0.0",
"to-vfile": "^8.0.0",
"unified": "^11.0.4",
"vfile-matter": "^5.0.0"
},
"devDependencies": {
"@types/node": "^20.11.20",
"rimraf": "^5.0.5",
"tsconfig": "*",
"types": "*",
"typescript": "^5.3.3"
},
"engines": {
"node": ">=18.17.0"
}
}

View File

@@ -0,0 +1,23 @@
import { readFileSync } from "fs"
const REGEX = /export const metadata = {[\s\S]*title: `(?<title>.*)`/
export function findMetadataTitle(content: string): string | undefined {
const headingMatch = REGEX.exec(content)
return headingMatch?.groups?.title
}
const HEADING_REGEX = /# (?<title>.*)/
export function findPageHeading(content: string): string | undefined {
const headingMatch = HEADING_REGEX.exec(content)
return headingMatch?.groups?.title
}
export function findPageTitle(filePath: string): string | undefined {
const content = readFileSync(filePath, "utf-8")
return findMetadataTitle(content) || findPageHeading(content)
}

View File

@@ -1,12 +1,12 @@
import { getFrontMatterUtil } from "./get-front-matter.js"
import { matter } from "vfile-matter"
import { readSync } from "to-vfile"
import { FrontMatter } from "../types/index.js"
import { FrontMatter } from "types"
import { getFrontMatter } from "./get-front-matter.js"
export async function getFileSlugUtil(
export async function getFileSlug(
filePath: string
): Promise<string | undefined> {
const fileFrontmatter = await getFrontMatterUtil(filePath)
const fileFrontmatter = await getFrontMatter(filePath)
if (fileFrontmatter.slug) {
// add to slugs array
@@ -14,7 +14,7 @@ export async function getFileSlugUtil(
}
}
export function getFileSlugSyncUtil(filePath: string): string | undefined {
export function getFileSlugSync(filePath: string): string | undefined {
const content = readSync(filePath)
matter(content)

View File

@@ -2,13 +2,11 @@ import remarkFrontmatter from "remark-frontmatter"
import remarkParse from "remark-parse"
import remarkStringify from "remark-stringify"
import { unified } from "unified"
import { read } from "to-vfile"
import { read, readSync } from "to-vfile"
import { matter } from "vfile-matter"
import { FrontMatter } from "../types/index.js"
import { FrontMatter } from "types"
export async function getFrontMatterUtil(
filePath: string
): Promise<FrontMatter> {
export async function getFrontMatter(filePath: string): Promise<FrontMatter> {
return (
await unified()
.use(remarkParse)
@@ -22,3 +20,11 @@ export async function getFrontMatterUtil(
.process(await read(filePath))
).data.matter as FrontMatter
}
export function getFrontMatterSync(filePath: string): FrontMatter {
const content = readSync(filePath)
matter(content)
return content.data.matter as FrontMatter
}

View File

@@ -0,0 +1,3 @@
export * from "./find-title.js"
export * from "./get-file-slug.js"
export * from "./get-front-matter.js"

View File

@@ -0,0 +1,20 @@
{
"extends": ["tsconfig/base.json"],
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"tsBuildInfoFile": "./dist/.tsbuildinfo-client",
"noEmit": false,
"lib": ["es2022"],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ESNext",
"declaration": true,
"declarationMap": true,
"esModuleInterop": true,
"strict": true,
"skipLibCheck": true,
"resolveJsonModule": true
},
"include": ["src"]
}

View File

@@ -24,10 +24,12 @@
],
"scripts": {
"build": "yarn clean && tsc",
"clean": "rimraf dist"
"clean": "rimraf dist",
"watch": "tsc --watch"
},
"dependencies": {
"@cloudinary/url-gen": "^1.17.0",
"docs-utils": "*",
"remark-frontmatter": "^5.0.0",
"remark-parse": "^11.0.0",
"remark-stringify": "^11.0.0",

View File

@@ -9,5 +9,3 @@ export * from "./type-list-link-fixer.js"
export * from "./workflow-diagram-link-fixer.js"
export * from "./utils/fix-link.js"
export * from "./utils/get-file-slug.js"
export * from "./utils/get-front-matter.js"

View File

@@ -98,15 +98,6 @@ export declare type CloudinaryConfig = {
roundCorners?: number
}
export declare type FrontMatter = {
slug?: string
sidebar_label?: string
sidebar_group?: string
sidebar_group_main?: boolean
sidebar_position?: number
sidebar_autogenerate_exclude?: boolean
}
export declare type CrossProjectLinksOptions = {
baseUrl: string
projectUrls?: {

View File

@@ -1,5 +1,5 @@
import path from "path"
import { getFileSlugSyncUtil } from "./get-file-slug.js"
import { getFileSlugSync } from "../../../docs-utils/dist/index.js"
export type FixLinkOptions = {
currentPageFilePath: string
@@ -20,7 +20,7 @@ export function fixLinkUtil({
fullLinkedFilePath = fullLinkedFilePath.replace(hash, "")
// get absolute path of the URL
const linkedFilePath = fullLinkedFilePath.replace(basePath, "")
const linkedFileSlug = getFileSlugSyncUtil(fullLinkedFilePath)
const linkedFileSlug = getFileSlugSync(fullLinkedFilePath)
const newLink =
linkedFileSlug ||

View File

@@ -0,0 +1,45 @@
{
"name": "tags",
"version": "0.0.0",
"private": true,
"license": "MIT",
"publishConfig": {
"access": "public"
},
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"module": "./dist/index.js",
"exports": {
".": {
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
}
},
"sideEffects": false,
"files": [
"dist/**"
],
"scripts": {
"build": "yarn clean && yarn generate:tags && tsc",
"generate:tags": "node --loader ts-node/esm src/scripts/generate-tags.ts",
"clean": "rimraf dist",
"watch": "tsc --watch"
},
"dependencies": {
"docs-utils": "*"
},
"devDependencies": {
"@types/node": "^20.11.20",
"rimraf": "^5.0.5",
"ts-node": "^10.9.1",
"tsconfig": "*",
"types": "*",
"typescript": "^5.3.3"
},
"engines": {
"node": ">=18.17.0"
}
}

View File

@@ -0,0 +1 @@
export * from "./utils/index.js"

View File

@@ -0,0 +1,3 @@
import { generateTags } from "../utils/generate-tags.js"
void generateTags()

View File

View File

@@ -0,0 +1,119 @@
import { statSync } from "fs"
import { mkdir, readdir, rm, writeFile } from "fs/promises"
import path from "path"
import type { Tags } from "types"
import { findPageTitle, getFrontMatterSync } from "docs-utils"
type ConfigItem = {
path: string
contentPaths: string[]
}
const config: ConfigItem[] = [
{
path: path.resolve("..", "..", "apps", "book"),
contentPaths: ["app"],
},
{
path: path.resolve("..", "..", "apps", "resources"),
contentPaths: ["app", "references"],
},
{
path: path.resolve("..", "..", "apps", "ui"),
contentPaths: [path.join("src", "content", "docs")],
},
{
path: path.resolve("..", "..", "apps", "user-guide"),
contentPaths: ["app"],
},
]
function normalizePageTitle(title: string): string {
// remove variables from title
return title.replaceAll(/\$\{.+\}/g, "").trim()
}
function tagNameToFileName(tagName: string): string {
return `${tagName.toLowerCase().replaceAll(" ", "-")}.ts`
}
function tagNameToVarName(tagName: string): string {
return tagName
.toLowerCase()
.replaceAll(/\s([a-zA-Z\d])/g, (captured) => captured.toUpperCase().trim())
}
export async function generateTags(basePath?: string) {
basePath = basePath || path.resolve()
const tags: Tags = {}
async function getTags(item: ConfigItem) {
async function scanDirectory(dirPath: string) {
const files = await readdir(dirPath)
for (const file of files) {
const fullPath = path.join(dirPath, file)
if (!file.endsWith(".mdx") || file.startsWith("_")) {
if (statSync(fullPath).isDirectory()) {
await scanDirectory(fullPath)
}
continue
}
const frontmatter = getFrontMatterSync(fullPath)
const fileBasename = path.basename(file)
frontmatter.tags?.forEach((tag) => {
if (!Object.hasOwn(tags, tag)) {
tags[tag] = []
}
tags[tag].push({
title: normalizePageTitle(
frontmatter.sidebar_label || findPageTitle(fullPath) || ""
),
path:
frontmatter.slug ||
fullPath.replace(item.path, "").replace(`/${fileBasename}`, ""),
})
})
}
}
for (const contentPath of item.contentPaths) {
const basePath = path.join(item.path, contentPath)
await scanDirectory(basePath)
}
}
await Promise.all(
config.map(async (item) => {
await getTags(item)
})
)
const tagsDir = path.join(basePath, "src", "tags")
// clear existing tags
await rm(tagsDir, {
recursive: true,
force: true,
})
await mkdir(tagsDir)
// write tags
const files: string[] = []
await Promise.all(
Object.keys(tags).map(async (tagName) => {
const fileName = tagNameToFileName(tagName)
const varName = tagNameToVarName(tagName)
const content = `export const ${varName} = ${JSON.stringify(tags[tagName], null, 2)}`
await writeFile(path.join(tagsDir, fileName), content)
files.push(fileName.replace(/\.ts$/, ".js"))
})
)
// write index.ts
const indexContent = files.map((file) => `export * from "./${file}"\n`)
await writeFile(path.join(tagsDir, "index.ts"), indexContent)
}

View File

@@ -0,0 +1,2 @@
export * from "./generate-tags.js"
export * from "./tags.js"

View File

@@ -0,0 +1,13 @@
import { Tag, Tags } from "types"
import * as tags from "../tags/index.js"
export const getTagItems = (tagName: string): Tag | undefined => {
if (!Object.hasOwn(tags, tagName)) {
return
}
return tags[tagName as keyof typeof tags]
}
export const getAllTags = (): Tags => {
return tags
}

View File

@@ -0,0 +1,25 @@
{
"extends": "tsconfig/base.json",
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"tsBuildInfoFile": "./dist/.tsbuildinfo-client",
"noEmit": false,
"lib": ["es2022"],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ESNext",
"declaration": true,
"declarationMap": true,
"esModuleInterop": true,
"strict": true,
"skipLibCheck": true,
"resolveJsonModule": true
},
"include": ["src"],
"ts-node": {
"esm": true,
"experimentalSpecifierResolution": "node",
"transpileOnly": true
}
}

View File

@@ -28,7 +28,8 @@
],
"scripts": {
"build": "yarn clean && tsc",
"clean": "rimraf dist"
"clean": "rimraf dist",
"watch": "tsc --watch"
},
"devDependencies": {
"@types/node": "^20.11.20",

View File

@@ -0,0 +1,9 @@
export declare type FrontMatter = {
slug?: string
sidebar_label?: string
sidebar_group?: string
sidebar_group_main?: boolean
sidebar_position?: number
sidebar_autogenerate_exclude?: boolean
tags?: string[]
}

View File

@@ -1,10 +1,12 @@
export * from "./api-testing.js"
export * from "./build-scripts.js"
export * from "./config.js"
export * from "./frontmatter.js"
export * from "./general.js"
export * from "./menu.js"
export * from "./navigation.js"
export * from "./navigation-dropdown.js"
export * from "./sidebar.js"
export * from "./tags.js"
export * from "./toc.js"
export * from "./workflow.js"

View File

@@ -0,0 +1,8 @@
export type Tag = {
title: string
path: string
}[]
export type Tags = {
[k: string]: Tag
}

View File

@@ -22,6 +22,7 @@
},
"lint": { },
"lint:content": { },
"watch": { },
"dev:monorepo": {
"dependsOn": [
"^build-scripts#build",

View File

@@ -1804,6 +1804,15 @@ __metadata:
languageName: node
linkType: hard
"@next/bundle-analyzer@npm:^15.1.1":
version: 15.1.1
resolution: "@next/bundle-analyzer@npm:15.1.1"
dependencies:
webpack-bundle-analyzer: 4.10.1
checksum: 2a85db96b1b37a6273d7ee81978ffd11181335b2017c636bcccfc6bca3fdd9a7f1b3c45fba35b4b51f5ca844da76f604d5e8568c894fbdae5a14ff27548cc0d6
languageName: node
linkType: hard
"@next/env@npm:15.0.4":
version: 15.0.4
resolution: "@next/env@npm:15.0.4"
@@ -7142,6 +7151,7 @@ __metadata:
rehype-mdx-code-props: ^2.0.0
rehype-slug: ^6.0.0
remark-rehype-plugins: "*"
tags: "*"
tailwind: "*"
tailwindcss: ^3.3.0
tsconfig: "*"
@@ -7204,6 +7214,7 @@ __metadata:
resolution: "build-scripts@workspace:packages/build-scripts"
dependencies:
"@types/node": ^20.11.20
docs-utils: "*"
remark-rehype-plugins: "*"
rimraf: ^5.0.5
tsconfig: "*"
@@ -8460,6 +8471,24 @@ __metadata:
languageName: unknown
linkType: soft
"docs-utils@*, docs-utils@workspace:packages/docs-utils":
version: 0.0.0-use.local
resolution: "docs-utils@workspace:packages/docs-utils"
dependencies:
"@types/node": ^20.11.20
remark-frontmatter: ^5.0.0
remark-parse: ^11.0.0
remark-stringify: ^11.0.0
rimraf: ^5.0.5
to-vfile: ^8.0.0
tsconfig: "*"
types: "*"
typescript: ^5.3.3
unified: ^11.0.4
vfile-matter: ^5.0.0
languageName: unknown
linkType: soft
"doctrine@npm:^2.1.0":
version: 2.1.0
resolution: "doctrine@npm:2.1.0"
@@ -14615,6 +14644,7 @@ __metadata:
"@cloudinary/url-gen": ^1.17.0
"@types/node": ^20.11.20
docs-ui: "*"
docs-utils: "*"
remark-frontmatter: ^5.0.0
remark-parse: ^11.0.0
remark-stringify: ^11.0.0
@@ -14797,6 +14827,7 @@ __metadata:
"@mdx-js/loader": ^3.1.0
"@mdx-js/react": ^3.1.0
"@medusajs/icons": ^2.0.0
"@next/bundle-analyzer": ^15.1.1
"@next/mdx": 15.0.4
"@types/mdx": ^2.0.13
"@types/node": ^20
@@ -14806,6 +14837,7 @@ __metadata:
build-scripts: "*"
clsx: ^2.1.0
docs-ui: "*"
docs-utils: "*"
eslint: ^9.13.0
eslint-plugin-prettier: ^5.2.1
eslint-plugin-react-hooks: ^5.0.0
@@ -14817,6 +14849,7 @@ __metadata:
remark-directive: ^3.0.0
remark-frontmatter: ^5.0.0
remark-rehype-plugins: "*"
tags: "*"
tailwind: "*"
tailwindcss: ^3.3.0
ts-node: ^10.9.2
@@ -15649,6 +15682,20 @@ __metadata:
languageName: node
linkType: hard
"tags@*, tags@workspace:packages/tags":
version: 0.0.0-use.local
resolution: "tags@workspace:packages/tags"
dependencies:
"@types/node": ^20.11.20
docs-utils: "*"
rimraf: ^5.0.5
ts-node: ^10.9.1
tsconfig: "*"
types: "*"
typescript: ^5.3.3
languageName: unknown
linkType: soft
"tailwind-merge@npm:^2.2.1":
version: 2.2.1
resolution: "tailwind-merge@npm:2.2.1"