[docs] Migrate documentation and cookbook to Blume (#1969)
Co-authored-by: Aryan Kumar <aryan5v@users.noreply.github.com>
This commit is contained in:
Aryan Kumar
co-authored by
Aryan Kumar
Aryan Kumar
parent
ba371e33ed
commit
6212e7262e
@@ -0,0 +1,35 @@
|
||||
import { sourceRecipes } from "../lib/cookbook-pages.ts";
|
||||
/** Prepare the docs transfer and publish the existing cookbook catalogs. */
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { cp, mkdir, rm } from "node:fs/promises";
|
||||
|
||||
import { transferDocs } from "./transfer-docs.mjs";
|
||||
|
||||
await transferDocs();
|
||||
|
||||
const repo = new URL("../../", import.meta.url);
|
||||
execFileSync("npm", ["run", "build:catalog", "--prefix", "docs"], {
|
||||
cwd: repo,
|
||||
stdio: "inherit",
|
||||
});
|
||||
const publicDir = new URL("../public/", import.meta.url);
|
||||
await mkdir(publicDir, { recursive: true });
|
||||
const destination = new URL("assets/cookbook-config/", publicDir);
|
||||
await rm(destination, { recursive: true, force: true });
|
||||
await cp(new URL("docs/assets/cookbook-config/", repo), destination, {
|
||||
recursive: true,
|
||||
});
|
||||
await cp(new URL("docs/assets/logos/icon_simple.svg", repo), new URL("icon.svg", publicDir));
|
||||
|
||||
await cp(new URL("../assets/hao-ai-lab.png", import.meta.url), new URL("assets/hao-ai-lab.png", publicDir));
|
||||
|
||||
// Publish exact checked-in YAML for inference-only model pages. No settings are synthesized.
|
||||
const sourceDir = new URL("assets/cookbook-sources/", publicDir);
|
||||
await rm(sourceDir, { recursive: true, force: true });
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
for (const recipe of sourceRecipes) {
|
||||
if (/\.ya?ml$/.test(recipe.source))
|
||||
await cp(new URL(recipe.source, repo), new URL(`${recipe.id}-inference.yaml`, sourceDir));
|
||||
if (recipe.serving?.source)
|
||||
await cp(new URL(recipe.serving.source, repo), new URL(`${recipe.id}-serving.yaml`, sourceDir));
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Build source-only Python reference pages with Griffe, without importing FastVideo."""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import runpy
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlsplit, urlunsplit
|
||||
|
||||
import griffe
|
||||
import markdown
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SITE = ROOT / "docs-site"
|
||||
GENERATED = SITE / ".generated"
|
||||
|
||||
|
||||
def signature(obj) -> str:
|
||||
"""Preserve parameter kinds, annotations, defaults, and return annotations."""
|
||||
if obj.is_attribute:
|
||||
annotation = f": {obj.annotation}" if obj.annotation else ""
|
||||
value = f" = {obj.value}" if obj.value is not None else ""
|
||||
return f"{obj.name}{annotation}{value}"
|
||||
parameters = list(getattr(obj, "parameters", []))
|
||||
parts = []
|
||||
keyword_separator = False
|
||||
for i, param in enumerate(parameters):
|
||||
if param.name in {"self", "cls"}:
|
||||
continue
|
||||
kind = param.kind.value
|
||||
if kind == "keyword-only" and not keyword_separator:
|
||||
parts.append("*")
|
||||
keyword_separator = True
|
||||
prefix = "**" if kind == "variadic keyword" else "*" if kind == "variadic positional" else ""
|
||||
if prefix == "*":
|
||||
keyword_separator = True
|
||||
text = prefix + param.name
|
||||
if param.annotation is not None:
|
||||
text += f": {param.annotation}"
|
||||
if param.default is not None and not prefix:
|
||||
text += f" = {param.default}"
|
||||
parts.append(text)
|
||||
if kind == "positional-only" and (i + 1 == len(parameters) or parameters[i + 1].kind.value != kind):
|
||||
parts.append("/")
|
||||
result = f"{obj.name}({', '.join(parts)})"
|
||||
if obj.is_function and obj.returns is not None:
|
||||
result += f" -> {obj.returns}"
|
||||
return result
|
||||
|
||||
|
||||
def code_html(text: str) -> str:
|
||||
return html.escape(text).replace("[", "[").replace("]", "]")
|
||||
|
||||
|
||||
def prose(text: str) -> str:
|
||||
# Docstrings are trusted repository source, rendered as Markdown rather than executed.
|
||||
return markdown.markdown(text, extensions=["fenced_code", "tables", "sane_lists"])
|
||||
|
||||
|
||||
def api_overview(text: str, package) -> str:
|
||||
def reference(match):
|
||||
label, target = match.groups()
|
||||
path = (target or label).strip("`")
|
||||
obj = package if path == package.path else package[path.removeprefix(package.path + ".")]
|
||||
module = obj
|
||||
while not module.is_module:
|
||||
module = module.parent
|
||||
route = "/api/" + module.path.replace(".", "/")
|
||||
return f"[{label}]({route}#{path})"
|
||||
|
||||
text = re.sub(r"\[([^\]\n]+)\]\[([^\]\n]*)\]", reference, text)
|
||||
|
||||
def guide(match):
|
||||
label, target = match.groups()
|
||||
url = urlsplit(target)
|
||||
if url.scheme or url.netloc or not re.search(r"\.mdx?$", url.path):
|
||||
return match[0]
|
||||
url = urlsplit(urljoin("/api/index.md", target))
|
||||
route = re.sub(r"\.mdx?$", "", url.path)
|
||||
route = re.sub(r"/index$", "", route) or "/"
|
||||
return f"[{label}]({urlunsplit(('', '', route, url.query, url.fragment))})"
|
||||
|
||||
return prose(re.sub(r"\[([^\]\n]+)\]\(([^\s)]+)\)", guide, text))
|
||||
|
||||
|
||||
def docstring(obj, level: int) -> str:
|
||||
if not obj.docstring:
|
||||
return ""
|
||||
output = []
|
||||
for section in obj.docstring.parsed:
|
||||
kind = section.kind.value
|
||||
if kind == "text":
|
||||
output.append(prose(section.value))
|
||||
continue
|
||||
title = section.title or kind.replace("_", " ").capitalize()
|
||||
output.append(f"<h{level}>{html.escape(title)}</h{level}>")
|
||||
if kind == "examples":
|
||||
for example_kind, value in section.value:
|
||||
output.append(f"<pre><code>{html.escape(value)}</code></pre>" if example_kind.value == "examples" else prose(value))
|
||||
elif isinstance(section.value, list):
|
||||
output.append("<dl>")
|
||||
for item in section.value:
|
||||
name = getattr(item, "name", "") or ""
|
||||
annotation = getattr(item, "annotation", "") or ""
|
||||
label = f"{name}: {annotation}" if name and annotation else str(name or annotation)
|
||||
output.append(f"<dt><code>{html.escape(label)}</code></dt><dd>{prose(getattr(item, 'description', ''))}</dd>")
|
||||
output.append("</dl>")
|
||||
else:
|
||||
output.append(prose(getattr(section.value, "description", str(section.value))))
|
||||
return "\n".join(output)
|
||||
|
||||
|
||||
def symbol(obj, path: str, level: int = 2, members: list[str] | None = None, source: bool = True) -> str:
|
||||
level = min(level, 6)
|
||||
output = [f'<h{level} id="{html.escape(path)}"><code>{html.escape(path)}</code></h{level}>']
|
||||
output.append(f'<pre class="doc-signature"><code>{code_html(signature(obj))}</code></pre>')
|
||||
output.append(docstring(obj, min(level + 1, 6)))
|
||||
if obj.is_class:
|
||||
constructor = obj.members.get("__init__")
|
||||
if constructor is not None:
|
||||
output.append(docstring(constructor, min(level + 1, 6)))
|
||||
if source and not obj.is_alias:
|
||||
try:
|
||||
code = obj.source
|
||||
except (ValueError, KeyError):
|
||||
code = ""
|
||||
if code:
|
||||
output.append(f'<details><summary>Source</summary><pre><code>{code_html(code)}</code></pre></details>')
|
||||
if obj.is_class:
|
||||
for name, member in obj.members.items():
|
||||
if members is not None and name not in members:
|
||||
continue
|
||||
if name.startswith("_") or member.is_module:
|
||||
continue
|
||||
if member.is_attribute and not member.docstring:
|
||||
continue
|
||||
try:
|
||||
output.append(symbol(member, f"{path}.{name}", min(level + 1, 6), source=source))
|
||||
except griffe.AliasResolutionError:
|
||||
continue
|
||||
return "\n".join(output)
|
||||
|
||||
|
||||
def write_page(route: str, title: str, body: str, anchors: dict) -> None:
|
||||
# Preserve incoming fragments, including historical category labels whose layout changed.
|
||||
present = set(re.findall(r'id="([^"]+)"', body))
|
||||
aliases = "\n".join(f'<a id="{html.escape(h["id"])}"></a>' for h in anchors.get(route, []) if h["id"] not in present)
|
||||
target = GENERATED / "python-reference" / route / "index.md"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("---\n" + yaml.safe_dump({"title": title}, sort_keys=False) +
|
||||
f'---\n\n<div class="python-reference">\n{aliases}\n{body}\n</div>\n')
|
||||
|
||||
|
||||
def prepare() -> None:
|
||||
GENERATED.mkdir(exist_ok=True)
|
||||
examples = runpy.run_path(str(ROOT / "docs/generate_examples.py"))
|
||||
examples["validate_cookbook"]()
|
||||
examples["generate_examples"](generate_main_index=True)
|
||||
anchors = json.loads((SITE / "migration-anchors.json").read_text())
|
||||
shutil.rmtree(GENERATED / "python-reference", ignore_errors=True)
|
||||
package = griffe.load("fastvideo", search_paths=[ROOT], allow_inspection=False, resolve_aliases=True,
|
||||
resolve_external=False, docstring_parser="google", docstring_options={"warnings": False})
|
||||
modules = []
|
||||
|
||||
def visit(module):
|
||||
if module.is_alias or any(part.startswith("_") or part in {"tests", "third_party"} for part in module.path.split(".")):
|
||||
return
|
||||
modules.append(module)
|
||||
for child in module.modules.values():
|
||||
visit(child)
|
||||
|
||||
visit(package)
|
||||
for module in sorted(modules, key=lambda item: item.path):
|
||||
body = [f'<a id="{module.path}"></a>', docstring(module, 2)]
|
||||
for name, member in module.members.items():
|
||||
if name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
# External imports belong to their upstream reference, not FastVideo's API.
|
||||
if member.is_alias and not member.target_path.startswith("fastvideo."):
|
||||
continue
|
||||
if member.is_module or member.is_attribute and not member.docstring:
|
||||
continue
|
||||
body.append(symbol(member, f"{module.path}.{name}"))
|
||||
except griffe.AliasResolutionError:
|
||||
continue
|
||||
write_page("api/" + module.path.replace(".", "/"), module.name, "\n".join(body), anchors)
|
||||
write_page("api", "Python API reference", api_overview((ROOT / "docs/api/index.md").read_text(), package), anchors)
|
||||
v1 = (ROOT / "docs/getting_started/v1_api.md").read_text()
|
||||
|
||||
def reference(match):
|
||||
path, level, members = match.groups()
|
||||
return "\n" + symbol(package[path.removeprefix("fastvideo.")], path,
|
||||
int(level), members.split(","), source=False) + "\n"
|
||||
|
||||
v1 = re.sub(r'<api-reference path="([^"]+)" heading="(\d+)" members="([^"]+)" />', reference, v1)
|
||||
write_page("getting_started/v1_api", "V1 API", prose(v1.removeprefix("# V1 API\n")), anchors)
|
||||
(GENERATED / "old-headings.json").write_text(json.dumps(anchors))
|
||||
print(f"Rendered {len(modules) + 1} Python reference pages directly from source, without MkDocs or runtime imports.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
prepare()
|
||||
@@ -0,0 +1,370 @@
|
||||
import { modelDirectory } from "../lib/cookbook-pages.ts";
|
||||
/** Rebuild the Blume view from canonical Markdown and the existing Python reference renderer. */
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, statSync } from "node:fs";
|
||||
import { cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parse, stringify } from "yaml";
|
||||
|
||||
const site = fileURLToPath(new URL("../", import.meta.url));
|
||||
const repo = resolve(site, "..");
|
||||
const source = join(repo, "docs");
|
||||
const generated = join(site, ".generated");
|
||||
const content = join(generated, "content");
|
||||
const publicDir = join(site, "public");
|
||||
const attachments = new Set();
|
||||
const routeFor = (file) =>
|
||||
"/" +
|
||||
file
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/\.(md|mdx)$/, "")
|
||||
.replace(/(^|\/)index$/, "")
|
||||
.replace(/\/$/, "");
|
||||
|
||||
async function files(root) {
|
||||
const result = [];
|
||||
for (const entry of await readdir(root, { withFileTypes: true })) {
|
||||
const path = join(root, entry.name);
|
||||
if (entry.isDirectory()) result.push(...(await files(path)));
|
||||
else result.push(path);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function splitPage(text) {
|
||||
const match = /^---\n([\s\S]*?)\n---\n/.exec(text);
|
||||
return match ? { data: parse(match[1]), body: text.slice(match[0].length) } : { data: {}, body: text };
|
||||
}
|
||||
|
||||
function localTarget(target, from, pretty = false) {
|
||||
if (/^(?:#|mailto:|data:|tel:)/.test(target)) return target;
|
||||
let url;
|
||||
try {
|
||||
url = new URL(target, `https://transfer.invalid${pretty ? `${routeFor(from)}/` : `/${from}`}`);
|
||||
} catch {
|
||||
return target;
|
||||
}
|
||||
if (url.host !== "transfer.invalid") {
|
||||
if (url.origin !== "https://haoailab.com" || !url.pathname.startsWith("/FastVideo/")) return target;
|
||||
url = new URL(url.pathname.replace(/^\/FastVideo/, "") + url.search + url.hash, "https://transfer.invalid");
|
||||
}
|
||||
const path = decodeURIComponent(url.pathname).replace(/^\//, "");
|
||||
// Links leaving docs/ name repository files, not files served by the docs host.
|
||||
if (!pretty && !target.startsWith("/") && !/^[a-z]+:/i.test(target)) {
|
||||
const actual = resolve(source, dirname(from), decodeURIComponent(target.split(/[?#]/)[0]));
|
||||
if (!actual.startsWith(source + sep) && existsSync(actual)) {
|
||||
return `https://github.com/hao-ai-lab/FastVideo/blob/main/${relative(repo, actual).replaceAll(sep, "/")}${url.hash}`;
|
||||
}
|
||||
}
|
||||
if (existsSync(join(source, path)) && statSync(join(source, path)).isFile() && !/\.mdx?$/.test(path))
|
||||
attachments.add(path);
|
||||
if (/\.mdx?$/.test(path)) return `${routeFor(path)}${url.search}${url.hash}`;
|
||||
return `/${path}${url.search}${url.hash}`;
|
||||
}
|
||||
|
||||
function rewriteLinks(body, from) {
|
||||
let fence;
|
||||
return body
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
|
||||
if (marker) {
|
||||
if (!fence) fence = marker;
|
||||
else if (marker[0] === fence[0] && marker.length >= fence.length) fence = undefined;
|
||||
return line;
|
||||
}
|
||||
if (fence) return line;
|
||||
return line
|
||||
.replace(/<(?=[0-9=])/g, "<")
|
||||
.replace(/(\]\()([^\s)]+)(\))/g, (_, before, url, after) => before + localTarget(url, from) + after)
|
||||
.replace(
|
||||
/((?:href|src)=["'])([^"']+)(["'])/g,
|
||||
(_, before, url, after) => before + localTarget(url, from, from.startsWith("cookbook/")) + after,
|
||||
);
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function pinHeadings(body, old, mdx) {
|
||||
const normalize = (text) =>
|
||||
text
|
||||
.replace(/<[^>]*>/g, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
|
||||
.replace(/[^\p{L}\p{N}]/gu, "")
|
||||
.toLowerCase();
|
||||
const candidates = [...(old ?? [])];
|
||||
let fence;
|
||||
let text = body
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
|
||||
if (marker) {
|
||||
if (!fence) fence = marker;
|
||||
else if (marker[0] === fence[0] && marker.length >= fence.length) fence = undefined;
|
||||
return line;
|
||||
}
|
||||
if (fence) return line;
|
||||
const heading = /^(#{1,6})\s+(.+?)\s*$/.exec(line);
|
||||
if (!heading) return line;
|
||||
const clean = heading[2].replace(/\s*(?:\[#.*?\]|\{#.*?\})\s*$/, "");
|
||||
const index = candidates.findIndex(
|
||||
(item) => item.level === heading[1].length && normalize(item.text.replace(/¶$/, "")) === normalize(clean),
|
||||
);
|
||||
if (index < 0) return line;
|
||||
const [original] = candidates.splice(index, 1);
|
||||
if (original.id.includes("__")) return `${heading[1]} <a id="${original.id}"></a>${clean}`;
|
||||
return `${heading[1]} ${clean} ${mdx ? `[#${original.id}]` : `{#${original.id}}`}`;
|
||||
})
|
||||
.join("\n");
|
||||
const first = candidates.find((item) => item.level === 1);
|
||||
if (first && !text.includes(`id="${first.id}"`)) text = `<a id="${first.id}"></a>\n\n${text}`;
|
||||
return text;
|
||||
}
|
||||
|
||||
/** Blume owns the page title, including examples whose Source link precedes their H1. */
|
||||
function extractPageTitle(body, data, deduplicate = false) {
|
||||
const normalize = (text) => text.replace(/[^\p{L}\p{N}]/gu, "").toLowerCase();
|
||||
let fence;
|
||||
let found = false;
|
||||
return body
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
|
||||
if (marker) {
|
||||
if (!fence) fence = marker;
|
||||
else if (marker[0] === fence[0] && marker.length >= fence.length) fence = undefined;
|
||||
return line;
|
||||
}
|
||||
if (fence || (found && !deduplicate)) return line;
|
||||
const heading = (deduplicate ? /^(#{1,2})\s+(.+)$/ : /^(#)\s+(.+)$/).exec(line);
|
||||
if (!heading) return line;
|
||||
if (heading[1].length === 2 && !data.title) return line;
|
||||
const anchor = /(?:\[#([^\]]+)\]|\{#([^}]+)\})\s*$/.exec(heading[2]);
|
||||
const embedded = /<a\s+id="[^"]*"><\/a>/g;
|
||||
const anchors = heading[2].match(embedded) ?? [];
|
||||
const title = heading[2]
|
||||
.replace(/\s*(?:\[#[^\]]+\]|\{#[^}]+\})\s*$/, "")
|
||||
.replace(embedded, "")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/[`*_]/g, "")
|
||||
.trim();
|
||||
if (data.title && normalize(data.title) !== normalize(title)) return line;
|
||||
if (heading[1].length === 1) {
|
||||
data.title ??= title;
|
||||
found = true;
|
||||
}
|
||||
if (anchor) anchors.push(`<a id="${anchor[1] ?? anchor[2]}"></a>`);
|
||||
return anchors.join("\n");
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export async function transferDocs() {
|
||||
const localPython = join(site, ".venv/bin/python");
|
||||
const python = process.env.DOCS_PYTHON || (existsSync(localPython) ? localPython : "python3");
|
||||
execFileSync(python, [join(site, "scripts/prepare-python-docs.py")], { cwd: repo, stdio: "inherit" });
|
||||
await rm(content, { recursive: true, force: true });
|
||||
await rm(publicDir, { recursive: true, force: true });
|
||||
await mkdir(content, { recursive: true });
|
||||
const ignored = new Set(["js", "tests", "node_modules", "__pycache__"]);
|
||||
await cp(source, content, {
|
||||
recursive: true,
|
||||
filter: async (path) => {
|
||||
const rel = relative(source, path);
|
||||
if (ignored.has(rel.split(sep)[0]) || rel === "README.md") return false;
|
||||
return (await stat(path)).isDirectory() || /\.mdx?$/.test(path);
|
||||
},
|
||||
});
|
||||
// Native callout directives and islands compile as MDX, while source links keep .md filenames.
|
||||
for (const file of await files(content)) {
|
||||
if (!file.endsWith(".md")) continue;
|
||||
const text = await readFile(file, "utf8");
|
||||
if (/^:::(?:note|tip|warning|danger|info|success)\b/m.test(text) || text.includes("<RecipeFamily ")) {
|
||||
await writeFile(file.replace(/\.md$/, ".mdx"), text);
|
||||
await rm(file);
|
||||
}
|
||||
}
|
||||
const oldHeadings = JSON.parse(await readFile(join(generated, "old-headings.json"), "utf8"));
|
||||
for (const path of await files(content)) {
|
||||
if (!/\.mdx?$/.test(path)) continue;
|
||||
const rel = relative(content, path).replaceAll(sep, "/");
|
||||
const route = routeFor(rel);
|
||||
const { data, body } = splitPage(await readFile(path, "utf8"));
|
||||
delete data.noindex;
|
||||
data.slug = route.replace(/^\//, "") || "/";
|
||||
const pinned = extractPageTitle(
|
||||
pinHeadings(body, oldHeadings[route.slice(1) || "."], path.endsWith(".mdx")),
|
||||
data,
|
||||
rel.includes("/examples/") || rel.startsWith("examples/"),
|
||||
);
|
||||
// GitHub's buttons remain ordinary links; no third-party script is needed in the new shell.
|
||||
let converted = rewriteLinks(pinned, rel).replace(/<script\b[^>]*>[\s\S]*?<\/script>/g, "");
|
||||
converted = converted.replace(/^(\s*`{3,})slurm\b/gm, "$1bash");
|
||||
if (path.endsWith(".mdx"))
|
||||
converted = converted.replace(
|
||||
/<style>([\s\S]*?)<\/style>/g,
|
||||
(_, css) => `<style>{${JSON.stringify(css)}}</style>`,
|
||||
);
|
||||
{
|
||||
const aliases = (oldHeadings[route.slice(1)] ?? []).filter(
|
||||
(heading) =>
|
||||
!converted.includes(`id="${heading.id}"`) &&
|
||||
!converted.includes(`[#${heading.id}]`) &&
|
||||
!converted.includes(`{#${heading.id}}`),
|
||||
);
|
||||
converted += "\n" + aliases.map((heading) => `<a id="${heading.id}"></a>`).join("\n");
|
||||
}
|
||||
await writeFile(path, `---\n${stringify(data)}---\n${converted}`);
|
||||
}
|
||||
// API HTML is extracted directly from Python source with Griffe.
|
||||
await rm(join(content, "getting_started/v1_api.md"), { force: true });
|
||||
await rm(join(content, "getting_started/v1_api.mdx"), { force: true });
|
||||
await cp(join(generated, "python-reference"), content, { recursive: true });
|
||||
const originalHome = splitPage(await readFile(join(content, "index.md"), "utf8")).body;
|
||||
const section = originalHome.indexOf("## Key Features");
|
||||
if (section < 0) throw new Error("The homepage source needs an updated section boundary.");
|
||||
await mkdir(join(content, "_snippets"), { recursive: true });
|
||||
await mkdir(join(content, "_snippets/clients"), { recursive: true });
|
||||
for (const [file, language] of [
|
||||
["video.py", "python"],
|
||||
["video.mjs", "javascript"],
|
||||
["video.sh", "bash"],
|
||||
]) {
|
||||
const code = await readFile(join(repo, "examples/serving/clients", file), "utf8");
|
||||
await writeFile(join(content, "_snippets/clients", `${file}.md`), `\`\`\`${language}\n${code}\n\`\`\`\n`);
|
||||
}
|
||||
const asMdx = (body) =>
|
||||
body
|
||||
.replace(/\{#([^}]+)\}/g, "[#$1]")
|
||||
.replace(/<img\b([^>]*?)\/?\s*>/g, "<img$1 />")
|
||||
.replace(/ style="[^"]*"/g, "");
|
||||
const description = originalHome
|
||||
.slice(0, section)
|
||||
.split(/\n\s*\n/)
|
||||
.find((paragraph) => paragraph.trim().startsWith("FastVideo is an inference"));
|
||||
if (!description) throw new Error("The homepage description paragraph needs updating.");
|
||||
await writeFile(join(content, "_snippets/home-description.mdx"), asMdx(description.trim()));
|
||||
const documentation = originalHome.indexOf("## Documentation", section);
|
||||
if (documentation < 0) throw new Error("The homepage documentation section boundary needs updating.");
|
||||
await writeFile(join(content, "_snippets/home-details.mdx"), asMdx(originalHome.slice(section, documentation)));
|
||||
|
||||
// The README owns announcements and the community list; the welcome page renders a short news digest.
|
||||
const readme = await readFile(join(repo, "README.md"), "utf8");
|
||||
const news = readme.match(/^## NEWS\n([\s\S]*?)(?=^##|^###)/m)?.[1];
|
||||
const projects = readme.match(/^## Awesome work using FastVideo or our research projects\n([\s\S]*?)(?=^## )/m)?.[1];
|
||||
if (!news || !projects) throw new Error("The README news or project section boundary needs updating.");
|
||||
const recent = [...news.matchAll(/^- `([\d/]+)`: (.+)$/gm)].slice(0, 2).map(([, date, text]) => {
|
||||
const links = [...text.matchAll(/\[([^\]]+)\]\(([^)]+)\)/g)];
|
||||
const headline = text
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.split(/:\s|\.\s/)[0]
|
||||
.replace(/\.$/, "");
|
||||
const href = links.at(-1)?.[2] ?? "https://github.com/hao-ai-lab/FastVideo#news";
|
||||
const iso = date.replaceAll("/", "-");
|
||||
const label = new Intl.DateTimeFormat("en", {
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
year: "numeric",
|
||||
timeZone: "UTC",
|
||||
}).format(new Date(iso));
|
||||
return `<li><time dateTime="${iso}">${label}</time><a href={${JSON.stringify(href)}}>{${JSON.stringify(headline)}}</a></li>`;
|
||||
});
|
||||
await writeFile(
|
||||
join(content, "_snippets/home-news.mdx"),
|
||||
`<section className="home-news not-prose" aria-labelledby="recent-news">\n<div className="home-news-heading"><h2 id="recent-news">Recent news</h2><a className="home-news-more" href="https://github.com/hao-ai-lab/FastVideo#news">All announcements →</a></div>\n<ul>\n${recent.join("\n")}\n</ul>\n</section>\n`,
|
||||
);
|
||||
await writeFile(join(content, "_snippets/home-projects.mdx"), asMdx(projects.trim()));
|
||||
await rm(join(content, "index.md"));
|
||||
await rm(join(content, "cookbook/index.md"), { force: true });
|
||||
await rm(join(content, "cookbook/index.mdx"), { force: true });
|
||||
await cp(join(site, "content"), content, { recursive: true });
|
||||
// Model-specific inference pages come from the maintained recipe inventory.
|
||||
for (const model of modelDirectory.filter((model) => !model.configurable)) {
|
||||
const file = join(content, `${model.path.slice(1)}.mdx`);
|
||||
await mkdir(dirname(file), { recursive: true });
|
||||
await writeFile(
|
||||
file,
|
||||
`---\ntitle: ${JSON.stringify(model.title)}\nmode: frame\n---\n\n<Cookbook modelId="${model.id}" />\n`,
|
||||
);
|
||||
}
|
||||
|
||||
await cp(join(source, "assets"), join(publicDir, "assets"), {
|
||||
recursive: true,
|
||||
filter: (path) => !path.endsWith(".md"),
|
||||
});
|
||||
|
||||
for (const path of attachments) {
|
||||
await mkdir(dirname(join(publicDir, path)), { recursive: true });
|
||||
await cp(join(source, path), join(publicDir, path));
|
||||
}
|
||||
const routes = new Set(
|
||||
(await files(content))
|
||||
.filter(
|
||||
(path) =>
|
||||
/\.mdx?$/.test(path) &&
|
||||
!relative(content, path)
|
||||
.split(sep)
|
||||
.some((part) => /^[_.]/.test(part)),
|
||||
)
|
||||
.map((path) => routeFor(relative(content, path))),
|
||||
);
|
||||
const navigation = JSON.parse(await readFile(join(site, "navigation.json"), "utf8"));
|
||||
function checkNavigation(items) {
|
||||
for (const item of items) {
|
||||
if (typeof item === "string") {
|
||||
if (!routes.has(item)) throw new Error(`Navigation target is missing: ${item}`);
|
||||
} else {
|
||||
if (item.href && !routes.has(item.href)) throw new Error(`Navigation target is missing: ${item.href}`);
|
||||
if (item.items) checkNavigation(item.items);
|
||||
}
|
||||
}
|
||||
}
|
||||
checkNavigation(navigation);
|
||||
// Keep generated modules discoverable without putting all 880 entries in every page's sidebar.
|
||||
for (const path of await files(join(content, "api"))) {
|
||||
if (!path.endsWith(".md")) continue;
|
||||
const { data, body } = splitPage(await readFile(path, "utf8"));
|
||||
const folder = dirname(path);
|
||||
const children = (await readdir(folder, { withFileTypes: true })).filter((item) => item.isDirectory());
|
||||
const links = children
|
||||
.map(
|
||||
(item) =>
|
||||
`<li><a href="${routeFor(relative(content, join(folder, item.name, "index.md")))}">${item.name}</a></li>`,
|
||||
)
|
||||
.join("\n");
|
||||
await writeFile(
|
||||
path,
|
||||
`---\n${stringify(data)}---\n${body.replaceAll("{", "{").replaceAll("}", "}")}${links ? `\n<h2>Submodules</h2>\n<ul>\n${links}\n</ul>\n` : ""}`,
|
||||
);
|
||||
}
|
||||
function addModelPages(items) {
|
||||
for (const item of items) {
|
||||
const entries = modelDirectory.filter((model) => !model.configurable && model.familyPath === item.href);
|
||||
if (entries.length && !/Family guide|Recipes and setup/.test(item.label)) {
|
||||
const guide = { label: "Family guide", href: item.href };
|
||||
delete item.href;
|
||||
item.display = "group";
|
||||
item.collapsed = true;
|
||||
item.items = [...entries.map((model) => ({ label: model.title, href: model.path })), guide];
|
||||
} else if (item.items) {
|
||||
const guide = item.items.find((child) => /Family guide|Recipes and setup/.test(child.label));
|
||||
if (guide) {
|
||||
const extras = modelDirectory.filter((model) => !model.configurable && model.familyPath === guide.href);
|
||||
item.items.splice(
|
||||
item.items.indexOf(guide),
|
||||
0,
|
||||
...extras.map((model) => ({ label: model.title, href: model.path })),
|
||||
);
|
||||
}
|
||||
addModelPages(item.items);
|
||||
}
|
||||
}
|
||||
}
|
||||
addModelPages(navigation);
|
||||
await writeFile(join(generated, "navigation.json"), JSON.stringify(navigation));
|
||||
await writeFile(join(generated, "routes.json"), JSON.stringify([...routes].sort()));
|
||||
console.log(
|
||||
`Transferred ${routes.size} documentation routes. Canonical guide routes and cookbook runbooks are published together.`,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user