ignacio.ballesteros/quartz-org-roam
1
0
Fork 0
forked from github/quartz
Code Pull Requests Activity

make emitters async generators

Browse Source
This commit is contained in:
Jacky Zhao
2025-03-12 13:31:43 -07:00
parent 5d50282124
commit c5a8b199ae
18 changed files with 561 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
quartz/components/Breadcrumbs.tsx
View File

@@ -102,7 +102,7 @@ export default ((opts?: Partial<BreadcrumbOptions>) => {
// Add current slug to full path
currentPath = joinSegments(currentPath, slugParts[i])
const includeTrailingSlash = !isTagPath || i < 1
const includeTrailingSlash = !isTagPath || i < slugParts.length - 1
// Format and add current crumb
const crumb = formatCrumb(

16
quartz/plugins/emitters/404.tsx
View File

@@ -31,7 +31,7 @@ export const NotFoundPage: QuartzEmitterPlugin = () => {
async getDependencyGraph(_ctx, _content, _resources) {
return new DepGraph<FilePath>()
},
async emit(ctx, _content, resources): Promise<FilePath[]> {
async *emit(ctx, _content, resources) {
const cfg = ctx.cfg.configuration
const slug = "404" as FullSlug
@@ -55,14 +55,12 @@ export const NotFoundPage: QuartzEmitterPlugin = () => {
allFiles: [],
}
return [
await write({
ctx,
content: renderPage(cfg, slug, componentData, opts, externalResources),
slug,
ext: ".html",
}),
]
yield write({
ctx,
content: renderPage(cfg, slug, componentData, opts, externalResources),
slug,
ext: ".html",
})
},
}
}

9
quartz/plugins/emitters/aliases.ts
View File

@@ -18,15 +18,13 @@ export const AliasRedirects: QuartzEmitterPlugin = () => ({
return graph
},
async emit(ctx, content, _resources): Promise<FilePath[]> {
const fps: FilePath[] = []
async *emit(ctx, content, _resources) {
for (const [_tree, file] of content) {
const ogSlug = simplifySlug(file.data.slug!)
for (const slug of file.data.aliases ?? []) {
const redirUrl = resolveRelative(slug, file.data.slug!)
const fp = await write({
yield write({
ctx,
content: `
<!DOCTYPE html>
@@ -43,10 +41,7 @@ export const AliasRedirects: QuartzEmitterPlugin = () => ({
slug,
ext: ".html",
})
fps.push(fp)
}
}
return fps
},
})

7
quartz/plugins/emitters/assets.ts
View File

@@ -33,10 +33,9 @@ export const Assets: QuartzEmitterPlugin = () => {
return graph
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
async *emit({ argv, cfg }, _content, _resources) {
const assetsPath = argv.output
const fps = await filesToCopy(argv, cfg)
const res: FilePath[] = []
for (const fp of fps) {
const ext = path.extname(fp)
const src = joinSegments(argv.directory, fp) as FilePath
@@ -46,10 +45,8 @@ export const Assets: QuartzEmitterPlugin = () => {
const dir = path.dirname(dest) as FilePath
await fs.promises.mkdir(dir, { recursive: true }) // ensure dir exists
await fs.promises.copyFile(src, dest)
res.push(dest)
yield dest
}
return res
},
}
}

4
quartz/plugins/emitters/cname.ts
View File

@@ -14,7 +14,7 @@ export const CNAME: QuartzEmitterPlugin = () => ({
async getDependencyGraph(_ctx, _content, _resources) {
return new DepGraph<FilePath>()
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
async emit({ argv, cfg }, _content, _resources) {
if (!cfg.configuration.baseUrl) {
console.warn(chalk.yellow("CNAME emitter requires `baseUrl` to be set in your configuration"))
return []
@@ -24,7 +24,7 @@ export const CNAME: QuartzEmitterPlugin = () => ({
if (!content) {
return []
}
fs.writeFileSync(path, content)
await fs.promises.writeFile(path, content)
return [path] as FilePath[]
},
})

111
quartz/plugins/emitters/componentResources.ts
View File

@@ -9,7 +9,7 @@ import styles from "../../styles/custom.scss"
import popoverStyle from "../../components/styles/popover.scss"
import { BuildCtx } from "../../util/ctx"
import { QuartzComponent } from "../../components/types"
import { googleFontHref, joinStyles } from "../../util/theme"
import { googleFontHref, joinStyles, processGoogleFonts } from "../../util/theme"
import { Features, transform } from "lightningcss"
import { transform as transpile } from "esbuild"
import { write } from "./helpers"
@@ -207,8 +207,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
async getDependencyGraph(_ctx, _content, _resources) {
return new DepGraph<FilePath>()
},
async emit(ctx, _content, _resources): Promise<FilePath[]> {
const promises: Promise<FilePath>[] = []
async *emit(ctx, _content, _resources) {
const cfg = ctx.cfg.configuration
// component specific scripts and styles
const componentResources = getComponentResources(ctx)
@@ -217,42 +216,35 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
// let the user do it themselves in css
} else if (cfg.theme.fontOrigin === "googleFonts" && !cfg.theme.cdnCaching) {
// when cdnCaching is true, we link to google fonts in Head.tsx
let match
const response = await fetch(googleFontHref(ctx.cfg.configuration.theme))
googleFontsStyleSheet = await response.text()
const fontSourceRegex = /url\((https:\/\/fonts.gstatic.com\/s\/[^)]+\.(woff2|ttf))\)/g
googleFontsStyleSheet = await (
await fetch(googleFontHref(ctx.cfg.configuration.theme))
).text()
while ((match = fontSourceRegex.exec(googleFontsStyleSheet)) !== null) {
// match[0] is the `url(path)`, match[1] is the `path`
const url = match[1]
// the static name of this file.
const [filename, ext] = url.split("/").pop()!.split(".")
googleFontsStyleSheet = googleFontsStyleSheet.replace(
url,
`https://${cfg.baseUrl}/static/fonts/${filename}.ttf`,
if (!cfg.baseUrl) {
throw new Error(
"baseUrl must be defined when using Google Fonts without cfg.theme.cdnCaching",
)
}
promises.push(
fetch(url)
.then((res) => {
if (!res.ok) {
throw new Error(`Failed to fetch font`)
}
return res.arrayBuffer()
})
.then((buf) =>
write({
ctx,
slug: joinSegments("static", "fonts", filename) as FullSlug,
ext: `.${ext}`,
content: Buffer.from(buf),
}),
),
)
const { processedStylesheet, fontFiles } = await processGoogleFonts(
googleFontsStyleSheet,
cfg.baseUrl,
)
googleFontsStyleSheet = processedStylesheet
// Download and save font files
for (const fontFile of fontFiles) {
const res = await fetch(fontFile.url)
if (!res.ok) {
throw new Error(`failed to fetch font ${fontFile.filename}`)
}
const buf = await res.arrayBuffer()
yield write({
ctx,
slug: joinSegments("static", "fonts", fontFile.filename) as FullSlug,
ext: `.${fontFile.extension}`,
content: Buffer.from(buf),
})
}
}
@@ -267,45 +259,42 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
...componentResources.css,
styles,
)
const [prescript, postscript] = await Promise.all([
joinScripts(componentResources.beforeDOMLoaded),
joinScripts(componentResources.afterDOMLoaded),
])
promises.push(
write({
ctx,
slug: "index" as FullSlug,
ext: ".css",
content: transform({
filename: "index.css",
code: Buffer.from(stylesheet),
minify: true,
targets: {
safari: (15 << 16) | (6 << 8), // 15.6
ios_saf: (15 << 16) | (6 << 8), // 15.6
edge: 115 << 16,
firefox: 102 << 16,
chrome: 109 << 16,
},
include: Features.MediaQueries,
}).code.toString(),
}),
write({
yield write({
ctx,
slug: "index" as FullSlug,
ext: ".css",
content: transform({
filename: "index.css",
code: Buffer.from(stylesheet),
minify: true,
targets: {
safari: (15 << 16) | (6 << 8), // 15.6
ios_saf: (15 << 16) | (6 << 8), // 15.6
edge: 115 << 16,
firefox: 102 << 16,
chrome: 109 << 16,
},
include: Features.MediaQueries,
}).code.toString(),
}),
yield write({
ctx,
slug: "prescript" as FullSlug,
ext: ".js",
content: prescript,
}),
write({
yield write({
ctx,
slug: "postscript" as FullSlug,
ext: ".js",
content: postscript,
}),
)
return await Promise.all(promises)
})
},
}
}

47
quartz/plugins/emitters/contentIndex.tsx
View File

@@ -116,9 +116,8 @@ export const ContentIndex: QuartzEmitterPlugin<Partial<Options>> = (opts) => {
return graph
},
async emit(ctx, content, _resources) {
async *emit(ctx, content, _resources) {
const cfg = ctx.cfg.configuration
const emitted: FilePath[] = []
const linkIndex: ContentIndexMap = new Map()
for (const [tree, file] of content) {
const slug = file.data.slug!
@@ -140,25 +139,21 @@ export const ContentIndex: QuartzEmitterPlugin<Partial<Options>> = (opts) => {
}
if (opts?.enableSiteMap) {
emitted.push(
await write({
ctx,
content: generateSiteMap(cfg, linkIndex),
slug: "sitemap" as FullSlug,
ext: ".xml",
}),
)
yield write({
ctx,
content: generateSiteMap(cfg, linkIndex),
slug: "sitemap" as FullSlug,
ext: ".xml",
})
}
if (opts?.enableRSS) {
emitted.push(
await write({
ctx,
content: generateRSSFeed(cfg, linkIndex, opts.rssLimit),
slug: (opts?.rssSlug ?? "index") as FullSlug,
ext: ".xml",
}),
)
yield write({
ctx,
content: generateRSSFeed(cfg, linkIndex, opts.rssLimit),
slug: (opts?.rssSlug ?? "index") as FullSlug,
ext: ".xml",
})
}
const fp = joinSegments("static", "contentIndex") as FullSlug
@@ -173,16 +168,12 @@ export const ContentIndex: QuartzEmitterPlugin<Partial<Options>> = (opts) => {
}),
)
emitted.push(
await write({
ctx,
content: JSON.stringify(simplifiedIndex),
slug: fp,
ext: ".json",
}),
)
return emitted
yield write({
ctx,
content: JSON.stringify(simplifiedIndex),
slug: fp,
ext: ".json",
})
},
externalResources: (ctx) => {
if (opts?.enableRSS) {

9
quartz/plugins/emitters/contentPage.tsx
View File

@@ -94,9 +94,8 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
return graph
},
async emit(ctx, content, resources): Promise<FilePath[]> {
async *emit(ctx, content, resources) {
const cfg = ctx.cfg.configuration
const fps: FilePath[] = []
const allFiles = content.map((c) => c[1].data)
let containsIndex = false
@@ -118,14 +117,12 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
}
const content = renderPage(cfg, slug, componentData, opts, externalResources)
const fp = await write({
yield write({
ctx,
content,
slug,
ext: ".html",
})
fps.push(fp)
}
if (!containsIndex && !ctx.argv.fastRebuild) {
@@ -135,8 +132,6 @@ export const ContentPage: QuartzEmitterPlugin<Partial<FullPageLayout>> = (userOp
),
)
}
return fps
},
}
}

8
quartz/plugins/emitters/folderPage.tsx
View File

@@ -69,8 +69,7 @@ export const FolderPage: QuartzEmitterPlugin<Partial<FolderPageOptions>> = (user
return graph
},
async emit(ctx, content, resources): Promise<FilePath[]> {
const fps: FilePath[] = []
async *emit(ctx, content, resources) {
const allFiles = content.map((c) => c[1].data)
const cfg = ctx.cfg.configuration
@@ -119,16 +118,13 @@ export const FolderPage: QuartzEmitterPlugin<Partial<FolderPageOptions>> = (user
}
const content = renderPage(cfg, slug, componentData, opts, externalResources)
const fp = await write({
yield write({
ctx,
content,
slug,
ext: ".html",
})
fps.push(fp)
}
return fps
},
}
}

13
quartz/plugins/emitters/ogImage.tsx
View File

@@ -1,7 +1,7 @@
import { QuartzEmitterPlugin } from "../types"
import { i18n } from "../../i18n"
import { unescapeHTML } from "../../util/escape"
import { FilePath, FullSlug, getFileExtension } from "../../util/path"
import { FullSlug, getFileExtension } from "../../util/path"
import { ImageOptions, SocialImageOptions, defaultImage, getSatoriFont } from "../../util/og"
import { getFontSpecificationName } from "../../util/theme"
import sharp from "sharp"
@@ -52,10 +52,8 @@ export const CustomOgImages: QuartzEmitterPlugin<Partial<SocialImageOptions>> =
getQuartzComponents() {
return []
},
async emit(ctx, content, _resources) {
async *emit(ctx, content, _resources) {
const cfg = ctx.cfg.configuration
const emittedFiles: FilePath[] = []
const headerFont = getFontSpecificationName(cfg.theme.typography.header)
const bodyFont = getFontSpecificationName(cfg.theme.typography.body)
const fonts = await getSatoriFont(headerFont, bodyFont)
@@ -88,17 +86,13 @@ export const CustomOgImages: QuartzEmitterPlugin<Partial<SocialImageOptions>> =
fullOptions,
)
const fp = await write({
yield write({
ctx,
content: stream,
slug: `${slug}-og-image` as FullSlug,
ext: ".webp",
})
emittedFiles.push(fp)
}
return emittedFiles
},
externalResources: (ctx) => {
if (!ctx.cfg.configuration.baseUrl) {
@@ -115,7 +109,6 @@ export const CustomOgImages: QuartzEmitterPlugin<Partial<SocialImageOptions>> =
? `https://${baseUrl}/${pageData.slug!}-og-image.webp`
: undefined
const defaultOgImagePath = `https://${baseUrl}/static/og-image.png`
const ogImagePath = userDefinedOgImagePath ?? generatedOgImagePath ?? defaultOgImagePath
const ogImageMimeType = `image/${getFileExtension(ogImagePath) ?? "png"}`

17
quartz/plugins/emitters/static.ts
View File

@@ -3,6 +3,7 @@ import { QuartzEmitterPlugin } from "../types"
import fs from "fs"
import { glob } from "../../util/glob"
import DepGraph from "../../depgraph"
import { dirname } from "path"
export const Static: QuartzEmitterPlugin = () => ({
name: "Static",
@@ -20,13 +21,17 @@ export const Static: QuartzEmitterPlugin = () => ({
return graph
},
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
async *emit({ argv, cfg }, _content) {
const staticPath = joinSegments(QUARTZ, "static")
const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), {
recursive: true,
dereference: true,
})
return fps.map((fp) => joinSegments(argv.output, "static", fp)) as FilePath[]
const outputStaticPath = joinSegments(argv.output, "static")
await fs.promises.mkdir(outputStaticPath, { recursive: true })
for (const fp of fps) {
const src = joinSegments(staticPath, fp) as FilePath
const dest = joinSegments(outputStaticPath, fp) as FilePath
await fs.promises.mkdir(dirname(dest), { recursive: true })
await fs.promises.copyFile(src, dest)
yield dest
}
},
})

8
quartz/plugins/emitters/tagPage.tsx
View File

@@ -71,8 +71,7 @@ export const TagPage: QuartzEmitterPlugin<Partial<TagPageOptions>> = (userOpts)
return graph
},
async emit(ctx, content, resources): Promise<FilePath[]> {
const fps: FilePath[] = []
async *emit(ctx, content, resources) {
const allFiles = content.map((c) => c[1].data)
const cfg = ctx.cfg.configuration
@@ -127,16 +126,13 @@ export const TagPage: QuartzEmitterPlugin<Partial<TagPageOptions>> = (userOpts)
}
const content = renderPage(cfg, slug, componentData, opts, externalResources)
const fp = await write({
yield write({
ctx,
content,
slug: file.data.slug!,
ext: ".html",
})
fps.push(fp)
}
return fps
},
}
}

6
quartz/plugins/types.ts
View File

@@ -38,7 +38,11 @@ export type QuartzEmitterPlugin<Options extends OptionType = undefined> = (
) => QuartzEmitterPluginInstance
export type QuartzEmitterPluginInstance = {
name: string
emit(ctx: BuildCtx, content: ProcessedContent[], resources: StaticResources): Promise<FilePath[]>
emit(
ctx: BuildCtx,
content: ProcessedContent[],
resources: StaticResources,
): Promise<FilePath[]> | AsyncGenerator<FilePath>
/**
* Returns the components (if any) that are used in rendering the page.
* This helps Quartz optimize the page by only including necessary resources

38
quartz/processors/emit.ts
View File

@@ -14,20 +14,34 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) {
let emittedFiles = 0
const staticResources = getStaticResourcesFromPlugins(ctx)
for (const emitter of cfg.plugins.emitters) {
try {
const emitted = await emitter.emit(ctx, content, staticResources)
emittedFiles += emitted.length
if (ctx.argv.verbose) {
for (const file of emitted) {
console.log(`[emit:${emitter.name}] ${file}`)
await Promise.all(
cfg.plugins.emitters.map(async (emitter) => {
try {
const emitted = await emitter.emit(ctx, content, staticResources)
if (Symbol.asyncIterator in emitted) {
// Async generator case
const files: string[] = []
for await (const file of emitted) {
files.push(file)
emittedFiles++
if (ctx.argv.verbose) {
console.log(`[emit:${emitter.name}] ${file}`)
}
}
} else {
// Array case
emittedFiles += emitted.length
if (ctx.argv.verbose) {
for (const file of emitted) {
console.log(`[emit:${emitter.name}] ${file}`)
}
}
}
} catch (err) {
trace(`Failed to emit from plugin \`${emitter.name}\``, err as Error)
}
} catch (err) {
trace(`Failed to emit from plugin \`${emitter.name}\``, err as Error)
}
}
}),
)
log.end(`Emitted ${emittedFiles} files to \`${argv.output}\` in ${perf.timeSince()}`)
}

66
quartz/util/og.tsx
View File

@@ -15,8 +15,10 @@ export async function getSatoriFont(headerFontName: string, bodyFontName: string
const bodyWeight = 400 as FontWeight
// Fetch fonts
const headerFont = await fetchTtf(headerFontName, headerWeight)
const bodyFont = await fetchTtf(bodyFontName, bodyWeight)
const [headerFont, bodyFont] = await Promise.all([
fetchTtf(headerFontName, headerWeight),
fetchTtf(bodyFontName, bodyWeight),
])
// Convert fonts to satori font format and return
const fonts: SatoriOptions["fonts"] = [
@@ -26,38 +28,48 @@ export async function getSatoriFont(headerFontName: string, bodyFontName: string
return fonts
}
// Cache for memoizing font data
const fontCache = new Map<string, Promise<ArrayBuffer>>()
/**
* Get the `.ttf` file of a google font
* @param fontName name of google font
* @param weight what font weight to fetch font
* @returns `.ttf` file of google font
*/
async function fetchTtf(fontName: string, weight: FontWeight): Promise<ArrayBuffer> {
try {
// Get css file from google fonts
const cssResponse = await fetch(
`https://fonts.googleapis.com/css2?family=${fontName}:wght@${weight}`,
)
const css = await cssResponse.text()
// Extract .ttf url from css file
const urlRegex = /url\((https:\/\/fonts.gstatic.com\/s\/.*?.ttf)\)/g
const match = urlRegex.exec(css)
if (!match) {
throw new Error("Could not fetch font")
}
// Retrieve font data as ArrayBuffer
const fontResponse = await fetch(match[1])
// fontData is an ArrayBuffer containing the .ttf file data (get match[1] due to google fonts response format, always contains link twice, but second entry is the "raw" link)
const fontData = await fontResponse.arrayBuffer()
return fontData
} catch (error) {
throw new Error(`Error fetching font: ${error}`)
export async function fetchTtf(fontName: string, weight: FontWeight): Promise<ArrayBuffer> {
const cacheKey = `${fontName}-${weight}`
if (fontCache.has(cacheKey)) {
return fontCache.get(cacheKey)!
}
// If not in cache, fetch and store the promise
const fontPromise = (async () => {
try {
// Get css file from google fonts
const cssResponse = await fetch(
`https://fonts.googleapis.com/css2?family=${fontName}:wght@${weight}`,
)
const css = await cssResponse.text()
// Extract .ttf url from css file
const urlRegex = /url\((https:\/\/fonts.gstatic.com\/s\/.*?.ttf)\)/g
const match = urlRegex.exec(css)
if (!match) {
throw new Error("Could not fetch font")
}
// fontData is an ArrayBuffer containing the .ttf file data (get match[1] due to google fonts response format, always contains link twice, but second entry is the "raw" link)
const fontResponse = await fetch(match[1])
return await fontResponse.arrayBuffer()
} catch (error) {
throw new Error(`Error fetching font: ${error}`)
}
})()
fontCache.set(cacheKey, fontPromise)
return fontPromise
}
export type SocialImageOptions = {

30
quartz/util/theme.ts
View File

@@ -90,6 +90,36 @@ export function googleFontHref(theme: Theme) {
return `https://fonts.googleapis.com/css2?family=${bodyFont}&family=${headerFont}&family=${codeFont}&display=swap`
}
export interface GoogleFontFile {
url: string
filename: string
extension: string
}
export async function processGoogleFonts(
stylesheet: string,
baseUrl: string,
): Promise<{
processedStylesheet: string
fontFiles: GoogleFontFile[]
}> {
const fontSourceRegex = /url\((https:\/\/fonts.gstatic.com\/s\/[^)]+\.(woff2|ttf))\)/g
const fontFiles: GoogleFontFile[] = []
let processedStylesheet = stylesheet
let match
while ((match = fontSourceRegex.exec(stylesheet)) !== null) {
const url = match[1]
const [filename, extension] = url.split("/").pop()!.split(".")
const staticUrl = `https://${baseUrl}/static/fonts/${filename}.${extension}`
processedStylesheet = processedStylesheet.replace(url, staticUrl)
fontFiles.push({ url, filename, extension })
}
return { processedStylesheet, fontFiles }
}
export function joinStyles(theme: Theme, ...stylesheet: string[]) {
return `
${stylesheet.join("\n\n")}