From 176d4d711e7f78bce5da55175815284228209390 Mon Sep 17 00:00:00 2001 From: William Laugesen Date: Fri, 21 Aug 2026 11:07:26 +1200 Subject: [PATCH] Rank from a title map, and fetch only the rows on screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ranking needed a url and a title, and Pagefind keeps both in the per-page fragment — so ordering thirty results meant fetching thirty files before the panel could draw, and a page ranked past thirty could not be reached at all. The build now writes a map of result id to url and title beside the index, which is the join Pagefind's own result stub already carries. So the whole result set is ranked before anything is fetched, and fragments are fetched only for the rows being drawn: ten per batch, against thirty to thirty-five for every settled query before. On Slow 4G with the map served uncompressed, first results arrive in 7.2s against 8.9s; the map is 28 KB gzipped, so most of that 122 KB is transfer a CDN removes. The shallow-page search this replaces is gone with it — the second Pagefind query, the landing filter, LANDING_DEPTH and the attribute it needed on every page's content div. A page the query names now wins from anywhere in the list rather than from a shortlist of 227. Relevance holds on both traffic-weighted sets: real-searches 57% w-S@1 and 84% w-S@5, top-pages 90% and 98%, unchanged either side. Co-Authored-By: Claude Opus 5 (1M context) --- src/integrations/pagefind-index.ts | 68 +++++++ src/layouts/Api.astro | 2 +- src/layouts/Default.astro | 2 +- src/lib/searchIndexing.ts | 29 +-- src/scripts/search-engine-pagefind.ts | 244 ++++++++++++++------------ tests/docs-search.spec.ts | 13 +- 6 files changed, 206 insertions(+), 152 deletions(-) diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index a61f83438b..3c574bfdd2 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -10,6 +10,8 @@ import type { AstroIntegration } from 'astro'; import { fileURLToPath } from 'node:url'; import * as path from 'node:path'; +import { gunzipSync } from 'node:zlib'; +import { readdir, readFile, writeFile } from 'node:fs/promises'; // Statically, because `astro:build:done` fires after Vite's module runner has // closed and a dynamic import from inside the hook cannot be resolved. import { createIndex, close } from 'pagefind'; @@ -81,6 +83,11 @@ export default function pagefindIndex(): AstroIntegration { // Files scanned, not pages indexed: the redirect stubs are counted // here and then dropped for having no article. logger.info(`scanned ${added.page_count} pages into docs/pagefind`); + + const titles = await writeTitleMap( + path.join(distDir, 'docs', 'pagefind') + ); + logger.info(`wrote ${titles} titles to ${TITLE_MAP}`); } finally { await close(); } @@ -88,3 +95,64 @@ export default function pagefindIndex(): AstroIntegration { }, }; } + +/** Where the map lands, and the name the client fetches it by. */ +const TITLE_MAP = 'docs-titles.json'; + +// Pagefind prefixes every decompressed chunk with this before the JSON. +const FRAGMENT_MAGIC = 'pagefind_dcd'; + +/** + * Writes what the overlay needs to rank a result without fetching it. + * + * Ranking needs a URL and a title, and Pagefind keeps both in the per-page + * fragment — so ranking thirty results meant fetching thirty files, and a + * landing page ranked past that could not be reached at all. A search result + * stub carries the id of its own fragment, so one map from id to url and title + * lets the whole result set be ranked from a single file. + * + * Read back out of the fragments rather than collected during indexing, because + * the ids are assigned by Pagefind as it writes them. + */ +async function writeTitleMap(pagefindDir: string): Promise { + const dir = path.join(pagefindDir, 'fragment'); + const files = (await readdir(dir)).filter((name) => + name.endsWith('.pf_fragment') + ); + + const map: Record = {}; + + for (const file of files) { + const raw = gunzipSync(await readFile(path.join(dir, file))).toString( + 'utf8' + ); + + // Checked before parsing: a Pagefind release that changes the chunk format + // has to fail the build here, rather than write a map the overlay silently + // cannot join against. + if (!raw.startsWith(FRAGMENT_MAGIC)) { + throw new Error( + `unexpected fragment format in ${file}: Pagefind's own prefix is missing, so ${TITLE_MAP} cannot be trusted` + ); + } + + const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as { + url: string; + meta?: Record; + }; + + // The stub's `id` is the filename without its extension, which is the join. + map[path.basename(file, '.pf_fragment')] = [ + fragment.url, + fragment.meta?.title ?? '', + ]; + } + + await writeFile( + path.join(pagefindDir, TITLE_MAP), + JSON.stringify(map), + 'utf8' + ); + + return files.length; +} diff --git a/src/layouts/Api.astro b/src/layouts/Api.astro index b584c954ae..f96c01b8cd 100644 --- a/src/layouts/Api.astro +++ b/src/layouts/Api.astro @@ -101,7 +101,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */ } -
+
diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a5711494fd..3d4f62a740 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); lang={lang} />
-
+
diff --git a/src/lib/searchIndexing.ts b/src/lib/searchIndexing.ts index b21aa20b17..ea46993cdb 100644 --- a/src/lib/searchIndexing.ts +++ b/src/lib/searchIndexing.ts @@ -13,30 +13,12 @@ type ArticleAttributes = { 'data-pagefind-default-meta'?: string; }; -/** - * A second filter, on an element inside the article rather than on the article - * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a - * comma-separated pair is taken as a single value. - */ -type ContentAttributes = { - 'data-pagefind-filter'?: string; -}; - type IndexAttributes = { article: ArticleAttributes; - content: ContentAttributes; }; /** - * How shallow a page has to be to count as one a reader might name. Two segments - * past `/docs/`, which covers `/docs/deployments/` and - * `/docs/infrastructure/deployment-targets/` but not the pages inside them. - */ -const LANDING_DEPTH = 3; - -/** - * The `data-pagefind-*` attributes for a page: `article` spreads onto the - * `
`, `content` onto the page content inside it. + * The `data-pagefind-*` attributes for a page, to spread onto the `
`. * * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that @@ -53,13 +35,7 @@ export function searchIndexAttributes( // `all` rather than the default `index`: a bare ignore still lets Pagefind // read a title or metadata out of the block. - if (!indexable) - return { article: { 'data-pagefind-ignore': 'all' }, content: {} }; - - // Marks the pages the overlay's second, narrowed search looks through. Only - // the shallow pages carry it, so the filter chunk stays small and that search - // has a few hundred candidates rather than the whole site. - const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH; + if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } }; return { article: { @@ -75,6 +51,5 @@ export function searchIndexAttributes( ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` } : {}), }, - content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {}, }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index e6b902add9..16a60e5c99 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -12,10 +12,13 @@ import { type SearchResult, } from './search-engine'; -// Rows fetched at a time. Pagefind's own UI shows five and offers the rest on -// demand; thirty, because `byNameThenDepth` reorders within a page and needs -// enough of the list to have something to reorder. -const PAGE_SIZE = 30; +// Rows fetched at a time. The panel shows about five, and the rest arrive as it +// is scrolled. Ranking no longer needs them, so this is only a drawing budget: +// ten covers the first screen with room to scroll into. +const PAGE_SIZE = 10; + +/** The map of result id to url and title, written beside the index at build. */ +const TITLE_MAP = 'docs-titles.json'; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -28,11 +31,6 @@ const PAGE_SIZE = 30; // 79%. A mash landing on the gentler message costs nothing; both offer no rows. const COMMON_TERM_SHARE = 0.8; -// How many shallow pages the named-page lookup looks through. Measured over 18 -// section queries: three finds the page for 16, five for 17, and twenty finds no -// more than five does. -const LANDING_CANDIDATES = 5; - type PagefindSubResult = { title: string; /** Carries the heading's `#anchor` when the match is below the page title. */ @@ -171,8 +169,10 @@ function claimsName(hit: { url: string; title: string }, term: string) { * order for none of them; `data-pagefind-weight` on the h1 measured as no change * at all. * - * Reaches only the results already fetched, which is what `namedPage` below is - * for: a page ranked past `PAGE_SIZE` on raw score cannot be rescued here. + * Runs over every result, because `rank` supplies url and title from the title + * map and nothing here has to be fetched. So a page the query names wins from + * anywhere in the list — `/docs/infrastructure/deployment-targets/` is 36th on + * raw score for "deployment targets" and still comes first. */ function byNameThenDepth< T extends { score: number; url: string; title: string }, @@ -196,19 +196,58 @@ function byNameThenDepth< } /** A stub's score paired with its fetched fragment, which is where the URL is. */ -type Hit = { - fragment: PagefindFragment; +/** Everything ranking needs, and nothing that has to be fetched to get it. */ +type Ranked = { + stub: PagefindResultStub; score: number; url: string; title: string; }; +type TitleMap = Record; + +/** + * Pairs each result with its url and title from the map, so the whole set can be + * ranked before anything is fetched. + * + * A result the map does not know is dropped from ranking. That only happens when + * the map and the index disagree, which means a stale deploy of one of them; the + * caller falls back to ranking what it fetches. + */ +function rank( + stubs: PagefindResultStub[], + titles: TitleMap, + term: string, + prefix: string +): Ranked[] { + const known = stubs.flatMap((stub) => { + const entry = titles[stub.id]; + if (!entry) return []; + // The map holds urls as the index does, relative to the indexed directory. + // Pagefind applies the same prefix to the urls it returns from `data()`. + const [path, title] = entry; + const url = prefix + path; + return [{ stub, score: stub.score, url, title: title || url }]; + }); + + return byNameThenDepth(known, term); +} + /** - * Fetches the fragment for each stub. A fragment that fails takes its own row - * out rather than the whole result set. + * Draws a slice of the ranked list, fetching a fragment for each row in it. The + * fragment supplies the excerpt and the matched headings; the order was settled + * before any of it was asked for. + * + * `from` is how many rows already precede these, which keeps the headings on the + * leading rows of the list rather than the leading rows of every batch. */ -async function hydrate(stubs: PagefindResultStub[]): Promise { - const settled = await Promise.allSettled(stubs.map((stub) => stub.data())); +async function draw( + ranked: Ranked[], + from: number, + count: number +): Promise { + const slice = ranked.slice(from, from + count); + const settled = await Promise.allSettled(slice.map((hit) => hit.stub.data())); return settled.flatMap((outcome, at) => { if (outcome.status === 'rejected') { @@ -219,77 +258,38 @@ async function hydrate(stubs: PagefindResultStub[]): Promise { return []; } + const hit = slice[at]; const fragment = outcome.value; + // The fragment is the fallback for both: without the title map, `rank` has + // no url or title to give and the fragment is the only source. const { pathname } = new URL(fragment.url, window.location.origin); return [ { - fragment, - score: stubs[at].score, - url: pathname, - title: fragment.meta?.title ?? pathname, + url: hit.url || pathname, + title: hit.title || fragment.meta?.title || pathname, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: fragment.excerpt, + breadcrumb: breadcrumbFrom(hit.url || pathname), + sections: + from + at < ROWS_WITH_SECTIONS ? sectionsOf(fragment) : undefined, + ...classify(hit.url || pathname), }, ]; }); } -/** - * One page of rows, ordered within itself. `from` is how many rows already - * precede them, which is what keeps the headings on the leading pages of the - * list rather than on the leading rows of every page. - */ -function rows(hits: Hit[], term: string, from: number): SearchResult[] { - return byNameThenDepth(hits, term).map((hit, rank) => ({ - url: hit.url, - title: hit.title, - // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: hit.fragment.excerpt, - breadcrumb: breadcrumbFrom(hit.url), - sections: - from + rank < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, - ...classify(hit.url), - })); -} - -/** - * The page the query names, when the first page of results missed it. - * - * `byNameThenDepth` can only promote what has been fetched, and a section's own - * page can rank far below the pages inside it: `/docs/infrastructure/ - * deployment-targets/` is 36th for "deployment targets", six places past the - * page size. These stubs come from a search narrowed to the shallow pages alone, - * where the page a query names sits near the top of a few hundred candidates. - * - * Only called when nothing already fetched names the query, so the extra - * fragments are paid for by the queries that need them and no others. - */ -async function namedPage( - stubs: PagefindResultStub[], - term: string, - already: Hit[] -): Promise { - const seen = new Set(already.map((hit) => hit.url)); - const candidates = await hydrate(stubs.slice(0, LANDING_CANDIDATES)); - - return ( - candidates.find((hit) => claimsName(hit, term) && !seen.has(hit.url)) ?? - null - ); -} - export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; - // Everything the last search matched, and how much of it has been handed over. - // A stub is a score and a promise of its fragment, so holding a thousand of - // them costs nothing and saves searching again to show row thirty-one. - // `promoted` is the page `namedPage` pulled forward, whose own stub is still - // waiting further down `stubs`. - let page: { - term: string; - stubs: PagefindResultStub[]; - at: number; - promoted: string | null; - } | null = null; + // The last search's whole result set, already ranked, and how much of it has + // been drawn. Ranking the tail costs nothing because it needs no fetches, so + // `more()` only has to draw the next slice. + let page: { term: string; ranked: Ranked[]; at: number } | null = null; + // The map from result id to url and title, fetched once with the index. + let titles: TitleMap | null = null; + // What Pagefind prepends to the urls it returns, and therefore what the map's + // own relative urls need. `/docs/pagefind/` leaves `/docs`. + const urlPrefix = bundlePath.replace(/\/?pagefind\/?$/, ''); // Which search owns `page`. Searches run concurrently and can settle out of // order, and an overtaken one must not leave its stubs behind for `more()`. let searches = 0; @@ -311,7 +311,20 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The filter index is a separate chunk, and a search returns empty filter // counts until it has been pulled down. Its section totals also add up to // the size of the corpus, which is what `COMMON_TERM_SHARE` is a share of. - const filters = await api.filters(); + const [filters] = await Promise.all([ + api.filters(), + // Ranking reads url and title out of this, so it has to be here before + // the first search returns. A failure leaves it null and ranking falls + // back to ordering the rows it draws. + fetch(`${bundlePath}${TITLE_MAP}`) + .then((response) => (response.ok ? response.json() : null)) + .then((map: TitleMap | null) => { + titles = map; + }) + .catch((error) => { + console.error(`[docs-search] could not load ${TITLE_MAP}`, error); + }), + ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, 0 @@ -358,16 +371,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const filters = facet && facet !== 'all' ? { section: [facet] } : undefined; - // Three searches at once, because a second await here would sit in front - // of every fragment fetch below it. The first supplies the rows; the - // second says whether the query has any answer at all, and only runs - // while a tab is narrowing the first; the third is the shallow-page - // shortlist `namedPage` draws on, which costs nothing until its - // fragments are fetched. - const [response, wholeCorpus, landing] = await Promise.all([ + // Together, because a second await here would sit in front of every + // fragment fetch below it. The first supplies the rows; the second says + // whether the query has any answer at all, and only runs while a tab is + // narrowing the first. + const [response, wholeCorpus] = await Promise.all([ api.search(query, { filters }), filters ? api.search(query) : null, - api.search(query, { filters: { ...filters, landing: ['true'] } }), ]); const unfiltered = wholeCorpus ?? response; @@ -395,29 +405,37 @@ export function pagefindEngine(bundlePath: string): SearchEngine { : empty; } - const hits = await hydrate(response.results.slice(0, PAGE_SIZE)); - - const named = hits.some((hit) => claimsName(hit, query)) - ? null - : await namedPage(landing.results, query, hits); - if (named) hits.push(named); - - // A promoted page was fetched precisely because its own stub ranks past - // `PAGE_SIZE`, so that stub is still ahead of `more()` and has to be - // skipped there rather than drawn a second time. - settle({ - term: query, - stubs: response.results, - at: PAGE_SIZE, - promoted: named?.url ?? null, - }); + // Every match is ranked here, whether it will be drawn or not. A page + // the query names wins from anywhere in the list, which is what the + // shallow-page search used to be for. + const ranked = titles + ? rank(response.results, titles, query, urlPrefix) + : []; + + // The map was missing or disagreed with the index. Ranking what gets + // drawn is worse than ranking everything, and it still answers. + const fallback = + ranked.length === 0 && response.results.length > 0 + ? response.results.map((stub) => ({ + stub, + score: stub.score, + url: '', + title: '', + })) + : null; + if (fallback) { + console.error( + `[docs-search] ranking without ${TITLE_MAP}: ${response.results.length} results, none of them in the map` + ); + } + + const ordered = fallback ?? ranked; + settle({ term: query, ranked: ordered, at: PAGE_SIZE }); return { - // The promoted page is already one of these stubs, so counting them - // is counting the rows the query has in all. - results: rows(hits, query, 0), + results: await draw(ordered, 0, PAGE_SIZE), counts, - total: response.results.length, + total: ordered.length, }; } catch (error) { // The search itself failed, rather than one row of it. Rejecting would @@ -433,18 +451,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Read off before the await: a search landing in the meantime replaces // `page`, and these rows still belong to the query that asked for them. - const { term, promoted } = page; - const slice = page.stubs.slice(page.at, page.at + PAGE_SIZE); - if (slice.length === 0) return []; - + const { ranked } = page; const from = page.at; - page.at += slice.length; + if (from >= ranked.length) return []; + + page.at = Math.min(from + PAGE_SIZE, ranked.length); try { - const hits = (await hydrate(slice)).filter( - (hit) => hit.url !== promoted - ); - return rows(hits, term, from); + return await draw(ranked, from, PAGE_SIZE); } catch (error) { // The rows already on screen are still good, so this fails quietly and // leaves them alone. diff --git a/tests/docs-search.spec.ts b/tests/docs-search.spec.ts index 69ec98d0a5..d4a6c400dd 100644 --- a/tests/docs-search.spec.ts +++ b/tests/docs-search.spec.ts @@ -408,14 +408,11 @@ test('scrolling to the end of the results loads more', async ({ page }) => { expect(new Set(ids).size, 'every option needs its own id').toBe(ids.length); }); -// The two features meeting: a page pulled onto the first screen for naming the -// query still has its own stub further down the list, because ranking past -// PAGE_SIZE is why it had to be pulled forward at all. Paging has to skip it. -// -// `deployment targets` rather than a query with more results: the promotion only -// happens when nothing on the first page already names the query, and -// /docs/infrastructure/deployment-targets/ ranks 36th on raw score. -test('a page pulled forward is not listed again further down', async ({ +// Ranking and paging meeting. The whole result set is ranked before anything is +// drawn, and paging walks that one list, so a page promoted from deep in it — +// /docs/infrastructure/deployment-targets/ ranks 36th for this query on raw +// score — must not come round again when its own position is reached. +test('a page promoted from deep in the list is drawn only once', async ({ page, }) => { await page.goto('/docs');