diff --git a/src/integrations/pagefind-index.ts b/src/integrations/pagefind-index.ts index a61f83438b..3c574bfdd2 100644 --- a/src/integrations/pagefind-index.ts +++ b/src/integrations/pagefind-index.ts @@ -10,6 +10,8 @@ import type { AstroIntegration } from 'astro'; import { fileURLToPath } from 'node:url'; import * as path from 'node:path'; +import { gunzipSync } from 'node:zlib'; +import { readdir, readFile, writeFile } from 'node:fs/promises'; // Statically, because `astro:build:done` fires after Vite's module runner has // closed and a dynamic import from inside the hook cannot be resolved. import { createIndex, close } from 'pagefind'; @@ -81,6 +83,11 @@ export default function pagefindIndex(): AstroIntegration { // Files scanned, not pages indexed: the redirect stubs are counted // here and then dropped for having no article. logger.info(`scanned ${added.page_count} pages into docs/pagefind`); + + const titles = await writeTitleMap( + path.join(distDir, 'docs', 'pagefind') + ); + logger.info(`wrote ${titles} titles to ${TITLE_MAP}`); } finally { await close(); } @@ -88,3 +95,64 @@ export default function pagefindIndex(): AstroIntegration { }, }; } + +/** Where the map lands, and the name the client fetches it by. */ +const TITLE_MAP = 'docs-titles.json'; + +// Pagefind prefixes every decompressed chunk with this before the JSON. +const FRAGMENT_MAGIC = 'pagefind_dcd'; + +/** + * Writes what the overlay needs to rank a result without fetching it. + * + * Ranking needs a URL and a title, and Pagefind keeps both in the per-page + * fragment — so ranking thirty results meant fetching thirty files, and a + * landing page ranked past that could not be reached at all. A search result + * stub carries the id of its own fragment, so one map from id to url and title + * lets the whole result set be ranked from a single file. + * + * Read back out of the fragments rather than collected during indexing, because + * the ids are assigned by Pagefind as it writes them. + */ +async function writeTitleMap(pagefindDir: string): Promise { + const dir = path.join(pagefindDir, 'fragment'); + const files = (await readdir(dir)).filter((name) => + name.endsWith('.pf_fragment') + ); + + const map: Record = {}; + + for (const file of files) { + const raw = gunzipSync(await readFile(path.join(dir, file))).toString( + 'utf8' + ); + + // Checked before parsing: a Pagefind release that changes the chunk format + // has to fail the build here, rather than write a map the overlay silently + // cannot join against. + if (!raw.startsWith(FRAGMENT_MAGIC)) { + throw new Error( + `unexpected fragment format in ${file}: Pagefind's own prefix is missing, so ${TITLE_MAP} cannot be trusted` + ); + } + + const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as { + url: string; + meta?: Record; + }; + + // The stub's `id` is the filename without its extension, which is the join. + map[path.basename(file, '.pf_fragment')] = [ + fragment.url, + fragment.meta?.title ?? '', + ]; + } + + await writeFile( + path.join(pagefindDir, TITLE_MAP), + JSON.stringify(map), + 'utf8' + ); + + return files.length; +} diff --git a/src/layouts/Api.astro b/src/layouts/Api.astro index b584c954ae..f96c01b8cd 100644 --- a/src/layouts/Api.astro +++ b/src/layouts/Api.astro @@ -101,7 +101,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */ } -
+
diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index a5711494fd..3d4f62a740 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter); lang={lang} />
-
+
diff --git a/src/lib/searchIndexing.ts b/src/lib/searchIndexing.ts index b21aa20b17..ea46993cdb 100644 --- a/src/lib/searchIndexing.ts +++ b/src/lib/searchIndexing.ts @@ -13,30 +13,12 @@ type ArticleAttributes = { 'data-pagefind-default-meta'?: string; }; -/** - * A second filter, on an element inside the article rather than on the article - * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a - * comma-separated pair is taken as a single value. - */ -type ContentAttributes = { - 'data-pagefind-filter'?: string; -}; - type IndexAttributes = { article: ArticleAttributes; - content: ContentAttributes; }; /** - * How shallow a page has to be to count as one a reader might name. Two segments - * past `/docs/`, which covers `/docs/deployments/` and - * `/docs/infrastructure/deployment-targets/` but not the pages inside them. - */ -const LANDING_DEPTH = 3; - -/** - * The `data-pagefind-*` attributes for a page: `article` spreads onto the - * `
`, `content` onto the page content inside it. + * The `data-pagefind-*` attributes for a page, to spread onto the `
`. * * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that @@ -53,13 +35,7 @@ export function searchIndexAttributes( // `all` rather than the default `index`: a bare ignore still lets Pagefind // read a title or metadata out of the block. - if (!indexable) - return { article: { 'data-pagefind-ignore': 'all' }, content: {} }; - - // Marks the pages the overlay's second, narrowed search looks through. Only - // the shallow pages carry it, so the filter chunk stays small and that search - // has a few hundred candidates rather than the whole site. - const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH; + if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } }; return { article: { @@ -75,6 +51,5 @@ export function searchIndexAttributes( ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` } : {}), }, - content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {}, }; } diff --git a/src/scripts/search-engine-pagefind.ts b/src/scripts/search-engine-pagefind.ts index e6b902add9..16a60e5c99 100644 --- a/src/scripts/search-engine-pagefind.ts +++ b/src/scripts/search-engine-pagefind.ts @@ -12,10 +12,13 @@ import { type SearchResult, } from './search-engine'; -// Rows fetched at a time. Pagefind's own UI shows five and offers the rest on -// demand; thirty, because `byNameThenDepth` reorders within a page and needs -// enough of the list to have something to reorder. -const PAGE_SIZE = 30; +// Rows fetched at a time. The panel shows about five, and the rest arrive as it +// is scrolled. Ranking no longer needs them, so this is only a drawing budget: +// ten covers the first screen with room to scroll into. +const PAGE_SIZE = 10; + +/** The map of result id to url and title, written beside the index at build. */ +const TITLE_MAP = 'docs-titles.json'; // Above this share of the corpus, a query is too general to rank rather than // unanswerable, and the overlay says so instead of reporting nothing found. @@ -28,11 +31,6 @@ const PAGE_SIZE = 30; // 79%. A mash landing on the gentler message costs nothing; both offer no rows. const COMMON_TERM_SHARE = 0.8; -// How many shallow pages the named-page lookup looks through. Measured over 18 -// section queries: three finds the page for 16, five for 17, and twenty finds no -// more than five does. -const LANDING_CANDIDATES = 5; - type PagefindSubResult = { title: string; /** Carries the heading's `#anchor` when the match is below the page title. */ @@ -171,8 +169,10 @@ function claimsName(hit: { url: string; title: string }, term: string) { * order for none of them; `data-pagefind-weight` on the h1 measured as no change * at all. * - * Reaches only the results already fetched, which is what `namedPage` below is - * for: a page ranked past `PAGE_SIZE` on raw score cannot be rescued here. + * Runs over every result, because `rank` supplies url and title from the title + * map and nothing here has to be fetched. So a page the query names wins from + * anywhere in the list — `/docs/infrastructure/deployment-targets/` is 36th on + * raw score for "deployment targets" and still comes first. */ function byNameThenDepth< T extends { score: number; url: string; title: string }, @@ -196,19 +196,58 @@ function byNameThenDepth< } /** A stub's score paired with its fetched fragment, which is where the URL is. */ -type Hit = { - fragment: PagefindFragment; +/** Everything ranking needs, and nothing that has to be fetched to get it. */ +type Ranked = { + stub: PagefindResultStub; score: number; url: string; title: string; }; +type TitleMap = Record; + +/** + * Pairs each result with its url and title from the map, so the whole set can be + * ranked before anything is fetched. + * + * A result the map does not know is dropped from ranking. That only happens when + * the map and the index disagree, which means a stale deploy of one of them; the + * caller falls back to ranking what it fetches. + */ +function rank( + stubs: PagefindResultStub[], + titles: TitleMap, + term: string, + prefix: string +): Ranked[] { + const known = stubs.flatMap((stub) => { + const entry = titles[stub.id]; + if (!entry) return []; + // The map holds urls as the index does, relative to the indexed directory. + // Pagefind applies the same prefix to the urls it returns from `data()`. + const [path, title] = entry; + const url = prefix + path; + return [{ stub, score: stub.score, url, title: title || url }]; + }); + + return byNameThenDepth(known, term); +} + /** - * Fetches the fragment for each stub. A fragment that fails takes its own row - * out rather than the whole result set. + * Draws a slice of the ranked list, fetching a fragment for each row in it. The + * fragment supplies the excerpt and the matched headings; the order was settled + * before any of it was asked for. + * + * `from` is how many rows already precede these, which keeps the headings on the + * leading rows of the list rather than the leading rows of every batch. */ -async function hydrate(stubs: PagefindResultStub[]): Promise { - const settled = await Promise.allSettled(stubs.map((stub) => stub.data())); +async function draw( + ranked: Ranked[], + from: number, + count: number +): Promise { + const slice = ranked.slice(from, from + count); + const settled = await Promise.allSettled(slice.map((hit) => hit.stub.data())); return settled.flatMap((outcome, at) => { if (outcome.status === 'rejected') { @@ -219,77 +258,38 @@ async function hydrate(stubs: PagefindResultStub[]): Promise { return []; } + const hit = slice[at]; const fragment = outcome.value; + // The fragment is the fallback for both: without the title map, `rank` has + // no url or title to give and the fragment is the only source. const { pathname } = new URL(fragment.url, window.location.origin); return [ { - fragment, - score: stubs[at].score, - url: pathname, - title: fragment.meta?.title ?? pathname, + url: hit.url || pathname, + title: hit.title || fragment.meta?.title || pathname, + // Already carries around the hits, and Pagefind escapes the + // surrounding text itself. + excerpt: fragment.excerpt, + breadcrumb: breadcrumbFrom(hit.url || pathname), + sections: + from + at < ROWS_WITH_SECTIONS ? sectionsOf(fragment) : undefined, + ...classify(hit.url || pathname), }, ]; }); } -/** - * One page of rows, ordered within itself. `from` is how many rows already - * precede them, which is what keeps the headings on the leading pages of the - * list rather than on the leading rows of every page. - */ -function rows(hits: Hit[], term: string, from: number): SearchResult[] { - return byNameThenDepth(hits, term).map((hit, rank) => ({ - url: hit.url, - title: hit.title, - // Already carries around the hits, and Pagefind escapes the - // surrounding text itself. - excerpt: hit.fragment.excerpt, - breadcrumb: breadcrumbFrom(hit.url), - sections: - from + rank < ROWS_WITH_SECTIONS ? sectionsOf(hit.fragment) : undefined, - ...classify(hit.url), - })); -} - -/** - * The page the query names, when the first page of results missed it. - * - * `byNameThenDepth` can only promote what has been fetched, and a section's own - * page can rank far below the pages inside it: `/docs/infrastructure/ - * deployment-targets/` is 36th for "deployment targets", six places past the - * page size. These stubs come from a search narrowed to the shallow pages alone, - * where the page a query names sits near the top of a few hundred candidates. - * - * Only called when nothing already fetched names the query, so the extra - * fragments are paid for by the queries that need them and no others. - */ -async function namedPage( - stubs: PagefindResultStub[], - term: string, - already: Hit[] -): Promise { - const seen = new Set(already.map((hit) => hit.url)); - const candidates = await hydrate(stubs.slice(0, LANDING_CANDIDATES)); - - return ( - candidates.find((hit) => claimsName(hit, term) && !seen.has(hit.url)) ?? - null - ); -} - export function pagefindEngine(bundlePath: string): SearchEngine { let loading: Promise | null = null; - // Everything the last search matched, and how much of it has been handed over. - // A stub is a score and a promise of its fragment, so holding a thousand of - // them costs nothing and saves searching again to show row thirty-one. - // `promoted` is the page `namedPage` pulled forward, whose own stub is still - // waiting further down `stubs`. - let page: { - term: string; - stubs: PagefindResultStub[]; - at: number; - promoted: string | null; - } | null = null; + // The last search's whole result set, already ranked, and how much of it has + // been drawn. Ranking the tail costs nothing because it needs no fetches, so + // `more()` only has to draw the next slice. + let page: { term: string; ranked: Ranked[]; at: number } | null = null; + // The map from result id to url and title, fetched once with the index. + let titles: TitleMap | null = null; + // What Pagefind prepends to the urls it returns, and therefore what the map's + // own relative urls need. `/docs/pagefind/` leaves `/docs`. + const urlPrefix = bundlePath.replace(/\/?pagefind\/?$/, ''); // Which search owns `page`. Searches run concurrently and can settle out of // order, and an overtaken one must not leave its stubs behind for `more()`. let searches = 0; @@ -311,7 +311,20 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // The filter index is a separate chunk, and a search returns empty filter // counts until it has been pulled down. Its section totals also add up to // the size of the corpus, which is what `COMMON_TERM_SHARE` is a share of. - const filters = await api.filters(); + const [filters] = await Promise.all([ + api.filters(), + // Ranking reads url and title out of this, so it has to be here before + // the first search returns. A failure leaves it null and ranking falls + // back to ordering the rows it draws. + fetch(`${bundlePath}${TITLE_MAP}`) + .then((response) => (response.ok ? response.json() : null)) + .then((map: TitleMap | null) => { + titles = map; + }) + .catch((error) => { + console.error(`[docs-search] could not load ${TITLE_MAP}`, error); + }), + ]); corpus = Object.values(filters.section ?? {}).reduce( (total, count) => total + count, 0 @@ -358,16 +371,13 @@ export function pagefindEngine(bundlePath: string): SearchEngine { const filters = facet && facet !== 'all' ? { section: [facet] } : undefined; - // Three searches at once, because a second await here would sit in front - // of every fragment fetch below it. The first supplies the rows; the - // second says whether the query has any answer at all, and only runs - // while a tab is narrowing the first; the third is the shallow-page - // shortlist `namedPage` draws on, which costs nothing until its - // fragments are fetched. - const [response, wholeCorpus, landing] = await Promise.all([ + // Together, because a second await here would sit in front of every + // fragment fetch below it. The first supplies the rows; the second says + // whether the query has any answer at all, and only runs while a tab is + // narrowing the first. + const [response, wholeCorpus] = await Promise.all([ api.search(query, { filters }), filters ? api.search(query) : null, - api.search(query, { filters: { ...filters, landing: ['true'] } }), ]); const unfiltered = wholeCorpus ?? response; @@ -395,29 +405,37 @@ export function pagefindEngine(bundlePath: string): SearchEngine { : empty; } - const hits = await hydrate(response.results.slice(0, PAGE_SIZE)); - - const named = hits.some((hit) => claimsName(hit, query)) - ? null - : await namedPage(landing.results, query, hits); - if (named) hits.push(named); - - // A promoted page was fetched precisely because its own stub ranks past - // `PAGE_SIZE`, so that stub is still ahead of `more()` and has to be - // skipped there rather than drawn a second time. - settle({ - term: query, - stubs: response.results, - at: PAGE_SIZE, - promoted: named?.url ?? null, - }); + // Every match is ranked here, whether it will be drawn or not. A page + // the query names wins from anywhere in the list, which is what the + // shallow-page search used to be for. + const ranked = titles + ? rank(response.results, titles, query, urlPrefix) + : []; + + // The map was missing or disagreed with the index. Ranking what gets + // drawn is worse than ranking everything, and it still answers. + const fallback = + ranked.length === 0 && response.results.length > 0 + ? response.results.map((stub) => ({ + stub, + score: stub.score, + url: '', + title: '', + })) + : null; + if (fallback) { + console.error( + `[docs-search] ranking without ${TITLE_MAP}: ${response.results.length} results, none of them in the map` + ); + } + + const ordered = fallback ?? ranked; + settle({ term: query, ranked: ordered, at: PAGE_SIZE }); return { - // The promoted page is already one of these stubs, so counting them - // is counting the rows the query has in all. - results: rows(hits, query, 0), + results: await draw(ordered, 0, PAGE_SIZE), counts, - total: response.results.length, + total: ordered.length, }; } catch (error) { // The search itself failed, rather than one row of it. Rejecting would @@ -433,18 +451,14 @@ export function pagefindEngine(bundlePath: string): SearchEngine { // Read off before the await: a search landing in the meantime replaces // `page`, and these rows still belong to the query that asked for them. - const { term, promoted } = page; - const slice = page.stubs.slice(page.at, page.at + PAGE_SIZE); - if (slice.length === 0) return []; - + const { ranked } = page; const from = page.at; - page.at += slice.length; + if (from >= ranked.length) return []; + + page.at = Math.min(from + PAGE_SIZE, ranked.length); try { - const hits = (await hydrate(slice)).filter( - (hit) => hit.url !== promoted - ); - return rows(hits, term, from); + return await draw(ranked, from, PAGE_SIZE); } catch (error) { // The rows already on screen are still good, so this fails quietly and // leaves them alone. diff --git a/tests/docs-search.spec.ts b/tests/docs-search.spec.ts index 69ec98d0a5..d4a6c400dd 100644 --- a/tests/docs-search.spec.ts +++ b/tests/docs-search.spec.ts @@ -408,14 +408,11 @@ test('scrolling to the end of the results loads more', async ({ page }) => { expect(new Set(ids).size, 'every option needs its own id').toBe(ids.length); }); -// The two features meeting: a page pulled onto the first screen for naming the -// query still has its own stub further down the list, because ranking past -// PAGE_SIZE is why it had to be pulled forward at all. Paging has to skip it. -// -// `deployment targets` rather than a query with more results: the promotion only -// happens when nothing on the first page already names the query, and -// /docs/infrastructure/deployment-targets/ ranks 36th on raw score. -test('a page pulled forward is not listed again further down', async ({ +// Ranking and paging meeting. The whole result set is ranked before anything is +// drawn, and paging walks that one list, so a page promoted from deep in it — +// /docs/infrastructure/deployment-targets/ ranks 36th for this query on raw +// score — must not come round again when its own position is reached. +test('a page promoted from deep in the list is drawn only once', async ({ page, }) => { await page.goto('/docs');