Speed up search by ranking from a title map

src/integrations/pagefind-index.ts

@@ -10,6 +10,8 @@
 import type { AstroIntegration } from 'astro';
 import { fileURLToPath } from 'node:url';
 import * as path from 'node:path';
+import { gunzipSync } from 'node:zlib';
+import { readdir, readFile, writeFile } from 'node:fs/promises';
 // Statically, because `astro:build:done` fires after Vite's module runner has
 // closed and a dynamic import from inside the hook cannot be resolved.
 import { createIndex, close } from 'pagefind';
@@ -81,10 +83,76 @@ export default function pagefindIndex(): AstroIntegration {
           // Files scanned, not pages indexed: the redirect stubs are counted
           // here and then dropped for having no article.
           logger.info(`scanned ${added.page_count} pages into docs/pagefind`);
+
+          const titles = await writeTitleMap(
+            path.join(distDir, 'docs', 'pagefind')
+          );
+          logger.info(`wrote ${titles} titles to ${TITLE_MAP}`);
         } finally {
           await close();
         }
       },
     },
   };
 }
+
+/** Where the map lands, and the name the client fetches it by. */
+const TITLE_MAP = 'docs-titles.json';
+
+// Pagefind prefixes every decompressed chunk with this before the JSON.
+const FRAGMENT_MAGIC = 'pagefind_dcd';
+
+/**
+ * Writes what the overlay needs to rank a result without fetching it.
+ *
+ * Ranking needs a URL and a title, and Pagefind keeps both in the per-page
+ * fragment — so ranking thirty results meant fetching thirty files, and a
+ * landing page ranked past that could not be reached at all. A search result
+ * stub carries the id of its own fragment, so one map from id to url and title
+ * lets the whole result set be ranked from a single file.
+ *
+ * Read back out of the fragments rather than collected during indexing, because
+ * the ids are assigned by Pagefind as it writes them.
+ */
+async function writeTitleMap(pagefindDir: string): Promise<number> {
+  const dir = path.join(pagefindDir, 'fragment');
+  const files = (await readdir(dir)).filter((name) =>
+    name.endsWith('.pf_fragment')
+  );
+
+  const map: Record<string, [url: string, title: string]> = {};
+
+  for (const file of files) {
+    const raw = gunzipSync(await readFile(path.join(dir, file))).toString(
+      'utf8'
+    );
+
+    // Checked before parsing: a Pagefind release that changes the chunk format
+    // has to fail the build here, rather than write a map the overlay silently
+    // cannot join against.
+    if (!raw.startsWith(FRAGMENT_MAGIC)) {
+      throw new Error(
+        `unexpected fragment format in ${file}: Pagefind's own prefix is missing, so ${TITLE_MAP} cannot be trusted`
+      );
+    }
+
+    const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as {
+      url: string;
+      meta?: Record<string, string>;
+    };
+
+    // The stub's `id` is the filename without its extension, which is the join.
+    map[path.basename(file, '.pf_fragment')] = [
+      fragment.url,
+      fragment.meta?.title ?? '',
+    ];
+  }
+
+  await writeFile(
+    path.join(pagefindDir, TITLE_MAP),
+    JSON.stringify(map),
+    'utf8'
+  );
+
+  return files.length;
+}

src/layouts/Api.astro

@@ -101,7 +101,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter);
               /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */
             }
           </div>
-          <div class="page-content anim-show-parent" {...searchIndex.content}>
+          <div class="page-content anim-show-parent">
             <slot />
             <Authors frontmatter={frontmatter} lang={lang} />
             <Taxonomy frontmatter={frontmatter} lang={lang} />

src/layouts/Default.astro

@@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter);
               lang={lang}
             />
           </div>
-          <div class="page-content anim-show-parent" {...searchIndex.content}>
+          <div class="page-content anim-show-parent">
             <slot />
             <Authors frontmatter={frontmatter} lang={lang} />
             <Taxonomy frontmatter={frontmatter} lang={lang} />

src/lib/searchIndexing.ts

@@ -13,30 +13,12 @@ type ArticleAttributes = {
   'data-pagefind-default-meta'?: string;
 };
 
-/**
- * A second filter, on an element inside the article rather than on the article
- * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a
- * comma-separated pair is taken as a single value.
- */
-type ContentAttributes = {
-  'data-pagefind-filter'?: string;
-};
-
 type IndexAttributes = {
   article: ArticleAttributes;
-  content: ContentAttributes;
 };
 
 /**
- * How shallow a page has to be to count as one a reader might name. Two segments
- * past `/docs/`, which covers `/docs/deployments/` and
- * `/docs/infrastructure/deployment-targets/` but not the pages inside them.
- */
-const LANDING_DEPTH = 3;
-
-/**
- * The `data-pagefind-*` attributes for a page: `article` spreads onto the
- * `<article>`, `content` onto the page content inside it.
+ * The `data-pagefind-*` attributes for a page, to spread onto the `<article>`.
  *
  * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page
  * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that
@@ -53,13 +35,7 @@ export function searchIndexAttributes(
 
   // `all` rather than the default `index`: a bare ignore still lets Pagefind
   // read a title or metadata out of the block.
-  if (!indexable)
-    return { article: { 'data-pagefind-ignore': 'all' }, content: {} };
-
-  // Marks the pages the overlay's second, narrowed search looks through. Only
-  // the shallow pages carry it, so the filter chunk stays small and that search
-  // has a few hundred candidates rather than the whole site.
-  const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH;
+  if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } };
 
   return {
     article: {
@@ -75,6 +51,5 @@ export function searchIndexAttributes(
         ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` }
         : {}),
     },
-    content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {},
   };
 }