From b33fb975464641ae4b7dd85d758e330731155f2c Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Mon, 24 Aug 2026 15:02:17 -0400 Subject: [PATCH 1/2] chore: left-shift insight filtering and pagination into EvalClient --- src/components/BatchEvaluationPicker.tsx | 20 +-- src/components/BatchInsightsPicker.tsx | 79 ++++++++ src/core/batchInsightsPagination.test.tsx | 168 ++++++++++++++++++ src/core/eval.tsx | 59 ++++++ .../batch-insights.screen.test.tsx | 27 +-- .../batch-insights/batch-insights.test.tsx | 13 +- .../eval/batch-insights/list/index.tsx | 13 +- .../eval/batch-insights/list/screen.tsx | 10 +- src/handlers/eval/types.tsx | 5 + src/testing/TestCoreClient.tsx | 25 +++ 10 files changed, 359 insertions(+), 60 deletions(-) create mode 100644 src/components/BatchInsightsPicker.tsx create mode 100644 src/core/batchInsightsPagination.test.tsx diff --git a/src/components/BatchEvaluationPicker.tsx b/src/components/BatchEvaluationPicker.tsx index b19b20200..ad6e45fa4 100644 --- a/src/components/BatchEvaluationPicker.tsx +++ b/src/components/BatchEvaluationPicker.tsx @@ -41,11 +41,6 @@ function toRow(summary: BatchEvaluationSummary): BatchEvaluationRow { export interface BatchEvaluationPickerProps extends ScreenProps { breadcrumb: string[]; description?: string; - queryKeyPrefix?: string; - include?: (summary: BatchEvaluationSummary) => boolean; - loadingMessage?: string; - emptyMessage?: string; - emptyPageMessage?: string; onSelect: (batchEvaluationId: string) => void; onEscape?: () => void; } @@ -60,11 +55,6 @@ export function BatchEvaluationPicker({ core, breadcrumb, description, - queryKeyPrefix = "batch-evaluations", - include, - loadingMessage = "Loading batch evaluations…", - emptyMessage = "No batch evaluations found in this Region.", - emptyPageMessage = "No batch evaluations on this page.", onSelect, onEscape, }: BatchEvaluationPickerProps) { @@ -76,11 +66,11 @@ export function BatchEvaluationPicker({ { const response = await core.eval.listBatchEvaluations(token, pageSize, opts); return { - items: (response.batchEvaluations ?? []).filter((summary) => include?.(summary) ?? true), + items: response.batchEvaluations ?? [], nextToken: response.nextToken, }; }} @@ -89,10 +79,10 @@ export function BatchEvaluationPicker({ getValue={(row) => row.batchEvaluationId} onSelect={onSelect} onBack={goBack} - loadingMessage={loadingMessage} + loadingMessage="Loading batch evaluations…" errorMessage={(error) => `Error: ${error.message}`} - emptyMessage={emptyMessage} - emptyPageMessage={emptyPageMessage} + emptyMessage="No batch evaluations found in this Region." + emptyPageMessage="No batch evaluations on this page." /> ); } diff --git a/src/components/BatchInsightsPicker.tsx b/src/components/BatchInsightsPicker.tsx new file mode 100644 index 000000000..99e32a67c --- /dev/null +++ b/src/components/BatchInsightsPicker.tsx @@ -0,0 +1,79 @@ +import type { BatchEvaluationSummary } from "@aws-sdk/client-bedrock-agentcore"; +import { useNavigate } from "react-router"; +import type { ScreenProps } from "../handlers/types"; +import { coreOptsFromCtx } from "../handlers/utils"; +import { formatTimestamp } from "./formatTimestamp"; +import { PaginatedTablePicker } from "./PaginatedTablePicker"; +import type { DataTableColumn } from "./ui/data-table"; + +interface BatchInsightsRow extends Record { + batchEvaluationId: string; + name: string; + status: string; + updatedAt: string; +} + +const batchInsightsColumns = [ + { key: "name", header: "name", flex: true }, + { key: "status", header: "status", width: 22 }, + { + key: "updatedAt", + header: "updated UTC", + width: 16, + render: formatTimestamp, + }, +] satisfies DataTableColumn[]; + +function toRow(summary: BatchEvaluationSummary): BatchInsightsRow { + const id = summary.batchEvaluationId ?? ""; + return { + batchEvaluationId: id, + name: summary.batchEvaluationName ?? id, + status: summary.status ?? "-", + updatedAt: summary.updatedAt?.toISOString() ?? "-", + }; +} + +export interface BatchInsightsPickerProps extends ScreenProps { + breadcrumb: string[]; + description?: string; + onSelect: (batchEvaluationId: string) => void; + onEscape?: () => void; +} + +export function BatchInsightsPicker({ + ctx, + core, + breadcrumb, + description, + onSelect, + onEscape, +}: BatchInsightsPickerProps) { + const opts = coreOptsFromCtx(ctx); + const navigate = useNavigate(); + const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); + + return ( + { + const response = await core.eval.listBatchInsights(token, pageSize, opts); + return { + items: response.batchEvaluations ?? [], + nextToken: response.nextToken, + }; + }} + toRow={toRow} + columns={batchInsightsColumns} + getValue={(row) => row.batchEvaluationId} + onSelect={onSelect} + onBack={goBack} + loadingMessage="Loading batch insights…" + errorMessage={(error) => `Error: ${error.message}`} + emptyMessage="No batch insights found in this Region." + emptyPageMessage="No batch insights on this page." + /> + ); +} diff --git a/src/core/batchInsightsPagination.test.tsx b/src/core/batchInsightsPagination.test.tsx new file mode 100644 index 000000000..9a673ae32 --- /dev/null +++ b/src/core/batchInsightsPagination.test.tsx @@ -0,0 +1,168 @@ +import { describe, expect, mock, test } from "bun:test"; +import { + ListBatchEvaluationsCommand, + type BatchEvaluationSummary, +} from "@aws-sdk/client-bedrock-agentcore"; +import { ResultTruncationError } from "../errors"; +import type { AwsClients } from "./types"; +import { EvalClient } from "./eval"; + +const options = { region: "us-west-2", endpointUrl: "https://agentcore.example.test" }; + +function insight(id: string): BatchEvaluationSummary { + return { + batchEvaluationId: id, + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + } as BatchEvaluationSummary; +} + +function evaluation(id: string): BatchEvaluationSummary { + return { + batchEvaluationId: id, + evaluators: [{ evaluatorId: "Builtin.Correctness" }], + } as BatchEvaluationSummary; +} + +function evalClient(send: (command: ListBatchEvaluationsCommand) => Promise): EvalClient { + return new EvalClient({ + data: () => ({ send: mock(send) }) as never, + } as unknown as AwsClients); +} + +describe("EvalClient.listBatchInsights", () => { + test("rejects an invalid logical page size before calling the service", async () => { + const send = mock(async () => ({ batchEvaluations: [] })); + const client = evalClient(send); + + await expect(client.listBatchInsights(undefined, 0, options)).rejects.toThrow( + "maxResults must be a positive integer", + ); + expect(send).not.toHaveBeenCalled(); + }); + + test("filters the final Batch Evaluation page", async () => { + const insightsJob = insight("insights-1"); + const client = evalClient(async (command) => { + expect(command).toBeInstanceOf(ListBatchEvaluationsCommand); + expect(command.input).toEqual({ nextToken: "page-2", maxResults: undefined }); + return { batchEvaluations: [evaluation("evaluation-1"), insightsJob] }; + }); + + await expect(client.listBatchInsights("page-2", 10, options)).resolves.toEqual({ + batchEvaluations: [insightsJob], + nextToken: undefined, + }); + }); + + test("scans sparse service pages to fill one logical Insights page", async () => { + const insights = [insight("insights-1"), insight("insights-2")]; + const requests: unknown[] = []; + const client = evalClient(async (command) => { + requests.push(command.input); + switch (command.input.nextToken) { + case undefined: + return { + batchEvaluations: [evaluation("evaluation-1")], + nextToken: "page-2", + }; + case "page-2": + return { + batchEvaluations: [insights[0], evaluation("evaluation-2")], + nextToken: "page-3", + }; + default: + return { + batchEvaluations: [evaluation("evaluation-3"), insights[1]], + }; + } + }); + + await expect(client.listBatchInsights(undefined, 2, options)).resolves.toEqual({ + batchEvaluations: insights, + nextToken: undefined, + }); + expect(requests).toEqual([ + { nextToken: undefined, maxResults: undefined }, + { nextToken: "page-2", maxResults: undefined }, + { nextToken: "page-3", maxResults: undefined }, + ]); + }); + + test("replays the exact Batch Evaluation prefix without skipping the next Insight", async () => { + const firstInsight = insight("insights-1"); + const secondInsight = insight("insights-2"); + const requests: unknown[] = []; + const client = evalClient(async (command) => { + requests.push(command.input); + + if (command.input.nextToken === "after-insights-1") { + return { + batchEvaluations: [evaluation("evaluation-2"), secondInsight], + }; + } + if (command.input.maxResults === 2) { + return { + batchEvaluations: [evaluation("evaluation-1"), firstInsight], + nextToken: "after-insights-1", + }; + } + return { + batchEvaluations: [ + evaluation("evaluation-1"), + firstInsight, + evaluation("evaluation-2"), + secondInsight, + ], + }; + }); + + const first = await client.listBatchInsights(undefined, 1, options); + const second = await client.listBatchInsights(first.nextToken, 1, options); + + expect(first).toEqual({ + batchEvaluations: [firstInsight], + nextToken: "after-insights-1", + }); + expect(second).toEqual({ + batchEvaluations: [secondInsight], + nextToken: undefined, + }); + expect(requests).toEqual([ + { nextToken: undefined, maxResults: undefined }, + { nextToken: undefined, maxResults: 2 }, + { nextToken: "after-insights-1", maxResults: undefined }, + ]); + }); + + test("returns a token only when it leads to another Insights job", async () => { + const firstInsight = insight("insights-1"); + const secondInsight = insight("insights-2"); + const client = evalClient(async (command) => { + if (command.input.nextToken === "page-2") { + return { batchEvaluations: [evaluation("evaluation-2"), secondInsight] }; + } + return { + batchEvaluations: [firstInsight, evaluation("evaluation-1")], + nextToken: "page-2", + }; + }); + + await expect(client.listBatchInsights(undefined, 1, options)).resolves.toEqual({ + batchEvaluations: [firstInsight], + nextToken: "page-2", + }); + }); + + test("throws when Insights discovery exceeds the Batch Evaluation scan cap", async () => { + let calls = 0; + const client = evalClient(async () => { + calls += 1; + return { batchEvaluations: [], nextToken: `page-${calls}` }; + }); + + await expect(client.listBatchInsights(undefined, 1, options)).rejects.toThrow( + ResultTruncationError, + ); + expect(calls).toBe(101); + }); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 3919d4d36..ebdb170d3 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -65,6 +65,7 @@ import { type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, + type BatchEvaluationSummary, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, @@ -92,6 +93,7 @@ import { InputValidationError, NetworkingError, ResourceNotFoundError, + ResultTruncationError, } from "../errors"; import type { BatchEvaluationDetail, @@ -165,6 +167,9 @@ const EVALUATE_TARGET_BATCH = 10; // CloudWatch Logs Insights hard ceiling: a query returns at most 100k rows. const INSIGHTS_MAX_ROWS = 100_000; +const DEFAULT_BATCH_INSIGHTS_PAGE_SIZE = 50; +const MAX_BATCH_INSIGHTS_SCAN_REQUESTS = 101; + // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can // omit it. Production (src/core/index.tsx) injects a real child logger. @@ -372,6 +377,56 @@ export class EvalClient implements CoreEvalClient { .send(new ListBatchEvaluationsCommand({ nextToken, maxResults })); } + async listBatchInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + const insightsPageSize = maxResults ?? DEFAULT_BATCH_INSIGHTS_PAGE_SIZE; + if (!Number.isInteger(insightsPageSize) || insightsPageSize < 1) { + throw new InputValidationError("maxResults must be a positive integer"); + } + const batchEvaluations: BatchEvaluationSummary[] = []; + let batchEvaluationToken = nextToken; + + for (let request = 0; request < MAX_BATCH_INSIGHTS_SCAN_REQUESTS; request++) { + const requestToken = batchEvaluationToken; + const response = await this.listBatchEvaluations(requestToken, undefined, options); + const serviceItems = response.batchEvaluations ?? []; + const insights = serviceItems.filter(EvalClient.isBatchInsights); + + if (batchEvaluations.length < insightsPageSize) { + const remaining = insightsPageSize - batchEvaluations.length; + if (insights.length > remaining) { + const boundaryInsight = insights[remaining - 1]!; + const boundarySize = serviceItems.indexOf(boundaryInsight) + 1; + // Re-read through the last returned Insights job so the service token + // cannot skip later matches from this Batch Evaluation page. + const boundaryResponse = await this.listBatchEvaluations( + requestToken, + boundarySize, + options, + ); + + batchEvaluations.push(...insights.slice(0, remaining)); + return { ...boundaryResponse, batchEvaluations }; + } + batchEvaluations.push(...insights); + } else if (insights.length > 0) { + return { ...response, batchEvaluations, nextToken: requestToken }; + } + + if (response.nextToken === undefined) { + return { ...response, batchEvaluations, nextToken: undefined }; + } + batchEvaluationToken = response.nextToken; + } + + throw new ResultTruncationError( + `Batch Insights discovery exceeded ${MAX_BATCH_INSIGHTS_SCAN_REQUESTS} Batch Evaluation scan requests; results are incomplete`, + ); + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions, @@ -444,6 +499,10 @@ export class EvalClient implements CoreEvalClient { }; } + private static isBatchInsights(job: BatchEvaluationSummary): boolean { + return Boolean(job.insights?.length); + } + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; diff --git a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx index 3e8998b0a..dbb067b8a 100644 --- a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx @@ -46,7 +46,7 @@ function detail(overrides: Partial = {}): GetBatchEv function coreWithBatchEvaluations(items: BatchEvaluationSummary[]): TestCoreClient { const core = new TestCoreClient(); - core.eval.setBatchEvalListResponse({ batchEvaluations: items }); + core.eval.setBatchInsightsListResponse({ batchEvaluations: items }); return core; } @@ -63,22 +63,12 @@ describe("batch-insights menu", () => { }); describe("batch-insights picker", () => { - test("filters evaluator-only jobs from the shared service page", async () => { - const core = coreWithBatchEvaluations([ - summary(), - summary({ - batchEvaluationId: "evaluation-1", - batchEvaluationName: "quality_evaluation", - insights: undefined, - evaluators: [{ evaluatorId: "Builtin.Correctness" }], - }), - ]); + test("loads Insights from the dedicated Core facade", async () => { + const core = coreWithBatchEvaluations([summary()]); const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); await waitForText(screen.lastFrame, "failure_analysis"); - const frame = screen.lastFrame()!; - expect(frame).not.toContain("quality_evaluation"); - expect(core.eval.calls[0]?.method).toBe("listBatchEvaluations"); + expect(core.eval.calls[0]?.method).toBe("listBatchInsights"); }); test("bare get redirects to the filtered picker", async () => { @@ -86,7 +76,7 @@ describe("batch-insights picker", () => { const screen = renderScreen("/agentcore/eval/batch-insights/get", { core }); await waitForText(screen.lastFrame, "failure_analysis"); - expect(core.eval.calls[0]?.method).toBe("listBatchEvaluations"); + expect(core.eval.calls[0]?.method).toBe("listBatchInsights"); }); test("selection opens the matching insights JSON", async () => { @@ -105,12 +95,7 @@ describe("batch-insights picker", () => { }); test("shows the Insights-specific empty state", async () => { - const core = coreWithBatchEvaluations([ - summary({ - insights: undefined, - evaluators: [{ evaluatorId: "Builtin.Correctness" }], - }), - ]); + const core = coreWithBatchEvaluations([]); const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); await waitForText(screen.lastFrame, "No batch insights found in this Region."); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index c1df87306..538f9f72e 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -190,24 +190,20 @@ describe("eval batch-insights get", () => { }); describe("eval batch-insights list", () => { - test("filters mixed service results and preserves pagination", async () => { + test("returns the logical Insights page from Core", async () => { const response = { batchEvaluations: [ { batchEvaluationId: "bi-1", insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], }, - { - batchEvaluationId: "be-1", - evaluators: [{ evaluatorId: "Builtin.Helpfulness" }], - }, { batchEvaluationId: "bi-2", insights: [{ insightId: "Builtin.Insight.UserIntent" }] }, ], nextToken: "next-page", } as unknown as ListBatchEvaluationsResponse; const { core, stdout } = await run( ["eval", "batch-insights", "list", "--next-token", "page-1", "--max-results", "20", "--json"], - (client) => client.eval.setBatchEvalListResponse(response, "page-1"), + (client) => client.eval.setBatchInsightsListResponse(response, "page-1"), ); const output = JSON.parse(stdout); @@ -216,6 +212,9 @@ describe("eval batch-insights list", () => { output.batchEvaluations.map((item: { batchEvaluationId: string }) => item.batchEvaluationId), ).toEqual(["bi-1", "bi-2"]); expect(output.batchEvaluations[0].consoleUrl).toBeUndefined(); - expect(core.eval.calls[0]?.args).toEqual(["page-1", 20, { region: REGION }]); + expect(core.eval.calls[0]).toEqual({ + method: "listBatchInsights", + args: ["page-1", 20, { region: REGION }], + }); }); }); diff --git a/src/handlers/eval/batch-insights/list/index.tsx b/src/handlers/eval/batch-insights/list/index.tsx index 6490e8f28..37db28c12 100644 --- a/src/handlers/eval/batch-insights/list/index.tsx +++ b/src/handlers/eval/batch-insights/list/index.tsx @@ -3,7 +3,6 @@ import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -import { InsightsJob } from "../insightsJob"; export const createListBatchInsightsHandler = (core: Core) => createHandler({ @@ -11,20 +10,16 @@ export const createListBatchInsightsHandler = (core: Core) => description: "list batch insights runs", flags: [ flag("next-token", "pagination token returned by a previous request", z.string().optional()), - flag("max-results", "maximum number of service items to inspect", z.number().optional()), + flag("max-results", "maximum number of batch insights runs to return", z.number().optional()), ], handle: async (ctx, flags) => { - const opts = coreOptsFromCtx(ctx); - const response = await core.eval.listBatchEvaluations( + const response = await core.eval.listBatchInsights( flags["next-token"], flags["max-results"], - opts, + coreOptsFromCtx(ctx), ); - ctx.require(JsonRendererKey).renderJson({ - ...response, - batchEvaluations: (response.batchEvaluations ?? []).filter(InsightsJob.is), - }); + ctx.require(JsonRendererKey).renderJson(response); }, }); diff --git a/src/handlers/eval/batch-insights/list/screen.tsx b/src/handlers/eval/batch-insights/list/screen.tsx index 50ac2065d..83f823393 100644 --- a/src/handlers/eval/batch-insights/list/screen.tsx +++ b/src/handlers/eval/batch-insights/list/screen.tsx @@ -1,20 +1,14 @@ import { useNavigate } from "react-router"; -import { BatchEvaluationPicker } from "../../../../components/BatchEvaluationPicker"; +import { BatchInsightsPicker } from "../../../../components/BatchInsightsPicker"; import type { ScreenProps } from "../../../types"; -import { InsightsJob } from "../insightsJob"; export function BatchInsightsListScreen(props: ScreenProps) { const navigate = useNavigate(); return ( - navigate(`/agentcore/eval/batch-insights/get/${encodeURIComponent(batchEvaluationId)}`) } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index d09eddd5f..8d2380e13 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -338,6 +338,11 @@ export interface CoreEvalClient { maxResults: number | undefined, options: CoreOptions, ): Promise; + listBatchInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 606e1d428..5a0784f36 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1401,6 +1401,7 @@ export class TestEvalClient implements CoreEvalClient { // simulates a CloudWatch read failure surfaced as `resultsError`. private batchEvalGetResponse: GetBatchEvaluationResponse = DEFAULT_GET_BATCH_EVAL_RESPONSE; private batchEvalListResponses = new Map(); + private batchInsightsListResponses = new Map(); private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; @@ -1578,6 +1579,16 @@ export class TestEvalClient implements CoreEvalClient { return this; } + // setBatchInsightsListResponse sets what listBatchInsights resolves to (when + // not erroring). Pass `forNextToken` to serve a later logical Insights page. + setBatchInsightsListResponse( + response: ListBatchEvaluationsResponse, + forNextToken?: string, + ): this { + this.batchInsightsListResponses.set(forNextToken, response); + return this; + } + // setBatchEvalResults sets the per-session results getBatchEvaluation merges in // (when results are requested, the job is terminal, and it has a CloudWatch // output config). @@ -1700,6 +1711,20 @@ export class TestEvalClient implements CoreEvalClient { ); } + async listBatchInsights( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listBatchInsights", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + const response = + this.batchInsightsListResponses.get(nextToken) ?? + this.batchInsightsListResponses.get(undefined) ?? + DEFAULT_LIST_BATCH_EVALS_RESPONSE; + return { ...response, batchEvaluations: response.batchEvaluations ?? [] }; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions, From 072c174ebb1f4cfd52b77a04b8ee0a832b5c2335 Mon Sep 17 00:00:00 2001 From: Nicolas Borges Date: Tue, 25 Aug 2026 10:54:34 -0400 Subject: [PATCH 2/2] chore: add dedicated getBatchInsights API --- ...nation.test.tsx => batchInsights.test.tsx} | 42 +++++++++++++++++++ src/core/eval.tsx | 10 ++++- .../batch-insights.screen.test.tsx | 8 ++-- .../batch-insights/batch-insights.test.tsx | 5 ++- .../eval/batch-insights/get/index.tsx | 7 +--- .../eval/batch-insights/get/screen.tsx | 9 +--- .../eval/batch-insights/insightsJob.ts | 13 ------ src/handlers/eval/types.tsx | 1 + src/testing/TestCoreClient.tsx | 12 ++++++ 9 files changed, 74 insertions(+), 33 deletions(-) rename src/core/{batchInsightsPagination.test.tsx => batchInsights.test.tsx} (78%) delete mode 100644 src/handlers/eval/batch-insights/insightsJob.ts diff --git a/src/core/batchInsightsPagination.test.tsx b/src/core/batchInsights.test.tsx similarity index 78% rename from src/core/batchInsightsPagination.test.tsx rename to src/core/batchInsights.test.tsx index 9a673ae32..af240200c 100644 --- a/src/core/batchInsightsPagination.test.tsx +++ b/src/core/batchInsights.test.tsx @@ -1,5 +1,6 @@ import { describe, expect, mock, test } from "bun:test"; import { + GetBatchEvaluationCommand, ListBatchEvaluationsCommand, type BatchEvaluationSummary, } from "@aws-sdk/client-bedrock-agentcore"; @@ -29,6 +30,47 @@ function evalClient(send: (command: ListBatchEvaluationsCommand) => Promise Promise): EvalClient { + return new EvalClient({ + data: () => ({ send: mock(send) }) as never, + } as unknown as AwsClients); +} + +describe("EvalClient.getBatchInsights", () => { + test("gets an Insights job without reading CloudWatch evaluation results", async () => { + const job = { + batchEvaluationId: "insights-1", + batchEvaluationArn: + "arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/insights-1", + batchEvaluationName: "insights-1", + status: "COMPLETED" as const, + createdAt: new Date("2026-08-25T12:00:00.000Z"), + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + outputConfig: { + cloudWatchConfig: { + logGroupName: "/aws/example", + logStreamName: "results", + }, + }, + }; + const client = getEvalClient(async (command) => { + expect(command).toBeInstanceOf(GetBatchEvaluationCommand); + expect(command.input).toEqual({ batchEvaluationId: "insights-1" }); + return job; + }); + + await expect(client.getBatchInsights("insights-1", options)).resolves.toEqual(job); + }); + + test("rejects an evaluator-only Batch Evaluation", async () => { + const client = getEvalClient(async () => evaluation("evaluation-1")); + + await expect(client.getBatchInsights("evaluation-1", options)).rejects.toThrow( + 'batch evaluation "evaluation-1" is not a batch insights run', + ); + }); +}); + describe("EvalClient.listBatchInsights", () => { test("rejects an invalid logical page size before calling the service", async () => { const send = mock(async () => ({ batchEvaluations: [] })); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index ebdb170d3..849054732 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -367,6 +367,14 @@ export class EvalClient implements CoreEvalClient { } } + async getBatchInsights(id: string, options: CoreOptions): Promise { + const { detail } = await this.getBatchEvaluation(id, options, { includeResults: false }); + if (!EvalClient.isBatchInsights(detail)) { + throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); + } + return detail; + } + async listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, @@ -499,7 +507,7 @@ export class EvalClient implements CoreEvalClient { }; } - private static isBatchInsights(job: BatchEvaluationSummary): boolean { + private static isBatchInsights(job: { insights?: unknown[] }): boolean { return Boolean(job.insights?.length); } diff --git a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx index dbb067b8a..32df004d2 100644 --- a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx @@ -89,7 +89,7 @@ describe("batch-insights picker", () => { await waitForText(screen.lastFrame, "agentcore → eval → batch-insights → get → insights-1"); await waitFor(() => core.eval.calls.some( - (call) => call.method === "getBatchEvaluation" && call.args[0] === "insights-1", + (call) => call.method === "getBatchInsights" && call.args[0] === "insights-1", ), ); }); @@ -111,9 +111,9 @@ describe("batch-insights detail", () => { await waitForText(screen.lastFrame, "failure_analysis"); const frame = screen.lastFrame()!; expect(frame).toContain('"failureAnalysisResult"'); - expect(core.eval.calls.find((call) => call.method === "getBatchEvaluation")).toEqual({ - method: "getBatchEvaluation", - args: ["insights-1", { region: "us-east-1" }, { includeResults: false }], + expect(core.eval.calls.find((call) => call.method === "getBatchInsights")).toEqual({ + method: "getBatchInsights", + args: ["insights-1", { region: "us-east-1" }], }); }); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index 538f9f72e..5a0837ad1 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -172,7 +172,10 @@ describe("eval batch-insights get", () => { failureAnalysisResult: { failures: [] }, }); expect(JSON.parse(stdout).consoleUrl).toBeUndefined(); - expect(core.eval.calls[0]?.args[2]).toEqual({ includeResults: false }); + expect(core.eval.calls[0]).toEqual({ + method: "getBatchInsights", + args: ["bi-1", { region: REGION }], + }); }); test("rejects an evaluator-only batch evaluation", async () => { diff --git a/src/handlers/eval/batch-insights/get/index.tsx b/src/handlers/eval/batch-insights/get/index.tsx index 348ee3c76..d5cb45657 100644 --- a/src/handlers/eval/batch-insights/get/index.tsx +++ b/src/handlers/eval/batch-insights/get/index.tsx @@ -4,7 +4,6 @@ import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -import { InsightsJob } from "../insightsJob"; export const createGetBatchInsightsHandler = (core: Core) => createHandler({ @@ -15,11 +14,7 @@ export const createGetBatchInsightsHandler = (core: Core) => const id = flags["id"]; if (!id) throw new InputValidationError("required option '--id ' not specified"); - const opts = coreOptsFromCtx(ctx); - const { detail } = await core.eval.getBatchEvaluation(id, opts, { - includeResults: false, - }); - InsightsJob.assert(detail, id); + const detail = await core.eval.getBatchInsights(id, coreOptsFromCtx(ctx)); ctx.require(JsonRendererKey).renderJson(detail); }, diff --git a/src/handlers/eval/batch-insights/get/screen.tsx b/src/handlers/eval/batch-insights/get/screen.tsx index 45f42f669..cb0836953 100644 --- a/src/handlers/eval/batch-insights/get/screen.tsx +++ b/src/handlers/eval/batch-insights/get/screen.tsx @@ -3,19 +3,12 @@ import { useParams } from "react-router"; import { JsonDetail } from "../../../../components/JsonDetail"; import type { ScreenProps } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -import { InsightsJob } from "../insightsJob"; function useBatchInsightsDetail({ ctx, core }: ScreenProps, id: string | undefined) { const opts = coreOptsFromCtx(ctx); return useQuery({ queryKey: ["batch-insights", opts.region, id], - queryFn: async () => { - const { detail } = await core.eval.getBatchEvaluation(id!, opts, { - includeResults: false, - }); - InsightsJob.assert(detail, id!); - return detail; - }, + queryFn: () => core.eval.getBatchInsights(id!, opts), enabled: id !== undefined, }); } diff --git a/src/handlers/eval/batch-insights/insightsJob.ts b/src/handlers/eval/batch-insights/insightsJob.ts deleted file mode 100644 index 2b4559755..000000000 --- a/src/handlers/eval/batch-insights/insightsJob.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { InputValidationError } from "../../../errors"; - -export class InsightsJob { - static is(job: { insights?: unknown[] }): boolean { - return Boolean(job.insights?.length); - } - - static assert(job: { insights?: unknown[] }, id: string): void { - if (!InsightsJob.is(job)) { - throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); - } - } -} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 8d2380e13..f34d4a081 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -333,6 +333,7 @@ export interface CoreEvalClient { options: CoreOptions, opts?: { includeResults?: boolean }, ): Promise; + getBatchInsights(id: string, options: CoreOptions): Promise; listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 5a0784f36..b835ded7f 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -153,6 +153,7 @@ import type { ReadWriteJson } from "../io"; import { createSilentLogger } from "./logging"; import { FsProjectManager, type ProjectBackend } from "../core/project"; import type { ManagedBy } from "../projectSchemas/project"; +import { InputValidationError } from "../errors"; // TestCoreClient is a hand-controllable `Core` for tests. It implements the same // interface the real CoreClient satisfies, so it drops straight into @@ -1697,6 +1698,17 @@ export class TestEvalClient implements CoreEvalClient { return { detail }; } + async getBatchInsights(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getBatchInsights", args: [id, options] }); + if (this.error) throw this.error; + + const detail: BatchEvaluationDetail = { ...this.batchEvalGetResponse }; + if (!detail.insights?.length) { + throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); + } + return detail; + } + async listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined,