-
Notifications
You must be signed in to change notification settings - Fork 86
feat: add readonly batch insights TUI #2078
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,6 +41,11 @@ function toRow(summary: BatchEvaluationSummary): BatchEvaluationRow { | |
| export interface BatchEvaluationPickerProps extends ScreenProps { | ||
| breadcrumb: string[]; | ||
| description?: string; | ||
| queryKeyPrefix?: string; | ||
| include?: (summary: BatchEvaluationSummary) => boolean; | ||
| loadingMessage?: string; | ||
| emptyMessage?: string; | ||
| emptyPageMessage?: string; | ||
| onSelect: (batchEvaluationId: string) => void; | ||
| onEscape?: () => void; | ||
| } | ||
|
|
@@ -55,6 +60,11 @@ export function BatchEvaluationPicker({ | |
| core, | ||
| breadcrumb, | ||
| description, | ||
| queryKeyPrefix = "batch-evaluations", | ||
| include, | ||
| loadingMessage = "Loading batch evaluations…", | ||
| emptyMessage = "No batch evaluations found in this Region.", | ||
| emptyPageMessage = "No batch evaluations on this page.", | ||
| onSelect, | ||
| onEscape, | ||
| }: BatchEvaluationPickerProps) { | ||
|
|
@@ -66,11 +76,11 @@ export function BatchEvaluationPicker({ | |
| <PaginatedTablePicker | ||
| breadcrumb={breadcrumb} | ||
| description={description} | ||
| queryKey={["batch-evaluations", opts.region]} | ||
| queryKey={[queryKeyPrefix, opts.region]} | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nice! |
||
| loadPage={async (token, pageSize) => { | ||
| const response = await core.eval.listBatchEvaluations(token, pageSize, opts); | ||
| return { | ||
| items: response.batchEvaluations ?? [], | ||
| items: (response.batchEvaluations ?? []).filter((summary) => include?.(summary) ?? true), | ||
| nextToken: response.nextToken, | ||
| }; | ||
| }} | ||
|
|
@@ -79,10 +89,10 @@ export function BatchEvaluationPicker({ | |
| getValue={(row) => row.batchEvaluationId} | ||
| onSelect={onSelect} | ||
| onBack={goBack} | ||
| loadingMessage="Loading batch evaluations…" | ||
| loadingMessage={loadingMessage} | ||
| errorMessage={(error) => `Error: ${error.message}`} | ||
| emptyMessage="No batch evaluations found in this Region." | ||
| emptyPageMessage="No batch evaluations on this page." | ||
| emptyMessage={emptyMessage} | ||
| emptyPageMessage={emptyPageMessage} | ||
| /> | ||
| ); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
149 changes: 149 additions & 0 deletions
149
src/handlers/eval/batch-insights/batch-insights.screen.test.tsx
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| import { afterEach, describe, expect, test } from "bun:test"; | ||
| import type { | ||
| BatchEvaluationSummary, | ||
| GetBatchEvaluationResponse, | ||
| } from "@aws-sdk/client-bedrock-agentcore"; | ||
| import { | ||
| cleanupScreens, | ||
| renderScreen, | ||
| TestCoreClient, | ||
| waitFor, | ||
| waitForText, | ||
| } from "../../../testing"; | ||
|
|
||
| afterEach(cleanupScreens); | ||
|
|
||
| function summary(overrides: Partial<BatchEvaluationSummary> = {}): BatchEvaluationSummary { | ||
| return { | ||
| batchEvaluationArn: | ||
| "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/insights-1", | ||
| batchEvaluationId: "insights-1", | ||
| batchEvaluationName: "failure_analysis", | ||
| status: "COMPLETED", | ||
| createdAt: new Date("2026-08-20T01:02:03.000Z"), | ||
| updatedAt: new Date("2026-08-21T12:34:56.000Z"), | ||
| insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], | ||
| ...overrides, | ||
| }; | ||
| } | ||
|
|
||
| function detail(overrides: Partial<GetBatchEvaluationResponse> = {}): GetBatchEvaluationResponse { | ||
| return { | ||
| batchEvaluationArn: | ||
| "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/insights-1", | ||
| batchEvaluationId: "insights-1", | ||
| batchEvaluationName: "failure_analysis", | ||
| status: "COMPLETED", | ||
| createdAt: new Date("2026-08-20T01:02:03.000Z"), | ||
| updatedAt: new Date("2026-08-21T12:34:56.000Z"), | ||
| insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], | ||
| failureAnalysisResult: { | ||
| failures: [], | ||
| }, | ||
| ...overrides, | ||
| } as GetBatchEvaluationResponse; | ||
| } | ||
|
|
||
| function coreWithBatchEvaluations(items: BatchEvaluationSummary[]): TestCoreClient { | ||
| const core = new TestCoreClient(); | ||
| core.eval.setBatchEvalListResponse({ batchEvaluations: items }); | ||
| return core; | ||
| } | ||
|
|
||
| describe("batch-insights menu", () => { | ||
| test("offers only read-only commands", async () => { | ||
| const screen = renderScreen("/agentcore/eval/batch-insights"); | ||
|
|
||
| await waitForText(screen.lastFrame, "list batch insights runs"); | ||
| const frame = screen.lastFrame()!; | ||
| expect(frame).toContain("list"); | ||
| expect(frame).toContain("get"); | ||
| expect(frame).not.toContain("start an asynchronous batch insights run"); | ||
| }); | ||
| }); | ||
|
|
||
| describe("batch-insights picker", () => { | ||
| test("filters evaluator-only jobs from the shared service page", async () => { | ||
| const core = coreWithBatchEvaluations([ | ||
| summary(), | ||
| summary({ | ||
| batchEvaluationId: "evaluation-1", | ||
| batchEvaluationName: "quality_evaluation", | ||
| insights: undefined, | ||
| evaluators: [{ evaluatorId: "Builtin.Correctness" }], | ||
| }), | ||
| ]); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "failure_analysis"); | ||
| const frame = screen.lastFrame()!; | ||
| expect(frame).not.toContain("quality_evaluation"); | ||
| expect(core.eval.calls[0]?.method).toBe("listBatchEvaluations"); | ||
| }); | ||
|
|
||
| test("bare get redirects to the filtered picker", async () => { | ||
| const core = coreWithBatchEvaluations([summary()]); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/get", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "failure_analysis"); | ||
| expect(core.eval.calls[0]?.method).toBe("listBatchEvaluations"); | ||
| }); | ||
|
|
||
| test("selection opens the matching insights JSON", async () => { | ||
| const core = coreWithBatchEvaluations([summary()]); | ||
| core.eval.setBatchEvalGetResponse(detail()); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "failure_analysis"); | ||
| await screen.press("return"); | ||
| await waitForText(screen.lastFrame, "agentcore → eval → batch-insights → get → insights-1"); | ||
| await waitFor(() => | ||
| core.eval.calls.some( | ||
| (call) => call.method === "getBatchEvaluation" && call.args[0] === "insights-1", | ||
| ), | ||
| ); | ||
| }); | ||
|
|
||
| test("shows the Insights-specific empty state", async () => { | ||
| const core = coreWithBatchEvaluations([ | ||
| summary({ | ||
| insights: undefined, | ||
| evaluators: [{ evaluatorId: "Builtin.Correctness" }], | ||
| }), | ||
| ]); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "No batch insights found in this Region."); | ||
| }); | ||
| }); | ||
|
|
||
| describe("batch-insights detail", () => { | ||
| test("renders service-side reports without requesting CloudWatch results", async () => { | ||
| const core = new TestCoreClient(); | ||
| core.eval.setBatchEvalGetResponse(detail()); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/get/insights-1", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "failure_analysis"); | ||
| const frame = screen.lastFrame()!; | ||
| expect(frame).toContain('"failureAnalysisResult"'); | ||
| expect(core.eval.calls.find((call) => call.method === "getBatchEvaluation")).toEqual({ | ||
| method: "getBatchEvaluation", | ||
| args: ["insights-1", { region: "us-east-1" }, { includeResults: false }], | ||
| }); | ||
| }); | ||
|
|
||
| test("rejects direct navigation to an evaluator-only job", async () => { | ||
| const core = new TestCoreClient(); | ||
| core.eval.setBatchEvalGetResponse( | ||
| detail({ | ||
| batchEvaluationId: "evaluation-1", | ||
| insights: undefined, | ||
| evaluators: [{ evaluatorId: "Builtin.Correctness" }], | ||
| failureAnalysisResult: undefined, | ||
| }), | ||
| ); | ||
| const screen = renderScreen("/agentcore/eval/batch-insights/get/evaluation-1", { core }); | ||
|
|
||
| await waitForText(screen.lastFrame, "is not a batch insights run"); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| import { useQuery } from "@tanstack/react-query"; | ||
| import { useParams } from "react-router"; | ||
| import { JsonDetail } from "../../../../components/JsonDetail"; | ||
| import type { ScreenProps } from "../../../types"; | ||
| import { coreOptsFromCtx } from "../../../utils"; | ||
| import { InsightsJob } from "../insightsJob"; | ||
|
|
||
| function useBatchInsightsDetail({ ctx, core }: ScreenProps, id: string | undefined) { | ||
| const opts = coreOptsFromCtx(ctx); | ||
| return useQuery({ | ||
| queryKey: ["batch-insights", opts.region, id], | ||
| queryFn: async () => { | ||
| const { detail } = await core.eval.getBatchEvaluation(id!, opts, { | ||
| includeResults: false, | ||
| }); | ||
| InsightsJob.assert(detail, id!); | ||
| return detail; | ||
| }, | ||
| enabled: id !== undefined, | ||
| }); | ||
| } | ||
|
|
||
| export function BatchInsightsGetJsonScreen(props: ScreenProps) { | ||
| const { batchEvaluationId } = useParams(); | ||
| const query = useBatchInsightsDetail(props, batchEvaluationId); | ||
|
|
||
| return ( | ||
| <JsonDetail | ||
| breadcrumb={["agentcore", "eval", "batch-insights", "get", batchEvaluationId ?? ""]} | ||
| isPending={query.isPending} | ||
| error={query.isError ? (query.error as Error) : null} | ||
| data={query.data} | ||
| loadingLabel="Loading batch insights…" | ||
| onRetry={() => void query.refetch()} | ||
| /> | ||
| ); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,20 @@ | ||
| import type { AppIO } from "../../../io"; | ||
| import { Router } from "../../../router"; | ||
| import { createHelpDefault } from "../../help"; | ||
| import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; | ||
| import { renderTui } from "../../../tui"; | ||
| import type { Core } from "../../types"; | ||
| import { createGetBatchInsightsHandler } from "./get"; | ||
| import { createListBatchInsightsHandler } from "./list"; | ||
| import { createRunBatchInsightsHandler } from "./run"; | ||
|
|
||
| export function createBatchInsightsHandler(core: Core, io: AppIO): Router { | ||
| return new Router("batch-insights", "run and inspect batch insights") | ||
| .default(createHelpDefault(io)) | ||
| .use(withTuiOnEmptyFlagsAndArgs(core, io)) | ||
| .default(renderTui(core, io)) | ||
| .supportedTuiCommands("get", "list") | ||
| .handler(createRunBatchInsightsHandler(core, io)) | ||
| .handler(createGetBatchInsightsHandler(core)) | ||
| .handler(createListBatchInsightsHandler(core)); | ||
| } | ||
|
|
||
| export { BatchInsightsScreen } from "./screen.tsx"; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| import { InputValidationError } from "../../../errors"; | ||
|
|
||
| export class InsightsJob { | ||
| static is(job: { insights?: unknown[] }): boolean { | ||
| return Boolean(job.insights?.length); | ||
| } | ||
|
|
||
| static assert(job: { insights?: unknown[] }, id: string): void { | ||
| if (!InsightsJob.is(job)) { | ||
| throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| import { useNavigate } from "react-router"; | ||
| import { BatchEvaluationPicker } from "../../../../components/BatchEvaluationPicker"; | ||
| import type { ScreenProps } from "../../../types"; | ||
| import { InsightsJob } from "../insightsJob"; | ||
|
|
||
| export function BatchInsightsListScreen(props: ScreenProps) { | ||
| const navigate = useNavigate(); | ||
|
|
||
| return ( | ||
| <BatchEvaluationPicker | ||
| {...props} | ||
| breadcrumb={["agentcore", "eval", "batch-insights", "list"]} | ||
| queryKeyPrefix="batch-insights" | ||
| include={InsightsJob.is} | ||
| loadingMessage="Loading batch insights…" | ||
| emptyMessage="No batch insights found in this Region." | ||
| emptyPageMessage="No batch insights on this page." | ||
| onSelect={(batchEvaluationId) => | ||
| navigate(`/agentcore/eval/batch-insights/get/${encodeURIComponent(batchEvaluationId)}`) | ||
| } | ||
| /> | ||
| ); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| import { RouterScreen } from "../../../components/RouterScreen"; | ||
| import type { ScreenProps } from "../../types"; | ||
|
|
||
| export function BatchInsightsScreen(props: ScreenProps) { | ||
| return <RouterScreen {...props} path={["agentcore", "eval", "batch-insights"]} />; | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you explain why we are reusing BatchEvaluationPicker constructs for BatchInsights. They are two different resource that will diverge in the future.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The CLI experience and semantics are separate, but they aren't two different resources. Batch evals and insights have the same
listBatchEvaluationsAPI, response model, and display fields. Insights is just a filtered projection of BatchEvaluations, and has its own screen, routes, validations, etc.If the resources diverge to separate APIs in the future, we'll need a significant refactor to the insights logic regardless of them sharing resources or not. Separating this constructs now creates more code to maintain upfront, and does not save us any effort in the future.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I remember Swarmim said they were two different resources and I think we should seperate them but I'm fine just rename all shared compoenent like BatchEvalautionPicker to BatchEvaluation/InsightPicker so we know that its used for both.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Our CLI experience does match the idea that they are two different resources to the customer/product side (screens, commands, routes, etc.). When I say they're the same resource I mean on the service/API abstraction side.
The name
BatchEvaluationPickeris accurate to the service resource it loads. It requests and renders all batch evaluations, which includes insights. I feel like renaming it to include insights implies that the picker operates on two service resources, when it doesn't. The picker does exactly what it is named for: gives all batch insights.src/handlers/eval/batch-insights/list/screen.tsxhas to filter out insights from the full list of batch evaluations. So that logic is distinct to the insights flow. I think the idea that insights is getting batch evals, and filtering down is accurate with whats really happening. Naming it otherwise would be misleading to whats actually happening