From 3d00189e7daa8f8de5887c98babd6cf0cb407bd9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 12 Aug 2026 19:40:02 +0100 Subject: [PATCH 1/2] fix(webapp): stop the billing limits page timing out under enforcement The loader counted queued runs with one sequential ClickHouse query per billable environment, and preview branches (archived included) all count as billable environments. Orgs with thousands of preview branches held the response open past the edge timeout, exactly when a billing limit was being enforced and the page was the only self-serve way out. The count is now a single org-level query on environment_type with a server-side execution cap, and a count failure falls back to 0 instead of failing the loader. The bulk-cancel path no longer enumerates archived environments. --- .../billing-limits-page-timeout.md | 6 + .../billingLimit/billingLimitConstants.ts | 3 + .../billingLimitQueuedRuns.server.ts | 47 +++-- .../getBillingLimitQueuedRunCount.server.ts | 19 +- .../test/billingLimitQueuedRuns.test.ts | 175 +++++++++++++++++- 5 files changed, 232 insertions(+), 18 deletions(-) create mode 100644 .server-changes/billing-limits-page-timeout.md diff --git a/.server-changes/billing-limits-page-timeout.md b/.server-changes/billing-limits-page-timeout.md new file mode 100644 index 0000000000..6edbccf48e --- /dev/null +++ b/.server-changes/billing-limits-page-timeout.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed the billing limits page timing out for organizations with many preview branches, especially while a spend limit was being enforced. The page now loads quickly, so you can raise or resolve your limit without delay. diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts index fbaeea845c..5aea0f1afd 100644 --- a/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts @@ -16,6 +16,9 @@ export const BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY = 10; /** Inline bulk-cancel budget for billing limit resolve (worker visibility is 10 min). */ export const BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS = 8 * 60_000; +/** Server-side cap (seconds) on the org-level queued-run count; it renders in a loader. */ +export const BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S = 10; + export type BillingLimitConvergeTargetState = "grace" | "rejected" | "ok"; export function isBillableEnvironmentType(type: RuntimeEnvironmentType): boolean { diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts index 7cf3cf7cd9..e56855cd6d 100644 --- a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts @@ -1,9 +1,13 @@ +import type { ClickHouse } from "@internal/clickhouse"; import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus"; import { prisma } from "~/db.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; -import { BILLABLE_ENVIRONMENT_TYPES } from "./billingLimitConstants"; +import { + BILLABLE_ENVIRONMENT_TYPES, + BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S, +} from "./billingLimitConstants"; import { boundedIn } from "@trigger.dev/database"; export type BillableEnvironmentRef = { @@ -19,6 +23,7 @@ export async function getBillableEnvironmentsForBillingLimit( where: { organizationId, type: { in: boundedIn([...BILLABLE_ENVIRONMENT_TYPES]) }, + archivedAt: null, }, select: { id: true, @@ -73,27 +78,37 @@ async function countRunsForBillableEnvironment( }); } -/** Same source as BillingLimitBulkCancelService — ClickHouse countRuns(QUEUED_STATUSES). */ +/** + * Same table and statuses as BillingLimitBulkCancelService's per-environment counts, but a + * single org-level ClickHouse query filtered on environment_type. Deliberately NOT a + * per-environment loop: an org can have thousands of (mostly archived) preview environments, + * and sequential per-env counts hold the billing-limits loader open past the edge timeout. + * The count is display-only, so a server-side execution cap beats an unbounded query. + */ export async function countBillableQueuedRunsForOrganization( - organizationId: string + organizationId: string, + clickhouse?: ClickHouse ): Promise { - const environments = await getBillableEnvironmentsForBillingLimit(organizationId); + const client = + clickhouse ?? + (await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard")); - if (environments.length === 0) { - return 0; - } + const queryBuilder = client.taskRuns.countQueryBuilder({ + settings: { max_execution_time: BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S }, + }); - const runsRepository = await createBillingLimitRunsRepository(organizationId); + queryBuilder + .where("organization_id = {organizationId: String}", { organizationId }) + .where("environment_type IN {environmentTypes: Array(String)}", { + environmentTypes: [...BILLABLE_ENVIRONMENT_TYPES], + }) + .where("status IN {statuses: Array(String)}", { statuses: [...QUEUED_STATUSES] }); - let total = 0; + const [queryError, result] = await queryBuilder.execute(); - for (const environment of environments) { - total += await countQueuedRunsForBillableEnvironment( - runsRepository, - organizationId, - environment - ); + if (queryError) { + throw queryError; } - return total; + return result[0]?.count ?? 0; } diff --git a/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts index 2689a3dc9b..73915d1237 100644 --- a/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts @@ -1,9 +1,26 @@ +import { tryCatch } from "@trigger.dev/core"; import { EnvironmentPauseSource } from "@trigger.dev/database"; import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; import { countBillableQueuedRunsForOrganization } from "./billingLimitQueuedRuns.server"; +/** + * Display-only count for the billing-limits page. Falls back to 0 on failure (the page hides + * the count label at 0) — the recovery panel must stay reachable even when the count errors, + * because it is the customer's only self-serve path out of an enforced limit. + */ export async function getBillingLimitQueuedRunCount(organizationId: string): Promise { - return countBillableQueuedRunsForOrganization(organizationId); + const [error, count] = await tryCatch(countBillableQueuedRunsForOrganization(organizationId)); + + if (error) { + logger.error("getBillingLimitQueuedRunCount failed, returning 0", { + organizationId, + error, + }); + return 0; + } + + return count; } export async function countBillingLimitPausedEnvironments(organizationId: string): Promise { diff --git a/apps/webapp/test/billingLimitQueuedRuns.test.ts b/apps/webapp/test/billingLimitQueuedRuns.test.ts index 9c923890eb..c279b692f5 100644 --- a/apps/webapp/test/billingLimitQueuedRuns.test.ts +++ b/apps/webapp/test/billingLimitQueuedRuns.test.ts @@ -2,7 +2,11 @@ import { describe, expect, vi } from "vitest"; import { setTimeout } from "node:timers/promises"; import { replicationContainerTest } from "@internal/testcontainers"; import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; -import { countQueuedRunsForBillableEnvironment } from "~/v3/services/billingLimit/billingLimitQueuedRuns.server"; +import { + countBillableQueuedRunsForOrganization, + countQueuedRunsForBillableEnvironment, + getBillableEnvironmentsForBillingLimit, +} from "~/v3/services/billingLimit/billingLimitQueuedRuns.server"; import { setupClickhouseReplication } from "./utils/replicationUtils"; vi.setConfig({ testTimeout: 60_000 }); @@ -109,4 +113,173 @@ describe("billingLimitQueuedRuns", () => { expect(developmentCount).toBe(1); } ); + + replicationContainerTest( + "counts queued runs org-wide with a single query, spanning billable environment types only", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + const { clickhouse } = await setupClickhouseReplication({ + prisma, + databaseUrl: postgresContainer.getConnectionUri(), + clickhouseUrl: clickhouseContainer.getConnectionUrl(), + redisOptions, + }); + + const organization = await prisma.organization.create({ + data: { title: "billing-limit-org-count", slug: "billing-limit-org-count" }, + }); + + const project = await prisma.project.create({ + data: { + name: "billing-limit-org-count", + slug: "billing-limit-org-count", + organizationId: organization.id, + externalRef: "billing-limit-org-count", + }, + }); + + const productionEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: "prod-org-count", + pkApiKey: "prod-org-count", + shortcode: "prod-org-count", + }, + }); + + const developmentEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: "dev-org-count", + pkApiKey: "dev-org-count", + shortcode: "dev-org-count", + }, + }); + + const previewEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "preview", + type: "PREVIEW", + projectId: project.id, + organizationId: organization.id, + apiKey: "preview-org-count", + pkApiKey: "preview-org-count", + shortcode: "preview-org-count", + branchName: "feature-branch", + archivedAt: new Date(), + }, + }); + + const runRows = [ + { + friendlyId: "run_org_prod_pending", + env: productionEnv, + type: "PRODUCTION", + status: "PENDING", + }, + { + friendlyId: "run_org_prod_delayed", + env: productionEnv, + type: "PRODUCTION", + status: "DELAYED", + }, + { + friendlyId: "run_org_prod_done", + env: productionEnv, + type: "PRODUCTION", + status: "COMPLETED_SUCCESSFULLY", + }, + { + friendlyId: "run_org_dev_pending", + env: developmentEnv, + type: "DEVELOPMENT", + status: "PENDING", + }, + { + friendlyId: "run_org_preview_pending", + env: previewEnv, + type: "PREVIEW", + status: "PENDING", + }, + ] as const; + + for (const row of runRows) { + await prisma.taskRun.create({ + data: { + friendlyId: row.friendlyId, + taskIdentifier: "queued-task", + status: row.status, + payload: JSON.stringify({}), + traceId: "trace", + spanId: "span", + queue: "main", + runtimeEnvironmentId: row.env.id, + projectId: project.id, + organizationId: organization.id, + environmentType: row.type, + engine: "V2", + }, + }); + } + + await setTimeout(1000); + + const count = await countBillableQueuedRunsForOrganization(organization.id, clickhouse); + + expect(count).toBe(3); + } + ); + + replicationContainerTest( + "excludes archived environments from the billable environment list", + async ({ prisma }) => { + const organization = await prisma.organization.create({ + data: { title: "billing-limit-archived", slug: "billing-limit-archived" }, + }); + + const project = await prisma.project.create({ + data: { + name: "billing-limit-archived", + slug: "billing-limit-archived", + organizationId: organization.id, + externalRef: "billing-limit-archived", + }, + }); + + const activeEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: "prod-archived-test", + pkApiKey: "prod-archived-test", + shortcode: "prod-archived-test", + }, + }); + + await prisma.runtimeEnvironment.create({ + data: { + slug: "preview-archived", + type: "PREVIEW", + projectId: project.id, + organizationId: organization.id, + apiKey: "preview-archived-test", + pkApiKey: "preview-archived-test", + shortcode: "preview-archived-test", + branchName: "old-branch", + archivedAt: new Date(), + }, + }); + + const environments = await getBillableEnvironmentsForBillingLimit(organization.id, prisma); + + expect(environments).toEqual([{ id: activeEnv.id, projectId: project.id }]); + } + ); }); From 86c70968decad7a0816788b9d19e3ab5b91f54f7 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 12 Aug 2026 20:01:19 +0100 Subject: [PATCH 2/2] fix(webapp): keep archived branches covered by billing limit enforcement Archiving a branch is a soft update that cancels nothing, so an archived branch can still hold executing runs a spend limit must cancel. Filtering archived environments out of the billable list opened an enforcement gap and made the displayed queued count disagree with what a resolve would act on, so the list keeps them and a test now pins that. Also guards the count fallback against falsy rejection values and imports tryCatch from the core utils subpath. --- .../billingLimitQueuedRuns.server.ts | 6 +++- .../getBillingLimitQueuedRunCount.server.ts | 4 +-- .../test/billingLimitQueuedRuns.test.ts | 31 ++++++++++++++----- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts index e56855cd6d..066af4be10 100644 --- a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts @@ -15,6 +15,11 @@ export type BillableEnvironmentRef = { projectId: string; }; +/** + * Environments whose runs a billing limit must act on. Archived branches stay included: + * archiving is a soft update that cancels nothing, so an archived branch can still hold + * executing runs that enforcement has to cancel. + */ export async function getBillableEnvironmentsForBillingLimit( organizationId: string, prismaClient: PrismaClient = prisma @@ -23,7 +28,6 @@ export async function getBillableEnvironmentsForBillingLimit( where: { organizationId, type: { in: boundedIn([...BILLABLE_ENVIRONMENT_TYPES]) }, - archivedAt: null, }, select: { id: true, diff --git a/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts index 73915d1237..af0e20a565 100644 --- a/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts +++ b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts @@ -1,4 +1,4 @@ -import { tryCatch } from "@trigger.dev/core"; +import { tryCatch } from "@trigger.dev/core/utils"; import { EnvironmentPauseSource } from "@trigger.dev/database"; import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; @@ -20,7 +20,7 @@ export async function getBillingLimitQueuedRunCount(organizationId: string): Pro return 0; } - return count; + return count ?? 0; } export async function countBillingLimitPausedEnvironments(organizationId: string): Promise { diff --git a/apps/webapp/test/billingLimitQueuedRuns.test.ts b/apps/webapp/test/billingLimitQueuedRuns.test.ts index c279b692f5..14dc0c0db8 100644 --- a/apps/webapp/test/billingLimitQueuedRuns.test.ts +++ b/apps/webapp/test/billingLimitQueuedRuns.test.ts @@ -236,7 +236,7 @@ describe("billingLimitQueuedRuns", () => { ); replicationContainerTest( - "excludes archived environments from the billable environment list", + "keeps archived environments in the billable environment list so enforcement can cancel their runs", async ({ prisma }) => { const organization = await prisma.organization.create({ data: { title: "billing-limit-archived", slug: "billing-limit-archived" }, @@ -253,17 +253,18 @@ describe("billingLimitQueuedRuns", () => { const activeEnv = await prisma.runtimeEnvironment.create({ data: { - slug: "prod", - type: "PRODUCTION", + slug: "preview-active", + type: "PREVIEW", projectId: project.id, organizationId: organization.id, - apiKey: "prod-archived-test", - pkApiKey: "prod-archived-test", - shortcode: "prod-archived-test", + apiKey: "preview-active-test", + pkApiKey: "preview-active-test", + shortcode: "preview-active-test", + branchName: "live-branch", }, }); - await prisma.runtimeEnvironment.create({ + const archivedEnv = await prisma.runtimeEnvironment.create({ data: { slug: "preview-archived", type: "PREVIEW", @@ -277,9 +278,23 @@ describe("billingLimitQueuedRuns", () => { }, }); + const developmentEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: "dev-archived-test", + pkApiKey: "dev-archived-test", + shortcode: "dev-archived-test", + }, + }); + const environments = await getBillableEnvironmentsForBillingLimit(organization.id, prisma); + const environmentIds = environments.map((environment) => environment.id).sort(); - expect(environments).toEqual([{ id: activeEnv.id, projectId: project.id }]); + expect(environmentIds).toEqual([activeEnv.id, archivedEnv.id].sort()); + expect(environmentIds).not.toContain(developmentEnv.id); } ); });