From ae6950043839e9f40125ae89f26c57e4e1c982d6 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Wed, 2 Sep 2026 15:03:26 +0100 Subject: [PATCH] Avoid double-charging for classification if it escalates to AI --- .../useClientSideClassification.test.tsx | 39 ++++++++++++++++--- .../policies/useClientSideClassification.ts | 12 ++++-- .../proprietary/data/classificationPolicy.ts | 9 ++++- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx index 86db46949c..dc69cb4699 100644 --- a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx +++ b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.test.tsx @@ -117,8 +117,10 @@ describe("useClientSideClassification delivery", () => { it("classifies pending uploads, writes labels, and meters once per file", async () => { mocks.workspace = [stub("a"), stub("b")]; + // A confident verdict stands locally, so the local pass is the billable run. mocks.classify.mockImplementation(async (file: File) => ({ labels: [file.name.startsWith("a") ? "invoice" : "resume"], + confidence: "high", })); renderHook(() => useClientSideClassification()); @@ -128,19 +130,43 @@ describe("useClientSideClassification delivery", () => { ); expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", { classificationLabels: ["invoice"], + classificationConfidence: "high", }); expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", { classificationLabels: ["resume"], + classificationConfidence: "high", }); expect(mocks.meter).toHaveBeenCalledTimes(2); expect(mocks.bumpRevision).toHaveBeenCalled(); }); + it("does not meter a low-confidence local pass; the AI escalation bills it", async () => { + mocks.workspace = [stub("unsure")]; + mocks.classify.mockResolvedValue({ + labels: ["maybe-invoice"], + confidence: "low", + }); + + renderHook(() => useClientSideClassification()); + + await waitFor(() => + expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("unsure", { + classificationLabels: ["maybe-invoice"], + classificationConfidence: "low", + }), + ); + // Labels still written locally, but no local charge - the file escalates to the AI run. + expect(mocks.meter).not.toHaveBeenCalled(); + }); + it("delivers a result computed while the effect re-fired mid-batch (upload-wave race)", async () => { - let resolveA!: (v: { labels: string[] }) => void; - const gateA = new Promise<{ labels: string[] }>((r) => (resolveA = r)); + type Verdict = { labels: string[]; confidence: string }; + let resolveA!: (v: Verdict) => void; + const gateA = new Promise((r) => (resolveA = r)); mocks.classify.mockImplementation((file: File) => - file.name.startsWith("a") ? gateA : Promise.resolve({ labels: ["nda"] }), + file.name.startsWith("a") + ? gateA + : Promise.resolve({ labels: ["nda"], confidence: "high" }), ); mocks.workspace = [stub("a")]; @@ -152,16 +178,18 @@ describe("useClientSideClassification delivery", () => { mocks.workspace = [stub("a"), stub("b")]; rerender(); - resolveA({ labels: ["purchase-order"] }); + resolveA({ labels: ["purchase-order"], confidence: "high" }); await waitFor(() => expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("a", { classificationLabels: ["purchase-order"], + classificationConfidence: "high", }), ); // The newly-arrived file classifies too, and neither is double-classified. await waitFor(() => expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("b", { classificationLabels: ["nda"], + classificationConfidence: "high", }), ); expect(mocks.classify).toHaveBeenCalledTimes(2); @@ -170,13 +198,14 @@ describe("useClientSideClassification delivery", () => { it("persists a definitive [] verdict for an unlabelled file and does not retry it", async () => { mocks.workspace = [stub("plain")]; - mocks.classify.mockResolvedValue({ labels: [] }); + mocks.classify.mockResolvedValue({ labels: [], confidence: "high" }); renderHook(() => useClientSideClassification()); await waitFor(() => expect(mocks.updateStirlingFileStub).toHaveBeenCalledWith("plain", { classificationLabels: [], + classificationConfidence: "high", }), ); expect(mocks.classify).toHaveBeenCalledTimes(1); diff --git a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts index 252b9a7780..c129930e54 100644 --- a/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts +++ b/frontend/editor/src/proprietary/components/policies/useClientSideClassification.ts @@ -20,7 +20,10 @@ import { import type { FileId } from "@app/types/file"; import type { StirlingFile, StirlingFileStub } from "@app/types/fileContext"; import type { HeuristicConfidence } from "@app/services/heuristic/types"; -import { CLASSIFICATION_CATEGORY_ID } from "@app/data/classificationPolicy"; +import { + CLASSIFICATION_CATEGORY_ID, + localVerdictStands, +} from "@app/data/classificationPolicy"; /** * Dispatch-store key namespace for "this file's local pass has been metered". Deliberately NOT the @@ -204,9 +207,10 @@ async function classifyStub( (alreadyMetered ? " [heal: not re-metered]" : ""), ); if (debug && result.explain) logExplanation(fileName, result); - // Meter on the first classification only; a healing re-run of an undelivered - // result (already dispatched) is not a new billable run. - if (!alreadyMetered) { + // Meter the local pass only when its verdict stands: an unsure verdict escalates to the AI + // run, which is billed there instead (avoids double-charging one classification). Metered on + // the first pass only; a healing re-run of an undelivered result is not a new billable run. + if (!alreadyMetered && localVerdictStands(result.confidence)) { meterAutomationRun({ automationName: "Classification", operations: [CLASSIFY_STEP], diff --git a/frontend/editor/src/proprietary/data/classificationPolicy.ts b/frontend/editor/src/proprietary/data/classificationPolicy.ts index 60ef0de23f..9a0d959ea4 100644 --- a/frontend/editor/src/proprietary/data/classificationPolicy.ts +++ b/frontend/editor/src/proprietary/data/classificationPolicy.ts @@ -55,6 +55,13 @@ export function orderRewritesFirst(categoryIds: string[]): string[] { */ const TRUSTED_CONFIDENCE: ClassificationConfidence = "high"; +/** Whether a local heuristic verdict is final and stands on its own. */ +export function localVerdictStands( + confidence: ClassificationConfidence | undefined, +): boolean { + return confidence === TRUSTED_CONFIDENCE; +} + /** * Whether the AI classifier should be asked about this file. For an upload, only once the * heuristic has reported: dispatching before then races the first pass and bills for an answer it @@ -69,5 +76,5 @@ export function shouldDispatchToAi( if (!isClassificationCategory(categoryId)) return true; const confidence = stub.classificationConfidence; if (confidence == null) return Boolean(stub.derivedFromTool); - return confidence !== TRUSTED_CONFIDENCE; + return !localVerdictStands(confidence); }