Files
OwnCord/.claude/workflows/bughunt-fix.harness.mjs
T
J3vb db0275a290 fix: batch of 29 correctness fixes across server and client (#1369)
* fix(ws): 2 defect(s) (OC-0013, OC-0140)

* fix(voice): 1 defect(s) (OC-0044)

* fix(ws): 1 defect(s) (OC-0024)

* fix(server): 1 defect(s) (OC-0027)

* fix(ws): 1 defect(s) (OC-0028)

* fix(server): 7 defect(s) (OC-0033, OC-0066, OC-0067, OC-0068, OC-0074, OC-0077, OC-0106)

* fix(ws): 1 defect(s) (OC-0051)

* fix(client): 1 defect(s) (OC-0053)

* fix(client): 1 defect(s) (OC-0055)

* fix(service): 1 defect(s) (OC-0069)

* fix(voice): 1 defect(s) (OC-0072)

* fix(service): 1 defect(s) (OC-0082)

* fix(client): 1 defect(s) (OC-0083)

* fix(plugin): 1 defect(s) (OC-0088)

* fix(plugin): 4 defect(s) (OC-0104, OC-0126, OC-0127, OC-0133)

* fix(admin): 1 defect(s) (OC-0110)

* fix(client): 1 defect(s) (OC-0114)

* fix(api): 1 defect(s) (OC-0139)

* fix(client): 1 defect(s) (OC-0149)

* test(server): adapt existing tests to updated OpenDM and IncrementMentionCounts signatures

* style(plugin): modernize loops and goroutine spawns in race test

* fix(ws): mirror the focus admission gate in the post-subscribe revalidation

* fix(service): detach DM post-commit side effects from the request ctx, fail delete closed, add empty-fan-out fallback

* fix(plugin): preserve enabled intent when upgrade reactivation hits a runtime-less build

* chore(skills): harden bughunt-fix workflow and fold review lessons into bughunt-run/db-change

* Add comprehensive documentation for task-observer skill

- Introduced environments.md to outline activation setup, compaction behavior, and handoff-doc mode.
- Created skill-authoring.md detailing taxonomy, licensing, confidentiality, and editing rules for skill creation.
- Added weekly-review.md for a structured review process of OPEN observations, including scheduled and in-session fallback modes.

* chore(go): pin toolchain go1.26.6 (stdlib CVE fixes flagged by govulncheck)
2026-08-14 10:05:40 +02:00

817 lines
35 KiB
JavaScript

// Offline harness for bughunt-fix.js - mirrors bughunt.harness.mjs: wraps the script body in an
// AsyncFunction with stubbed agent/parallel/pipeline/phase/log/args/budget.
// Run: node .claude/workflows/bughunt-fix.harness.mjs [nameFilter]
import { readFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import assert from 'node:assert/strict'
const here = dirname(fileURLToPath(import.meta.url))
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor
export async function run({ agentStub, args = undefined, budget = undefined }) {
const src = readFileSync(join(here, 'bughunt-fix.js'), 'utf8')
const body = src.replace('export const meta', 'const meta')
const calls = []
const logs = []
const agent = async (prompt, opts = {}) => {
calls.push({ prompt, opts })
return agentStub(prompt, opts)
}
const parallel = (thunks) =>
Promise.all(thunks.map((t) => Promise.resolve().then(t).catch(() => null)))
const pipeline = (items, ...stages) =>
Promise.all(
items.map(async (item, i) => {
let v = item
for (const stage of stages) {
try {
v = await stage(v, item, i)
} catch {
return null
}
}
return v
}),
)
const log = (m) => logs.push(String(m))
const phase = () => {}
const budgetImpl = budget || { total: null, spent: () => 0, remaining: () => Infinity }
const fn = new AsyncFunction('agent', 'parallel', 'pipeline', 'phase', 'log', 'args', 'budget', body)
const result = await fn(agent, parallel, pipeline, phase, log, args, budgetImpl)
return { result, calls, logs }
}
// ---------- fixtures ----------
export const rec = (id, over = {}) => ({
id,
title: `bug ${id}`,
file: 'Client/tauri-client/src/lib/livekitE2EE.ts',
line: 100,
severity: 'high',
why: 'w',
repro: 'r',
evidence: 'e',
status: 'open',
found: '2026-08-09',
hunt: 'h',
lens: 'l',
fix: null,
...over,
})
const scenarios = {}
// F1: clustering groups by file; cluster count equals distinct file count.
scenarios.f1_clusters_by_file = async () => {
const findings = [
rec('OC-0001'),
rec('OC-0002', { line: 800 }),
rec('OC-0003', { file: 'Server/ws/hub_sweep.go' }),
]
const { result } = await run({
args: { findings, branch: 'fix/test' },
agentStub: () => {
throw new Error('no agent should run in phase 1 with the later phases unimplemented')
},
})
assert.equal(result.branch, 'fix/test')
assert.equal(result.clusters.length, 2)
const byFile = Object.fromEntries(result.clusters.map((c) => [c.file, c.ids]))
assert.deepEqual(byFile['Client/tauri-client/src/lib/livekitE2EE.ts'], ['OC-0001', 'OC-0002'])
assert.deepEqual(byFile['Server/ws/hub_sweep.go'], ['OC-0003'])
}
// F1b: a backslash/case-only variant of the same path must not split into a second cluster -
// the same disjointness invariant the cross-cluster guard protects, at the grouping step.
scenarios.f1b_clustering_normalizes_backslashes = async () => {
const findings = [
rec('OC-0001', { file: 'Server/ws/hub_sweep.go' }),
rec('OC-0002', { file: 'Server\\ws\\hub_sweep.go' }),
]
const { result } = await run({
args: { findings },
agentStub: () => {
throw new Error('no agent should run in phase 1 with the later phases unimplemented')
},
})
assert.equal(result.clusters.length, 1, 'a backslash variant of the same path must merge into one cluster')
assert.deepEqual(result.clusters[0].ids.sort(), ['OC-0001', 'OC-0002'])
assert.equal(result.clusters[0].file, 'Server/ws/hub_sweep.go')
}
// F2: only / maxSeverity / non-open status all exclude, and every exclusion is logged by id.
scenarios.f2_exclusions_are_announced = async () => {
const findings = [
rec('OC-0001'),
rec('OC-0002', { severity: 'low' }),
rec('OC-0003', { status: 'fixed' }),
rec('OC-0004'),
]
const { result, logs } = await run({
args: { findings, only: ['OC-0001', 'OC-0002', 'OC-0003'], maxSeverity: 'medium' },
agentStub: () => {
throw new Error('no agent expected')
},
})
const reasons = Object.fromEntries(result.excluded.map((e) => [e.id, e.reason]))
assert.equal(reasons['OC-0002'], 'below maxSeverity')
assert.equal(reasons['OC-0003'], 'status is fixed, not open')
assert.equal(reasons['OC-0004'], 'not in only')
assert.equal(result.clusters.length, 1)
assert.deepEqual(result.clusters[0].ids, ['OC-0001'])
const joined = logs.join('\n')
for (const id of ['OC-0002', 'OC-0003', 'OC-0004']) {
assert.match(joined, new RegExp(id), `exclusion of ${id} must be logged, not silent`)
}
}
// F3: one sonnet/xhigh agent per cluster, and the prompt carries every finding in that file.
scenarios.f3_one_xhigh_agent_per_cluster = async () => {
const findings = [
rec('OC-0001'),
rec('OC-0002', { line: 800 }),
rec('OC-0003', { file: 'Server/ws/hub_sweep.go' }),
]
const { result, calls } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (!String(opts.label).startsWith('fix:')) throw new Error(`unexpected label ${opts.label}`)
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: `t/${id}.test.ts`, rationale: '' })), touchedPaths: [] }
},
})
const fixCalls = calls.filter((c) => String(c.opts.label).startsWith('fix:'))
assert.equal(fixCalls.length, 2, 'one agent per file cluster')
for (const c of fixCalls) {
assert.equal(c.opts.model, 'sonnet')
assert.equal(c.opts.effort, 'xhigh')
assert.equal(c.opts.phase, 'Fix')
}
const e2eeCall = fixCalls.find((c) => c.opts.label.includes('livekitE2EE'))
assert.match(e2eeCall.prompt, /OC-0001/)
assert.match(e2eeCall.prompt, /OC-0002/)
assert.ok(!e2eeCall.prompt.includes('OC-0003'), 'a cluster prompt must not leak another file\'s findings')
assert.match(e2eeCall.prompt, /write a test that fails/i, 'rule 1: test-first')
assert.match(e2eeCall.prompt, /weakening an assertion/i, 'rule 2: never weaken an assertion')
assert.match(e2eeCall.prompt, /grep every caller/i, 'rule 3: root cause, grep callers')
assert.match(e2eeCall.prompt, /touchedPaths/i, 'rule 4: shared-file edits must be listed in touchedPaths')
assert.match(e2eeCall.prompt, /one change that closes more than one/i, 'rule 5: one change closing several findings')
assert.match(e2eeCall.prompt, /do not run any git command/i, 'rule 6: no git')
assert.match(e2eeCall.prompt, /do not invent a fix/i, 'rule 7: declined with a rationale')
assert.match(e2eeCall.prompt, /mechanical reason/i, 'rule 8: blocked with a rationale')
assert.equal(result.results.length, 3)
}
// F4: a dead fix agent marks only its own cluster; siblings still report.
scenarios.f4_dead_agent_does_not_poison_siblings = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (opts.label.includes('hub_sweep')) throw new Error('agent died')
if (String(opts.label).startsWith('prove:'))
return {
committed: true,
sha: 'aaa0000',
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)',
note: '',
}
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
},
})
const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome]))
assert.equal(byId['OC-0001'], 'fixed')
assert.equal(byId['OC-0003'], 'blocked')
const reason = result.results.find((r) => r.id === 'OC-0003').rationale
assert.match(reason, /agent/i)
}
// F5: a declined finding keeps its rationale and is not treated as fixed.
scenarios.f5_decline_propagates = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: () => ({
results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'intended behaviour, locked by test X' }],
touchedPaths: [],
}),
})
assert.equal(result.results[0].outcome, 'declined')
assert.equal(result.results[0].rationale, 'intended behaviour, locked by test X')
}
// F5b: a foreign id (hallucinated, or copy-pasted from a different cluster) is dropped, not merged,
// and its id is announced in the logs rather than disappearing silently.
scenarios.f5b_foreign_id_is_dropped_and_announced = async () => {
const findings = [rec('OC-0001')]
const { result, logs } = await run({
args: { findings },
agentStub: () => ({
results: [
{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' },
{ id: 'OC-9999', outcome: 'fixed', testPath: 't2.ts', rationale: '' },
],
touchedPaths: [],
}),
})
assert.deepEqual(result.results.map((r) => r.id), ['OC-0001'])
assert.match(logs.join('\n'), /OC-9999/, 'the dropped foreign id must be announced in the logs')
}
// F6: prove agents run serially (never overlapping) and only for clusters that produced a fix.
scenarios.f6_prove_is_serial = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
let inFlight = 0
let maxInFlight = 0
const { result, calls } = await run({
args: { findings },
agentStub: async (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: `t/${id}.ts`, rationale: '' })), touchedPaths: [] }
}
inFlight++
maxInFlight = Math.max(maxInFlight, inFlight)
await new Promise((r) => setTimeout(r, 5))
inFlight--
return {
committed: true,
sha: 'abc1234',
redObserved: true,
greenObserved: true,
redOutput: '$ go test ./ws/ -run TestSweep\nFAIL: reverted source',
greenOutput: '$ go test ./ws/ -run TestSweep\nPASS: fix restored',
note: '',
}
},
})
assert.equal(maxInFlight, 1, 'prove/commit must be serial - git index contention')
const proveCalls = calls.filter((c) => String(c.opts.label).startsWith('prove:'))
assert.equal(proveCalls.length, 2)
for (const c of proveCalls) {
assert.equal(c.opts.model, 'opus')
assert.equal(c.opts.effort, 'medium')
}
assert.equal(result.commits.length, 2)
assert.deepEqual(result.commits[0], { sha: 'abc1234', file: findings[0].file, ids: ['OC-0001'] })
}
// F7: a vacuous test (revert-proof does not go RED) is NOT committed and its findings go blocked.
scenarios.f7_vacuous_test_is_not_committed = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
return {
committed: false,
sha: '',
redObserved: false,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (reverted, should have failed)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)',
note: 'test passed with the fix reverted',
}
},
})
assert.equal(result.commits.length, 0, 'a cluster that failed its revert-proof must not be committed')
assert.equal(result.results[0].outcome, 'blocked')
assert.match(result.results[0].rationale, /revert-proof/i)
}
// F7b: RED was observed but the restored fix does not go GREEN - also not committed.
scenarios.f7b_restored_fix_must_be_green = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
return {
committed: false,
sha: '',
redObserved: true,
greenObserved: false,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (reverted)',
greenOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (still failing after restore)',
note: 'still failing after restore',
}
},
})
assert.equal(result.commits.length, 0)
assert.equal(result.results[0].outcome, 'blocked')
assert.match(result.results[0].rationale, /after restoring the fix/i)
}
// F8: a cluster with only declines is never sent to prove, and produces no commit.
scenarios.f8_declined_cluster_skips_prove = async () => {
const findings = [rec('OC-0001')]
const { result, calls } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'by design' }], touchedPaths: [] }
throw new Error('prove must not run for a cluster with no fixes')
},
})
assert.ok(!calls.some((c) => String(c.opts.label).startsWith('prove:')))
assert.equal(result.commits.length, 0)
assert.equal(result.results[0].outcome, 'declined')
}
// F9: one failing cluster does not stop its siblings from committing.
scenarios.f9_blocked_cluster_does_not_block_siblings = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: 't.ts', rationale: '' })), touchedPaths: [] }
}
if (opts.label.includes('livekitE2EE'))
return {
committed: false,
sha: '',
redObserved: false,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (reverted, should have failed)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)',
note: 'vacuous',
}
return {
committed: true,
sha: 'def5678',
redObserved: true,
greenObserved: true,
redOutput: '$ go test ./ws/ -run TestSweep\nFAIL: reverted source',
greenOutput: '$ go test ./ws/ -run TestSweep\nPASS: fix restored',
note: '',
}
},
})
assert.equal(result.commits.length, 1)
assert.equal(result.commits[0].sha, 'def5678')
const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome]))
assert.equal(byId['OC-0001'], 'blocked')
assert.equal(byId['OC-0003'], 'fixed')
}
// F9b: a mixed cluster (one fixed + one declined) whose prove fails demotes only the fixed
// finding to blocked; the declined finding and its original rationale are left untouched.
scenarios.f9b_declined_survives_a_failed_prove = async () => {
const declinedRationale = 'intentional: rate limit is a product decision, not a bug'
const findings = [rec('OC-0001'), rec('OC-0002', { line: 800 })]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
return {
results: [
{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' },
{ id: 'OC-0002', outcome: 'declined', testPath: '', rationale: declinedRationale },
],
touchedPaths: [],
}
}
return {
committed: false,
sha: '',
redObserved: false,
greenObserved: true,
redOutput: '$ npx vitest run t/OC-0001.test.ts\nPASS (reverted, should have failed)',
greenOutput: '$ npx vitest run t/OC-0001.test.ts\nPASS (fixed)',
note: 'test passed with the fix reverted',
}
},
})
const byId = Object.fromEntries(result.results.map((r) => [r.id, r]))
assert.equal(byId['OC-0001'].outcome, 'blocked')
assert.match(byId['OC-0001'].rationale, /revert-proof/i)
assert.equal(byId['OC-0002'].outcome, 'declined')
assert.equal(byId['OC-0002'].rationale, declinedRationale)
}
// F10: the gate runs once, and only for the stacks the commits actually touched.
scenarios.f10_gate_targets_touched_stacks = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
let gatePrompt = ''
const { result, calls } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: 't.ts', rationale: '' })), touchedPaths: [] }
}
if (String(opts.label).startsWith('prove:'))
return {
committed: true,
sha: 'aaa1111',
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)',
note: '',
}
gatePrompt = prompt
return { passed: true, stacks: ['client', 'server'], output: 'ok' }
},
})
const gateCalls = calls.filter((c) => c.opts.label === 'gate')
assert.equal(gateCalls.length, 1, 'ci-check runs once, not per fix')
assert.equal(gateCalls[0].opts.model, 'sonnet')
assert.equal(gateCalls[0].opts.effort, 'medium')
assert.match(gatePrompt, /no-experimental-webstorage/, 'client gate command must be spelled out')
assert.match(gatePrompt, /go build -tags otel/, 'server gate must cover the tagged build variants')
assert.equal(result.gate.passed, true)
}
// F11: nothing committed means nothing to gate - skip it rather than burn 15 minutes.
scenarios.f11_no_commits_skips_gate = async () => {
const findings = [rec('OC-0001')]
const { result, calls } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'by design' }], touchedPaths: [] }
throw new Error(`no agent expected for label ${opts.label}`)
},
})
assert.ok(!calls.some((c) => c.opts.label === 'gate'))
assert.equal(result.gate, null)
}
// F12: a red gate does not rewrite history - commits stand, the failure is reported.
scenarios.f12_failing_gate_keeps_commits = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
if (String(opts.label).startsWith('prove:'))
return {
committed: true,
sha: 'bbb2222',
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)',
note: '',
}
return { passed: false, stacks: ['client'], output: 'tsc: 3 errors' }
},
})
assert.equal(result.commits.length, 1, 'a failing gate must not revert commits')
assert.equal(result.results[0].outcome, 'fixed')
assert.equal(result.gate.passed, false)
assert.match(result.gate.output, /tsc: 3 errors/)
}
// F12b: a truthy but wrongly-shaped gate response (no boolean `passed`, no `stacks` array) must
// still be treated as a failed gate - falling back to the computed stack list, but keeping the
// agent's own `output` string rather than overwriting it with the generic default message.
scenarios.f12b_malformed_gate_response_is_a_failed_gate = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
if (String(opts.label).startsWith('prove:'))
return {
committed: true,
sha: 'ccc3333',
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)',
note: '',
}
// wrongly shaped: no boolean `passed`, no `stacks` array - just a stray `output` string.
return { ok: true, output: 'ran partway: lint crashed before finishing' }
},
})
assert.equal(result.commits.length, 1, 'a malformed gate response must not lose an already-made commit')
assert.equal(result.gate.passed, false)
assert.ok(Array.isArray(result.gate.stacks), 'stacks must fall back to the computed list, not stay undefined')
assert.deepEqual(result.gate.stacks, ['client'])
assert.equal(
result.gate.output,
'ran partway: lint crashed before finishing',
"the agent's own output must be preserved, not replaced by the default message",
)
}
// F12c: the gate agent call itself throws. The .catch(() => null) guard must keep the rejection
// from escaping the workflow - same as Phase 3's prove agent - so commits already made survive
// and the gate is reported as failed rather than the run crashing before its final return.
scenarios.f12c_thrown_gate_agent_does_not_lose_commits = async () => {
const findings = [rec('OC-0001')]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] }
if (String(opts.label).startsWith('prove:'))
return {
committed: true,
sha: 'ddd4444',
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)',
note: '',
}
throw new Error('gate agent died')
},
})
assert.equal(result.commits.length, 1, 'a thrown gate agent must not lose an already-made commit')
assert.equal(result.commits[0].sha, 'ddd4444')
assert.equal(result.gate.passed, false)
assert.equal(result.results[0].outcome, 'fixed', 'a gate failure must not demote an already-committed result')
}
// F13: two clusters whose touchedPaths intersect (a shared root-cause file edited by both
// agents) must both be blocked before Phase 3 - neither may reach prove/commit, and the log
// must name both cluster files and the shared path.
scenarios.f13_intersecting_touched_paths_blocks_both_clusters = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
const { result, logs, calls } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (!String(opts.label).startsWith('fix:')) throw new Error(`only fix agents should run, got ${opts.label}`)
if (opts.label.includes('livekitE2EE'))
return {
results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }],
touchedPaths: ['Server/ws/shared_helper.go'],
}
return {
results: [{ id: 'OC-0003', outcome: 'fixed', testPath: 'Server/ws/hub_sweep_test.go', rationale: '' }],
touchedPaths: ['Server/ws/shared_helper.go'],
}
},
})
const byId = Object.fromEntries(result.results.map((r) => [r.id, r]))
assert.equal(byId['OC-0001'].outcome, 'blocked')
assert.equal(byId['OC-0003'].outcome, 'blocked')
assert.match(byId['OC-0001'].rationale, /Server\/ws\/shared_helper\.go/)
assert.match(byId['OC-0003'].rationale, /Server\/ws\/shared_helper\.go/)
assert.equal(result.commits.length, 0, 'neither cluster may commit once blocked by the overlap guard')
const joined = logs.join('\n')
assert.match(joined, /livekitE2EE\.ts/, 'log must name the first cluster file')
assert.match(joined, /hub_sweep\.go/, 'log must name the second cluster file')
assert.match(joined, /shared_helper\.go/, 'log must name the shared path')
assert.ok(!calls.some((c) => String(c.opts.label).startsWith('prove:')), 'blocked clusters must never reach prove')
}
// F14: two clusters with disjoint touchedPaths are unaffected by the guard and both commit.
scenarios.f14_disjoint_touched_paths_both_commit = async () => {
const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })]
const { result } = await run({
args: { findings },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
if (opts.label.includes('livekitE2EE'))
return {
results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }],
touchedPaths: ['Client/tauri-client/src/lib/otherHelper.ts'],
}
return {
results: [{ id: 'OC-0003', outcome: 'fixed', testPath: 'Server/ws/hub_sweep_test.go', rationale: '' }],
touchedPaths: ['Server/ws/other_helper.go'],
}
}
return {
committed: true,
sha: opts.label.includes('livekitE2EE') ? 'e2ee1111' : 'sweep222',
redObserved: true,
greenObserved: true,
redOutput: 'FAIL (reverted)',
greenOutput: 'PASS (fixed)',
note: '',
}
},
})
assert.equal(result.commits.length, 2, 'disjoint touchedPaths must not trip the overlap guard')
const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome]))
assert.equal(byId['OC-0001'], 'fixed')
assert.equal(byId['OC-0003'], 'fixed')
}
// F15: the prove prompt for a cluster whose agent reported extra touchedPaths names every one
// of those paths in both the revert (checkout) instruction and the staging (add) instruction -
// not just cluster.file.
scenarios.f15_prove_prompt_names_every_touched_path = async () => {
const findings = [rec('OC-0001')]
let provePromptText = ''
const { result } = await run({
args: { findings, branch: 'fix/test-touched' },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:'))
return {
results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }],
touchedPaths: ['Client/tauri-client/src/lib/sharedCrypto.ts'],
}
if (String(opts.label).startsWith('prove:')) {
provePromptText = prompt
return {
committed: true,
sha: 'aaa9999',
redObserved: true,
greenObserved: true,
redOutput: 'FAIL (reverted)',
greenOutput: 'PASS (fixed)',
note: '',
}
}
return { passed: true, stacks: ['client'], output: 'ok' }
},
})
assert.equal(result.commits.length, 1)
const checkoutLine = provePromptText.split('\n').find((l) => l.includes('git checkout HEAD --'))
assert.ok(checkoutLine, 'prove prompt must contain the checkout instruction')
assert.match(checkoutLine, /livekitE2EE\.ts/, 'checkout instruction must name the cluster file')
assert.match(checkoutLine, /sharedCrypto\.ts/, 'checkout instruction must also name the extra touched path')
const addLine = provePromptText.split('\n').find((l) => l.includes('git add'))
assert.ok(addLine, 'prove prompt must contain the staging instruction')
assert.match(addLine, /livekitE2EE\.ts/, 'add instruction must name the cluster file')
assert.match(addLine, /sharedCrypto\.ts/, 'add instruction must also name the extra touched path')
assert.match(provePromptText, /rev-parse --abbrev-ref HEAD/, 'prove prompt must guard the current branch')
assert.match(provePromptText, /fix\/test-touched/, 'branch guard must name the expected branch')
}
// ---------- circuit breaker ----------
// Four findings, one per file, so each becomes its own cluster.
const FOUR_FILES = [
rec('OC-0001'),
rec('OC-0002', { file: 'Server/ws/hub_sweep.go' }),
rec('OC-0003', { file: 'Client/tauri-client/src/lib/livekitSession.ts' }),
rec('OC-0004', { file: 'Client/tauri-client/src/components/VoiceWidget.ts' }),
]
// A fix stub that reports every id in its prompt as fixed. Ids go through a Set because rec()
// puts each id in both `id` and `title`, so the raw matchAll yields every id twice.
const fixAll = (prompt) => {
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: `t/${id}.ts`, rationale: '' })), touchedPaths: [] }
}
const PROVE_FAIL = {
committed: false,
sha: '',
redObserved: false,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nPASS t.ts (still passing with the fix reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts',
note: 'test did not exercise the bug',
}
const proveOk = (sha) => ({
committed: true,
sha,
redObserved: true,
greenObserved: true,
redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)',
greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)',
note: '',
})
// F16: three failed revert-proofs out of three attempts trips the breaker, and the fourth cluster
// is never handed to an agent. Load-bearing: this is the one that proves the loop actually stops.
scenarios.f16_breaker_trips_after_majority_prove_failures = async () => {
const { result, calls, logs } = await run({
args: { findings: FOUR_FILES },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) return fixAll(prompt)
if (String(opts.label).startsWith('prove:')) return PROVE_FAIL
return { passed: true, stacks: [], output: '' }
},
})
const proveCalls = calls.filter((c) => String(c.opts.label).startsWith('prove:'))
assert.equal(proveCalls.length, 3, 'breaker must stop the loop after minAttempts, not run all four')
assert.ok(result.breaker, 'breaker report must be present on the result')
assert.equal(result.breaker.trippedAt, 'prove')
assert.equal(result.breaker.attempted, 3)
assert.equal(result.breaker.failed, 3)
assert.equal(result.commits.length, 0)
// Every finding ends blocked, but the unreached one must say WHY it was never tried.
assert.ok(result.results.every((r) => r.outcome === 'blocked'), 'nothing may be left reported as fixed')
const unreached = result.results.filter((r) => /circuit breaker/i.test(r.rationale))
assert.equal(unreached.length, 1, 'exactly the one unreached finding carries the breaker rationale')
assert.match(unreached[0].rationale, /uncommitted/i, 'operator must be told the edits are still in the tree')
assert.ok(logs.some((l) => /CIRCUIT BREAKER/.test(l)), 'a trip must be announced in the log')
}
// F17: two failures out of two is 100%, but below minAttempts it is noise, not a signal.
scenarios.f17_breaker_holds_below_min_attempts = async () => {
const { result, calls } = await run({
args: { findings: FOUR_FILES.slice(0, 2) },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) return fixAll(prompt)
if (String(opts.label).startsWith('prove:')) return PROVE_FAIL
return { passed: true, stacks: [], output: '' }
},
})
assert.equal(calls.filter((c) => String(c.opts.label).startsWith('prove:')).length, 2, 'both clusters must be attempted')
assert.equal(result.breaker, null)
for (const r of result.results) {
assert.equal(r.outcome, 'blocked')
assert.match(r.rationale, /revert-proof failed/, 'rationale must be the real reason, not the breaker')
}
}
// F18: one failure in four is a bad cluster, not a bad run.
scenarios.f18_breaker_holds_under_threshold = async () => {
let n = 0
const { result } = await run({
args: { findings: FOUR_FILES },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) return fixAll(prompt)
if (String(opts.label).startsWith('prove:')) return ++n === 1 ? PROVE_FAIL : proveOk(`sha${n}`)
return { passed: true, stacks: [], output: '' }
},
})
assert.equal(result.breaker, null)
assert.equal(result.commits.length, 3, 'the three good clusters must still land')
}
// F19: the operator can turn the guard off entirely.
scenarios.f19_breaker_disabled_by_args = async () => {
const { result, calls } = await run({
args: { findings: FOUR_FILES, circuitBreaker: false },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) return fixAll(prompt)
if (String(opts.label).startsWith('prove:')) return PROVE_FAIL
return { passed: true, stacks: [], output: '' }
},
})
assert.equal(calls.filter((c) => String(c.opts.label).startsWith('prove:')).length, 4, 'all four must be attempted when disabled')
assert.equal(result.breaker, null)
}
// F20: when the FIX stage is what is failing, proving each of those costs a serial agent per
// cluster and cannot succeed. Load-bearing: this is the cheap early exit.
scenarios.f20_fix_stage_trip_skips_prove_entirely = async () => {
const { result, calls } = await run({
args: { findings: FOUR_FILES },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) {
const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))]
return {
results: ids.map((id) =>
id === 'OC-0001'
? { id, outcome: 'fixed', testPath: `t/${id}.ts`, rationale: '' }
: { id, outcome: 'blocked', testPath: '', rationale: 'could not run the test suite' },
),
touchedPaths: [],
}
}
return { passed: true, stacks: [], output: '' }
},
})
assert.ok(result.breaker, 'breaker report must be present')
assert.equal(result.breaker.trippedAt, 'fix')
assert.equal(
calls.filter((c) => String(c.opts.label).startsWith('prove:')).length,
0,
'a fix-stage trip must spend nothing on prove agents',
)
assert.equal(result.commits.length, 0)
assert.equal(result.gate, null, 'no commits means no gate')
}
// F21: a trip does not orphan whatever already landed - the operator needs to know if it is green.
scenarios.f21_trip_still_gates_existing_commits = async () => {
let n = 0
const { result, calls } = await run({
args: { findings: FOUR_FILES },
agentStub: (prompt, opts) => {
if (String(opts.label).startsWith('fix:')) return fixAll(prompt)
if (String(opts.label).startsWith('prove:')) return ++n === 1 ? proveOk('aaa1111') : PROVE_FAIL
return { passed: true, stacks: ['client'], output: 'all green' }
},
})
assert.ok(result.breaker, 'breaker report must be present')
assert.equal(result.breaker.trippedAt, 'prove')
assert.equal(result.commits.length, 1, 'the one proven cluster must survive the trip')
assert.equal(calls.filter((c) => c.opts.label === 'gate').length, 1, 'the gate must still run over what landed')
assert.equal(result.gate.passed, true)
}
// ---------- runner ----------
const only = process.argv[2]
for (const [name, fn] of Object.entries(scenarios)) {
if (only && !name.includes(only)) continue
try {
await fn()
} catch (e) {
console.error(`FAIL ${name}`)
throw e
}
console.log(`PASS ${name}`)
}
console.log('all scenarios pass')