From 6d964164b5c0f7420d6eb423ea08031fca79a534 Mon Sep 17 00:00:00 2001 From: J3vb <192430104+J3vb@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:33:19 +0200 Subject: [PATCH] feat(bughunt): add the fix pipeline, a findings ledger, and cross-run memory (#1361) * feat(bughunt): seed dedupe from the findings ledger via args.known * feat(bughunt): allow a scoped hunt via args.lenses * feat(bughunt): carry finder why/repro/evidence into confirmed records * feat(bughunt-fix): add workflow skeleton with per-file clustering * feat(bughunt-fix): add parallel per-file fix agents * fix(bughunt-fix): dedupe ids in the test stub, not in the merge loop * fix(bughunt-fix): drop foreign result ids loudly and assert the fix-prompt rules * feat(bughunt-fix): add serial revert-proof and per-cluster commits * fix(bughunt-fix): require real test output in the prove report * feat(bughunt-fix): add the ci-check gate and finalise the return shape * fix(bughunt-fix): harden the gate call and cover generated-code drift * docs(bughunt): add the bughunt-run operator skill * fix(bughunt-fix): guard cross-cluster edits, branch, and ledger handoff --- .claude/skills/bughunt-run/SKILL.md | 135 +++++ .claude/workflows/bughunt-fix.harness.mjs | 661 ++++++++++++++++++++++ .claude/workflows/bughunt-fix.js | 410 ++++++++++++++ .claude/workflows/bughunt.harness.mjs | 91 +++ .claude/workflows/bughunt.js | 25 +- 5 files changed, 1318 insertions(+), 4 deletions(-) create mode 100644 .claude/skills/bughunt-run/SKILL.md create mode 100644 .claude/workflows/bughunt-fix.harness.mjs create mode 100644 .claude/workflows/bughunt-fix.js diff --git a/.claude/skills/bughunt-run/SKILL.md b/.claude/skills/bughunt-run/SKILL.md new file mode 100644 index 00000000..82ef3958 --- /dev/null +++ b/.claude/skills/bughunt-run/SKILL.md @@ -0,0 +1,135 @@ +--- +name: bughunt-run +description: Run a bug hunt and turn its findings into committed fixes. Use when starting a hunt, resuming one, or fixing findings already in the ledger. Covers the ledger handoff between the bughunt and bughunt-fix workflows. +--- + +# Running the bughunt pipeline + +Two workflows with a human gate between them. The ledger at +`.superpowers/findings-ledger.json` is the interface. Both workflows are pure +functions of their `args` — **the session does all file I/O**, because workflow +scripts have no filesystem access. + +`.superpowers/` is gitignored. This repo is public and unfixed defects must never +reach a commit, an issue, or a PR body. + +## 1. Hunt + +Read the ledger and pass every record in as `known`, so the hunt does not +re-derive anything already found, fixed, declined, or refuted: + +``` +Workflow({ + name: "bughunt", + args: { + known: , + lenses: [ {key, prompt}, ... ], // optional: scope the hunt to one subsystem + maxRounds: 8, + dryThreshold: 2, + }, +}) +``` + +Omit `lenses` for a general hunt across the rotating families. + +Each lens object is `{key, prompt}`. `key` must match `^[a-z0-9-]+$` — +lowercase letters, digits, and hyphens only. Keys get interpolated into agent +labels of the form `r:hunt::`, and a key containing uppercase, +dots, or spaces breaks label parsing. A lens missing `key` or `prompt` is not +validated — it reaches the finder prompt as the literal string `undefined`, +silently degrading that lens instead of failing loudly. Check your lens +objects before passing them. + +When it returns, append each entry of `result.confirmed` to the ledger with +`status: "open"`, an id from `nextId`, and today's date. Bump `nextId`. The +incoming record carries a prose `fix` field (bughunt's suggested remedy) — +rename it to `suggestedFix` when appending, so the ledger's `fix` field starts +as `null` and is free for `bughunt-fix` to fill in with `{commit, test, +revertProof}` once something is actually fixed. Then: + +```bash +node .superpowers/render-ledger.mjs +``` + +## 2. Gate (human) + +Read `.superpowers/FINDINGS.md`. Mark anything you do not want fixed as +`declined` with a rationale — declined findings are fed back into the next hunt's +prompts and never re-reported. + +## 3. Fix + +``` +Workflow({ + name: "bughunt-fix", + args: { + findings: , + branch: "fix/bughunt-YYYY-MM-DD", + only: ["OC-0042"], // optional + maxSeverity: "medium", // optional + }, +}) +``` + +Create and check out the branch first — the workflow commits to whatever branch +is current and does not create one. + +When it returns, for each entry in `result.results`: + +- `fixed` → status `fixed`, `fix: {commit, test, revertProof: "self-reported"}` + using the matching `result.commits` entry — the prove agent's own report, not + yet independently checked (see step 4) +- `declined` → status `declined`, copy the rationale +- `blocked` → status `blocked` (not `open`), record the rationale; these failed + their revert-proof, tripped the cross-cluster overlap guard, or their agent + died, and want a human. Do NOT leave them `open` — `bughunt-fix` only picks up + `open` findings, so `open` would silently re-enter one of these into the next + fix run, exactly the retry loop the design deliberately excludes ("one human + look beats three agent attempts"). + +Check `result.gate`. A failed gate leaves the commits in place on the branch — +fix it yourself, do not re-run the workflow over it. + +## 4. Verify the fixes independently — REQUIRED + +The workflow's prove agent *self-reports* that each test went RED with the fix +reverted. Nothing inside the workflow can verify that: workflow scripts have no +filesystem access. You do. Run the independent proof over every commit the +workflow made: + +```bash +node .superpowers/verify-fixes.mjs ... +``` + +It reverts each commit's source files to the parent, runs that stack's tests, +and requires them to FAIL — then restores and requires them to PASS. This is the +only check in the pipeline no agent can fabricate. + +Any `FAIL ... VACUOUS TEST` means the fix was committed behind a test that +proves nothing. Revert that commit and set its findings back to `open`; do not +talk yourself into keeping it because the code change looks right. + +For every commit `verify-fixes.mjs` reports `PASS`, upgrade that commit's +findings' `fix.revertProof` from `"self-reported"` to `"pass"` — this +independent run is the only check in the pipeline no agent can fabricate, and +it is what earns the upgrade. A `FAIL` commit needs no further edit here — it +was already reverted and its findings set back to `open` above. + +Re-render. Before you review the branch and open the PR, inspect the working +tree: a blocked or declined cluster can leave its edits and any new failing +test it wrote sitting uncommitted. Discard what you don't want — a reflexive +`git add -A` would commit tests that describe unfixed defects into a public +repo. Then review the branch and open the PR by hand. The workflow never +pushes and never opens a PR. + +## Testing the workflows themselves + +```bash +node .claude/workflows/bughunt.harness.mjs +node .claude/workflows/bughunt-fix.harness.mjs +node .superpowers/render-ledger.mjs --selftest +node .superpowers/verify-fixes.mjs --selftest +``` + +All four run offline with zero API calls. Run them after any edit to the +relevant script. diff --git a/.claude/workflows/bughunt-fix.harness.mjs b/.claude/workflows/bughunt-fix.harness.mjs new file mode 100644 index 00000000..40946dc8 --- /dev/null +++ b/.claude/workflows/bughunt-fix.harness.mjs @@ -0,0 +1,661 @@ +// Offline harness for bughunt-fix.js - mirrors bughunt.harness.mjs: wraps the script body in an +// AsyncFunction with stubbed agent/parallel/pipeline/phase/log/args/budget. +// Run: node .claude/workflows/bughunt-fix.harness.mjs [nameFilter] +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import assert from 'node:assert/strict' + +const here = dirname(fileURLToPath(import.meta.url)) +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor + +export async function run({ agentStub, args = undefined, budget = undefined }) { + const src = readFileSync(join(here, 'bughunt-fix.js'), 'utf8') + const body = src.replace('export const meta', 'const meta') + const calls = [] + const logs = [] + const agent = async (prompt, opts = {}) => { + calls.push({ prompt, opts }) + return agentStub(prompt, opts) + } + const parallel = (thunks) => + Promise.all(thunks.map((t) => Promise.resolve().then(t).catch(() => null))) + const pipeline = (items, ...stages) => + Promise.all( + items.map(async (item, i) => { + let v = item + for (const stage of stages) { + try { + v = await stage(v, item, i) + } catch { + return null + } + } + return v + }), + ) + const log = (m) => logs.push(String(m)) + const phase = () => {} + const budgetImpl = budget || { total: null, spent: () => 0, remaining: () => Infinity } + const fn = new AsyncFunction('agent', 'parallel', 'pipeline', 'phase', 'log', 'args', 'budget', body) + const result = await fn(agent, parallel, pipeline, phase, log, args, budgetImpl) + return { result, calls, logs } +} + +// ---------- fixtures ---------- +export const rec = (id, over = {}) => ({ + id, + title: `bug ${id}`, + file: 'Client/tauri-client/src/lib/livekitE2EE.ts', + line: 100, + severity: 'high', + why: 'w', + repro: 'r', + evidence: 'e', + status: 'open', + found: '2026-08-09', + hunt: 'h', + lens: 'l', + fix: null, + ...over, +}) + +const scenarios = {} + +// F1: clustering groups by file; cluster count equals distinct file count. +scenarios.f1_clusters_by_file = async () => { + const findings = [ + rec('OC-0001'), + rec('OC-0002', { line: 800 }), + rec('OC-0003', { file: 'Server/ws/hub_sweep.go' }), + ] + const { result } = await run({ + args: { findings, branch: 'fix/test' }, + agentStub: () => { + throw new Error('no agent should run in phase 1 with the later phases unimplemented') + }, + }) + assert.equal(result.branch, 'fix/test') + assert.equal(result.clusters.length, 2) + const byFile = Object.fromEntries(result.clusters.map((c) => [c.file, c.ids])) + assert.deepEqual(byFile['Client/tauri-client/src/lib/livekitE2EE.ts'], ['OC-0001', 'OC-0002']) + assert.deepEqual(byFile['Server/ws/hub_sweep.go'], ['OC-0003']) +} + +// F1b: a backslash/case-only variant of the same path must not split into a second cluster - +// the same disjointness invariant the cross-cluster guard protects, at the grouping step. +scenarios.f1b_clustering_normalizes_backslashes = async () => { + const findings = [ + rec('OC-0001', { file: 'Server/ws/hub_sweep.go' }), + rec('OC-0002', { file: 'Server\\ws\\hub_sweep.go' }), + ] + const { result } = await run({ + args: { findings }, + agentStub: () => { + throw new Error('no agent should run in phase 1 with the later phases unimplemented') + }, + }) + assert.equal(result.clusters.length, 1, 'a backslash variant of the same path must merge into one cluster') + assert.deepEqual(result.clusters[0].ids.sort(), ['OC-0001', 'OC-0002']) + assert.equal(result.clusters[0].file, 'Server/ws/hub_sweep.go') +} + +// F2: only / maxSeverity / non-open status all exclude, and every exclusion is logged by id. +scenarios.f2_exclusions_are_announced = async () => { + const findings = [ + rec('OC-0001'), + rec('OC-0002', { severity: 'low' }), + rec('OC-0003', { status: 'fixed' }), + rec('OC-0004'), + ] + const { result, logs } = await run({ + args: { findings, only: ['OC-0001', 'OC-0002', 'OC-0003'], maxSeverity: 'medium' }, + agentStub: () => { + throw new Error('no agent expected') + }, + }) + const reasons = Object.fromEntries(result.excluded.map((e) => [e.id, e.reason])) + assert.equal(reasons['OC-0002'], 'below maxSeverity') + assert.equal(reasons['OC-0003'], 'status is fixed, not open') + assert.equal(reasons['OC-0004'], 'not in only') + assert.equal(result.clusters.length, 1) + assert.deepEqual(result.clusters[0].ids, ['OC-0001']) + const joined = logs.join('\n') + for (const id of ['OC-0002', 'OC-0003', 'OC-0004']) { + assert.match(joined, new RegExp(id), `exclusion of ${id} must be logged, not silent`) + } +} + +// F3: one sonnet/xhigh agent per cluster, and the prompt carries every finding in that file. +scenarios.f3_one_xhigh_agent_per_cluster = async () => { + const findings = [ + rec('OC-0001'), + rec('OC-0002', { line: 800 }), + rec('OC-0003', { file: 'Server/ws/hub_sweep.go' }), + ] + const { result, calls } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (!String(opts.label).startsWith('fix:')) throw new Error(`unexpected label ${opts.label}`) + const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))] + return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: `t/${id}.test.ts`, rationale: '' })), touchedPaths: [] } + }, + }) + const fixCalls = calls.filter((c) => String(c.opts.label).startsWith('fix:')) + assert.equal(fixCalls.length, 2, 'one agent per file cluster') + for (const c of fixCalls) { + assert.equal(c.opts.model, 'sonnet') + assert.equal(c.opts.effort, 'xhigh') + assert.equal(c.opts.phase, 'Fix') + } + const e2eeCall = fixCalls.find((c) => c.opts.label.includes('livekitE2EE')) + assert.match(e2eeCall.prompt, /OC-0001/) + assert.match(e2eeCall.prompt, /OC-0002/) + assert.ok(!e2eeCall.prompt.includes('OC-0003'), 'a cluster prompt must not leak another file\'s findings') + assert.match(e2eeCall.prompt, /write a test that fails/i, 'rule 1: test-first') + assert.match(e2eeCall.prompt, /weakening an assertion/i, 'rule 2: never weaken an assertion') + assert.match(e2eeCall.prompt, /grep every caller/i, 'rule 3: root cause, grep callers') + assert.match(e2eeCall.prompt, /touchedPaths/i, 'rule 4: shared-file edits must be listed in touchedPaths') + assert.match(e2eeCall.prompt, /one change that closes more than one/i, 'rule 5: one change closing several findings') + assert.match(e2eeCall.prompt, /do not run any git command/i, 'rule 6: no git') + assert.match(e2eeCall.prompt, /do not invent a fix/i, 'rule 7: declined with a rationale') + assert.match(e2eeCall.prompt, /mechanical reason/i, 'rule 8: blocked with a rationale') + assert.equal(result.results.length, 3) +} + +// F4: a dead fix agent marks only its own cluster; siblings still report. +scenarios.f4_dead_agent_does_not_poison_siblings = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (opts.label.includes('hub_sweep')) throw new Error('agent died') + if (String(opts.label).startsWith('prove:')) + return { + committed: true, + sha: 'aaa0000', + redObserved: true, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (reverted)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)', + note: '', + } + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + }, + }) + const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome])) + assert.equal(byId['OC-0001'], 'fixed') + assert.equal(byId['OC-0003'], 'blocked') + const reason = result.results.find((r) => r.id === 'OC-0003').rationale + assert.match(reason, /agent/i) +} + +// F5: a declined finding keeps its rationale and is not treated as fixed. +scenarios.f5_decline_propagates = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: () => ({ + results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'intended behaviour, locked by test X' }], + touchedPaths: [], + }), + }) + assert.equal(result.results[0].outcome, 'declined') + assert.equal(result.results[0].rationale, 'intended behaviour, locked by test X') +} + +// F5b: a foreign id (hallucinated, or copy-pasted from a different cluster) is dropped, not merged, +// and its id is announced in the logs rather than disappearing silently. +scenarios.f5b_foreign_id_is_dropped_and_announced = async () => { + const findings = [rec('OC-0001')] + const { result, logs } = await run({ + args: { findings }, + agentStub: () => ({ + results: [ + { id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }, + { id: 'OC-9999', outcome: 'fixed', testPath: 't2.ts', rationale: '' }, + ], + touchedPaths: [], + }), + }) + assert.deepEqual(result.results.map((r) => r.id), ['OC-0001']) + assert.match(logs.join('\n'), /OC-9999/, 'the dropped foreign id must be announced in the logs') +} + +// F6: prove agents run serially (never overlapping) and only for clusters that produced a fix. +scenarios.f6_prove_is_serial = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + let inFlight = 0 + let maxInFlight = 0 + const { result, calls } = await run({ + args: { findings }, + agentStub: async (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) { + const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))] + return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: `t/${id}.ts`, rationale: '' })), touchedPaths: [] } + } + inFlight++ + maxInFlight = Math.max(maxInFlight, inFlight) + await new Promise((r) => setTimeout(r, 5)) + inFlight-- + return { + committed: true, + sha: 'abc1234', + redObserved: true, + greenObserved: true, + redOutput: '$ go test ./ws/ -run TestSweep\nFAIL: reverted source', + greenOutput: '$ go test ./ws/ -run TestSweep\nPASS: fix restored', + note: '', + } + }, + }) + assert.equal(maxInFlight, 1, 'prove/commit must be serial - git index contention') + const proveCalls = calls.filter((c) => String(c.opts.label).startsWith('prove:')) + assert.equal(proveCalls.length, 2) + for (const c of proveCalls) { + assert.equal(c.opts.model, 'sonnet') + assert.equal(c.opts.effort, 'medium') + } + assert.equal(result.commits.length, 2) + assert.deepEqual(result.commits[0], { sha: 'abc1234', file: findings[0].file, ids: ['OC-0001'] }) +} + +// F7: a vacuous test (revert-proof does not go RED) is NOT committed and its findings go blocked. +scenarios.f7_vacuous_test_is_not_committed = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + return { + committed: false, + sha: '', + redObserved: false, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (reverted, should have failed)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)', + note: 'test passed with the fix reverted', + } + }, + }) + assert.equal(result.commits.length, 0, 'a cluster that failed its revert-proof must not be committed') + assert.equal(result.results[0].outcome, 'blocked') + assert.match(result.results[0].rationale, /revert-proof/i) +} + +// F7b: RED was observed but the restored fix does not go GREEN - also not committed. +scenarios.f7b_restored_fix_must_be_green = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + return { + committed: false, + sha: '', + redObserved: true, + greenObserved: false, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (reverted)', + greenOutput: '$ npx vitest run t.ts\nFAIL t.ts > OC-0001 (still failing after restore)', + note: 'still failing after restore', + } + }, + }) + assert.equal(result.commits.length, 0) + assert.equal(result.results[0].outcome, 'blocked') + assert.match(result.results[0].rationale, /after restoring the fix/i) +} + +// F8: a cluster with only declines is never sent to prove, and produces no commit. +scenarios.f8_declined_cluster_skips_prove = async () => { + const findings = [rec('OC-0001')] + const { result, calls } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'by design' }], touchedPaths: [] } + throw new Error('prove must not run for a cluster with no fixes') + }, + }) + assert.ok(!calls.some((c) => String(c.opts.label).startsWith('prove:'))) + assert.equal(result.commits.length, 0) + assert.equal(result.results[0].outcome, 'declined') +} + +// F9: one failing cluster does not stop its siblings from committing. +scenarios.f9_blocked_cluster_does_not_block_siblings = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) { + const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))] + return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: 't.ts', rationale: '' })), touchedPaths: [] } + } + if (opts.label.includes('livekitE2EE')) + return { + committed: false, + sha: '', + redObserved: false, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (reverted, should have failed)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts > OC-0001 (fixed)', + note: 'vacuous', + } + return { + committed: true, + sha: 'def5678', + redObserved: true, + greenObserved: true, + redOutput: '$ go test ./ws/ -run TestSweep\nFAIL: reverted source', + greenOutput: '$ go test ./ws/ -run TestSweep\nPASS: fix restored', + note: '', + } + }, + }) + assert.equal(result.commits.length, 1) + assert.equal(result.commits[0].sha, 'def5678') + const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome])) + assert.equal(byId['OC-0001'], 'blocked') + assert.equal(byId['OC-0003'], 'fixed') +} + +// F9b: a mixed cluster (one fixed + one declined) whose prove fails demotes only the fixed +// finding to blocked; the declined finding and its original rationale are left untouched. +scenarios.f9b_declined_survives_a_failed_prove = async () => { + const declinedRationale = 'intentional: rate limit is a product decision, not a bug' + const findings = [rec('OC-0001'), rec('OC-0002', { line: 800 })] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) { + return { + results: [ + { id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }, + { id: 'OC-0002', outcome: 'declined', testPath: '', rationale: declinedRationale }, + ], + touchedPaths: [], + } + } + return { + committed: false, + sha: '', + redObserved: false, + greenObserved: true, + redOutput: '$ npx vitest run t/OC-0001.test.ts\nPASS (reverted, should have failed)', + greenOutput: '$ npx vitest run t/OC-0001.test.ts\nPASS (fixed)', + note: 'test passed with the fix reverted', + } + }, + }) + const byId = Object.fromEntries(result.results.map((r) => [r.id, r])) + assert.equal(byId['OC-0001'].outcome, 'blocked') + assert.match(byId['OC-0001'].rationale, /revert-proof/i) + assert.equal(byId['OC-0002'].outcome, 'declined') + assert.equal(byId['OC-0002'].rationale, declinedRationale) +} + +// F10: the gate runs once, and only for the stacks the commits actually touched. +scenarios.f10_gate_targets_touched_stacks = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + let gatePrompt = '' + const { result, calls } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) { + const ids = [...new Set([...prompt.matchAll(/OC-\d{4}/g)].map((m) => m[0]))] + return { results: ids.map((id) => ({ id, outcome: 'fixed', testPath: 't.ts', rationale: '' })), touchedPaths: [] } + } + if (String(opts.label).startsWith('prove:')) + return { + committed: true, + sha: 'aaa1111', + redObserved: true, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)', + note: '', + } + gatePrompt = prompt + return { passed: true, stacks: ['client', 'server'], output: 'ok' } + }, + }) + const gateCalls = calls.filter((c) => c.opts.label === 'gate') + assert.equal(gateCalls.length, 1, 'ci-check runs once, not per fix') + assert.equal(gateCalls[0].opts.model, 'sonnet') + assert.equal(gateCalls[0].opts.effort, 'medium') + assert.match(gatePrompt, /no-experimental-webstorage/, 'client gate command must be spelled out') + assert.match(gatePrompt, /go build -tags otel/, 'server gate must cover the tagged build variants') + assert.equal(result.gate.passed, true) +} + +// F11: nothing committed means nothing to gate - skip it rather than burn 15 minutes. +scenarios.f11_no_commits_skips_gate = async () => { + const findings = [rec('OC-0001')] + const { result, calls } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'declined', testPath: '', rationale: 'by design' }], touchedPaths: [] } + throw new Error(`no agent expected for label ${opts.label}`) + }, + }) + assert.ok(!calls.some((c) => c.opts.label === 'gate')) + assert.equal(result.gate, null) +} + +// F12: a red gate does not rewrite history - commits stand, the failure is reported. +scenarios.f12_failing_gate_keeps_commits = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + if (String(opts.label).startsWith('prove:')) + return { + committed: true, + sha: 'bbb2222', + redObserved: true, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)', + note: '', + } + return { passed: false, stacks: ['client'], output: 'tsc: 3 errors' } + }, + }) + assert.equal(result.commits.length, 1, 'a failing gate must not revert commits') + assert.equal(result.results[0].outcome, 'fixed') + assert.equal(result.gate.passed, false) + assert.match(result.gate.output, /tsc: 3 errors/) +} + +// F12b: a truthy but wrongly-shaped gate response (no boolean `passed`, no `stacks` array) must +// still be treated as a failed gate - falling back to the computed stack list, but keeping the +// agent's own `output` string rather than overwriting it with the generic default message. +scenarios.f12b_malformed_gate_response_is_a_failed_gate = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + if (String(opts.label).startsWith('prove:')) + return { + committed: true, + sha: 'ccc3333', + redObserved: true, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)', + note: '', + } + // wrongly shaped: no boolean `passed`, no `stacks` array - just a stray `output` string. + return { ok: true, output: 'ran partway: lint crashed before finishing' } + }, + }) + assert.equal(result.commits.length, 1, 'a malformed gate response must not lose an already-made commit') + assert.equal(result.gate.passed, false) + assert.ok(Array.isArray(result.gate.stacks), 'stacks must fall back to the computed list, not stay undefined') + assert.deepEqual(result.gate.stacks, ['client']) + assert.equal( + result.gate.output, + 'ran partway: lint crashed before finishing', + "the agent's own output must be preserved, not replaced by the default message", + ) +} + +// F12c: the gate agent call itself throws. The .catch(() => null) guard must keep the rejection +// from escaping the workflow - same as Phase 3's prove agent - so commits already made survive +// and the gate is reported as failed rather than the run crashing before its final return. +scenarios.f12c_thrown_gate_agent_does_not_lose_commits = async () => { + const findings = [rec('OC-0001')] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't.ts', rationale: '' }], touchedPaths: [] } + if (String(opts.label).startsWith('prove:')) + return { + committed: true, + sha: 'ddd4444', + redObserved: true, + greenObserved: true, + redOutput: '$ npx vitest run t.ts\nFAIL t.ts (reverted)', + greenOutput: '$ npx vitest run t.ts\nPASS t.ts (fixed)', + note: '', + } + throw new Error('gate agent died') + }, + }) + assert.equal(result.commits.length, 1, 'a thrown gate agent must not lose an already-made commit') + assert.equal(result.commits[0].sha, 'ddd4444') + assert.equal(result.gate.passed, false) + assert.equal(result.results[0].outcome, 'fixed', 'a gate failure must not demote an already-committed result') +} + +// F13: two clusters whose touchedPaths intersect (a shared root-cause file edited by both +// agents) must both be blocked before Phase 3 - neither may reach prove/commit, and the log +// must name both cluster files and the shared path. +scenarios.f13_intersecting_touched_paths_blocks_both_clusters = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + const { result, logs, calls } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (!String(opts.label).startsWith('fix:')) throw new Error(`only fix agents should run, got ${opts.label}`) + if (opts.label.includes('livekitE2EE')) + return { + results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }], + touchedPaths: ['Server/ws/shared_helper.go'], + } + return { + results: [{ id: 'OC-0003', outcome: 'fixed', testPath: 'Server/ws/hub_sweep_test.go', rationale: '' }], + touchedPaths: ['Server/ws/shared_helper.go'], + } + }, + }) + const byId = Object.fromEntries(result.results.map((r) => [r.id, r])) + assert.equal(byId['OC-0001'].outcome, 'blocked') + assert.equal(byId['OC-0003'].outcome, 'blocked') + assert.match(byId['OC-0001'].rationale, /Server\/ws\/shared_helper\.go/) + assert.match(byId['OC-0003'].rationale, /Server\/ws\/shared_helper\.go/) + assert.equal(result.commits.length, 0, 'neither cluster may commit once blocked by the overlap guard') + const joined = logs.join('\n') + assert.match(joined, /livekitE2EE\.ts/, 'log must name the first cluster file') + assert.match(joined, /hub_sweep\.go/, 'log must name the second cluster file') + assert.match(joined, /shared_helper\.go/, 'log must name the shared path') + assert.ok(!calls.some((c) => String(c.opts.label).startsWith('prove:')), 'blocked clusters must never reach prove') +} + +// F14: two clusters with disjoint touchedPaths are unaffected by the guard and both commit. +scenarios.f14_disjoint_touched_paths_both_commit = async () => { + const findings = [rec('OC-0001'), rec('OC-0003', { file: 'Server/ws/hub_sweep.go' })] + const { result } = await run({ + args: { findings }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) { + if (opts.label.includes('livekitE2EE')) + return { + results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }], + touchedPaths: ['Client/tauri-client/src/lib/otherHelper.ts'], + } + return { + results: [{ id: 'OC-0003', outcome: 'fixed', testPath: 'Server/ws/hub_sweep_test.go', rationale: '' }], + touchedPaths: ['Server/ws/other_helper.go'], + } + } + return { + committed: true, + sha: opts.label.includes('livekitE2EE') ? 'e2ee1111' : 'sweep222', + redObserved: true, + greenObserved: true, + redOutput: 'FAIL (reverted)', + greenOutput: 'PASS (fixed)', + note: '', + } + }, + }) + assert.equal(result.commits.length, 2, 'disjoint touchedPaths must not trip the overlap guard') + const byId = Object.fromEntries(result.results.map((r) => [r.id, r.outcome])) + assert.equal(byId['OC-0001'], 'fixed') + assert.equal(byId['OC-0003'], 'fixed') +} + +// F15: the prove prompt for a cluster whose agent reported extra touchedPaths names every one +// of those paths in both the revert (checkout) instruction and the staging (add) instruction - +// not just cluster.file. +scenarios.f15_prove_prompt_names_every_touched_path = async () => { + const findings = [rec('OC-0001')] + let provePromptText = '' + const { result } = await run({ + args: { findings, branch: 'fix/test-touched' }, + agentStub: (prompt, opts) => { + if (String(opts.label).startsWith('fix:')) + return { + results: [{ id: 'OC-0001', outcome: 'fixed', testPath: 't/OC-0001.test.ts', rationale: '' }], + touchedPaths: ['Client/tauri-client/src/lib/sharedCrypto.ts'], + } + if (String(opts.label).startsWith('prove:')) { + provePromptText = prompt + return { + committed: true, + sha: 'aaa9999', + redObserved: true, + greenObserved: true, + redOutput: 'FAIL (reverted)', + greenOutput: 'PASS (fixed)', + note: '', + } + } + return { passed: true, stacks: ['client'], output: 'ok' } + }, + }) + assert.equal(result.commits.length, 1) + const checkoutLine = provePromptText.split('\n').find((l) => l.includes('git checkout HEAD --')) + assert.ok(checkoutLine, 'prove prompt must contain the checkout instruction') + assert.match(checkoutLine, /livekitE2EE\.ts/, 'checkout instruction must name the cluster file') + assert.match(checkoutLine, /sharedCrypto\.ts/, 'checkout instruction must also name the extra touched path') + const addLine = provePromptText.split('\n').find((l) => l.includes('git add')) + assert.ok(addLine, 'prove prompt must contain the staging instruction') + assert.match(addLine, /livekitE2EE\.ts/, 'add instruction must name the cluster file') + assert.match(addLine, /sharedCrypto\.ts/, 'add instruction must also name the extra touched path') + assert.match(provePromptText, /rev-parse --abbrev-ref HEAD/, 'prove prompt must guard the current branch') + assert.match(provePromptText, /fix\/test-touched/, 'branch guard must name the expected branch') +} + +// ---------- runner ---------- +const only = process.argv[2] +for (const [name, fn] of Object.entries(scenarios)) { + if (only && !name.includes(only)) continue + try { + await fn() + } catch (e) { + console.error(`FAIL ${name}`) + throw e + } + console.log(`PASS ${name}`) +} +console.log('all scenarios pass') diff --git a/.claude/workflows/bughunt-fix.js b/.claude/workflows/bughunt-fix.js new file mode 100644 index 00000000..9fadd141 --- /dev/null +++ b/.claude/workflows/bughunt-fix.js @@ -0,0 +1,410 @@ +export const meta = { + name: 'bughunt-fix', + description: 'Fix open ledger findings test-first: per-file agents, mechanical revert-proof, serial commits, one ci-check gate', + whenToUse: 'After a bughunt run has been written to the findings ledger and a human has skimmed it. Consumes open findings, produces commits on a branch. Never opens a PR.', + phases: [ + { title: 'Plan', detail: 'cluster open findings by file' }, + { title: 'Fix', detail: 'sonnet/xhigh: one agent per file, test-first, no git' }, + { title: 'Prove', detail: 'sonnet: serial revert-proof then commit per cluster' }, + { title: 'Gate', detail: 'sonnet: ci-check for the touched stacks, once' }, + ], +} + +// args has been observed arriving JSON-stringified; coerce it the same way bughunt.js does. +const ARGS = (() => { + if (typeof args === 'string') { + try { + return JSON.parse(args) || {} + } catch { + return {} + } + } + return args || {} +})() + +const SEV_RANK = { critical: 0, high: 1, medium: 2, low: 3 } +const BRANCH = ARGS.branch || 'fix/bughunt' +const ONLY = Array.isArray(ARGS.only) && ARGS.only.length ? new Set(ARGS.only) : null +const MAX_SEVERITY = ARGS.maxSeverity || 'low' +const ALL = Array.isArray(ARGS.findings) ? ARGS.findings : [] + +// ---------- phase 1: plan ---------- +phase('Plan') + +const excluded = [] +const selected = [] +for (const f of ALL) { + if (f.status && f.status !== 'open') { + excluded.push({ id: f.id, reason: `status is ${f.status}, not open` }) + } else if (ONLY && !ONLY.has(f.id)) { + excluded.push({ id: f.id, reason: 'not in only' }) + } else if ((SEV_RANK[f.severity] ?? 3) > (SEV_RANK[MAX_SEVERITY] ?? 3)) { + excluded.push({ id: f.id, reason: 'below maxSeverity' }) + } else { + selected.push(f) + } +} + +// Group by file. One agent per file is what removes merge conflicts (clusters are disjoint) +// and what makes a root-cause fix possible - the agent sees every defect in the file at once. +// Normalize the grouping key (backslashes -> forward slashes) so a path reported with the +// "wrong" separator does not silently split one real file into two clusters. +const byFile = new Map() +for (const f of selected) { + const key = String(f.file).replace(/\\/g, '/') + if (!byFile.has(key)) byFile.set(key, []) + byFile.get(key).push(f) +} +const clusters = [...byFile.entries()].map(([file, findings]) => ({ + file, + ids: findings.map((f) => f.id), + findings, +})) + +log(`plan: ${selected.length} finding(s) in ${clusters.length} file cluster(s) on ${BRANCH}`) +for (const c of clusters) log(` ${c.file}: ${c.ids.join(', ')}`) +// Announce every exclusion by id. Silent truncation reads as "covered everything" when it did not. +for (const e of excluded) log(` excluded ${e.id}: ${e.reason}`) + +const publicClusters = clusters.map((c) => ({ file: c.file, ids: c.ids })) + +// ---------- schemas ---------- +const FIX_RESULTS = { + type: 'object', + required: ['results', 'touchedPaths'], + properties: { + results: { + type: 'array', + items: { + type: 'object', + required: ['id', 'outcome', 'testPath', 'rationale'], + properties: { + id: { type: 'string', description: 'the ledger id, e.g. OC-0042' }, + outcome: { type: 'string', enum: ['fixed', 'declined', 'blocked'] }, + testPath: { type: 'string', description: 'repo-relative path of the test that pins this finding; empty if not fixed' }, + rationale: { type: 'string', description: 'required for declined and blocked; empty for fixed' }, + }, + }, + }, + touchedPaths: { + type: 'array', + items: { type: 'string' }, + description: + 'every non-test SOURCE file this agent modified while working this cluster, repo-relative, forward ' + + 'slashes - including cluster.file itself if it was touched, and any shared file outside the cluster ' + + 'the root-cause fix required. Test files belong in testPath (per finding), not here.', + }, + }, +} + +// ---------- phase 2: fix ---------- +phase('Fix') + +function fixPrompt(cluster) { + return ( + `You are fixing confirmed bugs in ONE file of the OwnCord repo (D:/Local-Lab/Repos/OwnCord).\n` + + `Your file: ${cluster.file}\n\n` + + `You own this file for this run. No other agent will touch it, so fix ALL of the findings below ` + + `together rather than one at a time.\n\n` + + `RULES\n` + + ` 1. Test first. For each finding, write a test that FAILS against the current code before you ` + + `change anything, and run it to watch it fail. A test that passes before the fix does not pin the ` + + `bug and will be rejected mechanically later.\n` + + ` 2. Never make a failing test pass by weakening an assertion. The existing suite is green and must ` + + `stay green on its current assertions.\n` + + ` 3. Fix the ROOT CAUSE. Grep every caller of the function you are about to change. One guard in a ` + + `shared function beats a guard in every caller, and patching only the path a finding names leaves its ` + + `siblings broken.\n` + + ` 4. You may edit a shared file outside your own cluster (${cluster.file}) when that is where the ` + + `root cause lives. If you do, you MUST list every SOURCE file you modified - including this cluster's ` + + `own file - in touchedPaths, repo-relative with forward slashes. Test files belong in testPath, not ` + + `touchedPaths.\n` + + ` 5. Because several findings share this file, look for one change that closes more than one of them ` + + `before writing separate patches.\n` + + ` 6. DO NOT run any git command. No add, no commit, no stash, no checkout. Other agents are working ` + + `in this same working tree and git operations collide on the index lock. Leave your changes in the ` + + `working tree; a later serial phase commits them.\n` + + ` 7. If a finding is wrong, or the correct fix is a deliberate product decision you should not make ` + + `alone, return outcome "declined" with a rationale. Do not invent a fix you do not believe in.\n` + + ` 8. If you cannot fix it for a mechanical reason (missing fixture, unclear repro), return "blocked" ` + + `with a rationale.\n\n` + + `Client tests run from Client/tauri-client with:\n` + + ` NODE_OPTIONS=--no-experimental-webstorage npx vitest run \n` + + `Server tests run from Server with:\n` + + ` go test .// -run \n\n` + + `Return one result per finding id, all ${cluster.ids.length} of them, plus touchedPaths.\n\n` + + `--- FINDINGS IN ${cluster.file} ---\n${JSON.stringify(cluster.findings, null, 2)}` + ) +} + +const fixOutcomes = await parallel( + clusters.map((cluster) => () => + agent(fixPrompt(cluster), { + label: `fix:${cluster.file}`, + phase: 'Fix', + model: 'sonnet', + effort: 'xhigh', + schema: FIX_RESULTS, + }).then((r) => ({ + cluster, + results: (r && r.results) || [], + touchedPaths: r && Array.isArray(r.touchedPaths) ? r.touchedPaths.filter((p) => typeof p === 'string' && p) : [], + })), + ), +) + +// A null slot means the agent died or threw. Its findings are blocked, its siblings are unaffected. +const fixed = [] +for (let i = 0; i < clusters.length; i++) { + const cluster = clusters[i] + const outcome = fixOutcomes[i] + if (!outcome) { + log(`fix ${cluster.file}: agent failed - ${cluster.ids.length} finding(s) blocked`) + fixed.push({ + cluster, + results: cluster.ids.map((id) => ({ id, outcome: 'blocked', testPath: '', rationale: 'fix agent failed or returned nothing' })), + touchedPaths: [], + union: [cluster.file], + }) + continue + } + // A hallucinated id, or one copy-pasted from a different cluster, must not merge in silently. + const ownIds = new Set(cluster.ids) + const ownResults = outcome.results.filter((r) => ownIds.has(r.id)) + const foreignResults = outcome.results.filter((r) => !ownIds.has(r.id)) + if (foreignResults.length) { + log(`fix ${cluster.file}: dropped ${foreignResults.length} result(s) for id(s) not in this cluster - ${foreignResults.map((r) => r.id).join(', ')}`) + } + // An agent that skipped a finding entirely leaves it blocked rather than silently dropped. + const reported = new Set(ownResults.map((r) => r.id)) + const missing = cluster.ids + .filter((id) => !reported.has(id)) + .map((id) => ({ id, outcome: 'blocked', testPath: '', rationale: 'fix agent returned no result for this finding' })) + if (missing.length) log(`fix ${cluster.file}: ${missing.length} finding(s) unreported by the agent - blocked`) + fixed.push({ + cluster, + results: [...ownResults, ...missing], + touchedPaths: outcome.touchedPaths, + union: [...new Set([cluster.file, ...outcome.touchedPaths])], + }) +} + +// ---------- phase 2.5: cross-cluster overlap guard ---------- +// Rule 4 above lets an agent fix a shared root cause outside its own file - the per-file +// disjointness the whole design leans on (no merge conflicts, a clean revert per cluster) no +// longer holds automatically once that happens. If two clusters' agents both touched the same +// path, Phase 3 cannot safely revert/stage per-cluster: one cluster's revert could silently +// undo the other's real fix (a misattributed VACUOUS TEST) or a real change could never get +// staged at all. Block both clusters rather than guess which one "owns" the shared file. +for (let i = 0; i < fixed.length; i++) { + for (let j = i + 1; j < fixed.length; j++) { + const a = fixed[i] + const b = fixed[j] + const shared = a.union.filter((p) => b.union.includes(p)) + if (!shared.length) continue + log(`blocked: ${a.cluster.file} and ${b.cluster.file} both touch ${shared.join(', ')} - both clusters blocked`) + for (const [entry, other] of [[a, b], [b, a]]) { + for (const r of entry.results) { + if (r.outcome === 'fixed') { + r.outcome = 'blocked' + r.rationale = `cross-cluster edit: shares ${shared.join(', ')} with ${other.cluster.file} - needs a human` + } + } + } + } +} + +const allResults = fixed.flatMap((f) => f.results) +log(`fix: ${allResults.filter((r) => r.outcome === 'fixed').length} fixed, ` + + `${allResults.filter((r) => r.outcome === 'declined').length} declined, ` + + `${allResults.filter((r) => r.outcome === 'blocked').length} blocked`) + +// ---------- phase 3: prove + commit ---------- +phase('Prove') + +const PROVE_RESULT = { + type: 'object', + required: ['committed', 'sha', 'redObserved', 'greenObserved', 'redOutput', 'greenOutput', 'note'], + properties: { + committed: { type: 'boolean' }, + sha: { type: 'string', description: 'short sha of the commit, empty when not committed' }, + redObserved: { type: 'boolean', description: 'did the tests FAIL with the source reverted' }, + greenObserved: { type: 'boolean', description: 'did the tests PASS with the fix restored' }, + redOutput: { + type: 'string', + description: + 'the ACTUAL output of the test run performed with the source reverted (step 4), including the ' + + 'command that was run. This run must FAIL. Paste the real captured output verbatim - not a ' + + 'summary, not a paraphrase.', + }, + greenOutput: { + type: 'string', + description: + 'the ACTUAL output of the test run performed after the fix was restored (step 6), including the ' + + 'command that was run. This run must PASS. Paste the real captured output verbatim - not a ' + + 'summary, not a paraphrase.', + }, + note: { type: 'string', description: 'why it was not committed, empty on success' }, + }, +} + +function provePrompt(cluster, fixedIds, testPaths, sourcePaths) { + return ( + `You are proving and committing ONE cluster of fixes in the OwnCord repo ` + + `(D:/Local-Lab/Repos/OwnCord), on branch ${BRANCH}.\n\n` + + `Source file(s): ${sourcePaths.join(', ')}\n` + + `Findings fixed here: ${fixedIds.join(', ')}\n` + + `Test files written: ${testPaths.join(', ') || '(none reported)'}\n\n` + + `You are running SERIALLY. No other agent is touching git right now, so you may use git freely.\n\n` + + `Do exactly this, in order:\n` + + ` 1. Run: git rev-parse --abbrev-ref HEAD\n` + + ` It MUST print exactly "${BRANCH}". If it does not, DO NOT touch git any further: set ` + + `committed=false, explain in note which branch you actually found, and STOP. Committing to the wrong ` + + `branch (e.g. main, because the operator forgot to create/checkout ${BRANCH} first) is not recoverable ` + + `by this agent.\n` + + ` 2. Copy the current (fixed) contents of ALL source file(s) listed above to a scratch location ` + + `outside the repo.\n` + + ` 3. Run: git checkout HEAD -- ${sourcePaths.join(' ')}\n` + + ` Revert every source path listed above, and nothing else. Do NOT revert or delete the test ` + + `files - a brand-new test file is untracked and this leaves it alone, and a new case in an existing ` + + `test file is a modification to a path you did not name, so it survives too. Either way the new ` + + `assertions are present while the fix is gone.\n` + + ` 4. Run the tests listed above. They MUST fail. Set redObserved accordingly. Capture the ACTUAL ` + + `output of this run, including the command you ran, and return it verbatim in redOutput - not a ` + + `summary, not a paraphrase.\n` + + ` If they PASS, the tests do not pin the bug - they are vacuous. Restore the fixed source from ` + + `scratch, set committed=false, explain in note, and STOP. Do not commit. Do not try to repair the ` + + `test yourself.\n` + + ` 5. Restore the fixed source file(s) from your scratch copy.\n` + + ` 6. Run the tests again. They MUST pass. Set greenObserved accordingly. Capture the ACTUAL output ` + + `of this run, including the command you ran, and return it verbatim in greenOutput - not a summary, ` + + `not a paraphrase. If they do not pass, set committed=false, explain in note, and STOP.\n` + + ` 7. Stage ALL source file(s) listed above (git add ${sourcePaths.join(' ')}) AND the test files, ` + + `then commit with subject:\n` + + ` fix(): ${fixedIds.length} defect(s) (${fixedIds.join(', ')})\n` + + ` Use a conventional-commit area matching the file (voice, ws, client, identity...). Do not add a ` + + `Co-Authored-By trailer.\n` + + ` 8. Return the short sha.\n\n` + + `Client tests run from Client/tauri-client with:\n` + + ` NODE_OPTIONS=--no-experimental-webstorage npx vitest run \n` + + `Server tests run from Server with:\n` + + ` go test .// -run ` + ) +} + +const commits = [] +// Serial on purpose: parallel git commands collide on .git/index.lock. +for (const { cluster, results, union } of fixed) { + const fixedHere = results.filter((r) => r.outcome === 'fixed') + if (!fixedHere.length) { + log(`prove ${cluster.file}: no fixes to prove - skipped`) + continue + } + const ids = fixedHere.map((r) => r.id) + const testPaths = [...new Set(fixedHere.map((r) => r.testPath).filter(Boolean))] + // A dead/thrown prove agent must not take down the sibling clusters still waiting in this + // serial loop - same "one blocked cluster does not poison the rest" rule Phase 2 gets from + // parallel()'s catch. Fold it into a null result so the ok/why logic below handles it uniformly. + const p = await agent(provePrompt(cluster, ids, testPaths, union), { + label: `prove:${cluster.file}`, + phase: 'Prove', + model: 'sonnet', + effort: 'medium', + schema: PROVE_RESULT, + }).catch(() => null) + + const ok = p && p.committed && p.redObserved && p.greenObserved && p.sha + if (!ok) { + const why = !p + ? 'prove agent failed' + : !p.redObserved + ? `revert-proof failed: tests still passed with the fix reverted (${p.note || 'no note'})` + : !p.greenObserved + ? `tests did not pass after restoring the fix (${p.note || 'no note'})` + : `not committed (${p.note || 'no note'})` + log(`prove ${cluster.file}: ${why} - ${ids.length} finding(s) blocked`) + for (const r of results) { + if (r.outcome === 'fixed') { + r.outcome = 'blocked' + r.rationale = why + } + } + continue + } + commits.push({ sha: p.sha, file: cluster.file, ids }) + log(`prove ${cluster.file}: committed ${p.sha} (${ids.join(', ')})`) +} + +// ---------- phase 4: gate ---------- +const GATE_RESULT = { + type: 'object', + required: ['passed', 'stacks', 'output'], + properties: { + passed: { type: 'boolean' }, + stacks: { type: 'array', items: { type: 'string' } }, + output: { type: 'string', description: 'the failing command and its output, or a short ok summary' }, + }, +} + +function stacksFor(files) { + const s = new Set() + for (const f of files) { + if (f.startsWith('Server/')) s.add('server') + else if (f.startsWith('Client/tauri-client/src-tauri/')) s.add('rust') + else if (f.startsWith('Client/')) s.add('client') + } + return [...s] +} + +const GATE_COMMANDS = { + client: + `From Client/tauri-client:\n` + + ` NODE_OPTIONS=--no-experimental-webstorage npm test\n` + + ` npm run typecheck\n` + + ` npm run lint\n` + + ` npm run format:check`, + server: + `From Server:\n` + + ` go build ./... && go build -tags otel ./... && go build -tags wazero ./... && go build -tags otel,wazero ./...\n` + + ` go vet ./...\n` + + ` go test -race ./...\n` + + ` go test -tags deadlock -count=1 ./ws/\n` + + ` golangci-lint run\n` + + ` make sqlc-verify protocol-verify # generated output must not be stale. If make is not on PATH, ` + + `run the equivalent commands directly instead: ` + + `"sqlc generate && git diff --exit-code db/dbgen" and ` + + `"go run ./scripts/genprotocol && git diff --exit-code ws/message_types.go ../Client/tauri-client/src/lib/protocolTypes.ts" ` + + `- a non-empty diff in either means generated code is stale and the gate fails`, + rust: + `From Client/tauri-client/src-tauri:\n` + + ` cargo test\n` + + ` cargo clippy --all-targets -- -D warnings`, +} + +let gate = null +if (commits.length) { + phase('Gate') + const stacks = stacksFor(commits.map((c) => c.file)) + gate = await agent( + `Run the OwnCord CI gates locally for the stacks touched by this fix run, on branch ${BRANCH}.\n\n` + + `This runs ONCE for the whole run - a full gate per fix would take longer than the fixing did.\n\n` + + `Touched stacks: ${stacks.join(', ')}\n\n` + + stacks.map((s) => GATE_COMMANDS[s]).join('\n\n') + + `\n\nRun every command for every touched stack. Report passed=false if ANY of them fails, and put ` + + `the failing command plus the relevant output in "output". Do NOT fix anything, do NOT amend or ` + + `revert any commit, and do NOT push. Reporting the failure accurately is the whole job.\n\n` + + `Known false alarm: a windows -race failure inside ws whose stack mentions runtime.scanstack or ` + + `runtime.(*unwinder).next is a Go 1.26.5 runtime GC fault, not a real failure - rerun that package ` + + `once before reporting it.`, + { label: 'gate', phase: 'Gate', model: 'sonnet', effort: 'medium', schema: GATE_RESULT }, + ).catch(() => null) + // A malformed/missing report (dead agent, or a schema the caller didn't honor) is treated as a + // failed gate, same as the null-check pattern in phases 2 and 3 - never crash on shape here. + if (!gate || typeof gate.passed !== 'boolean' || !Array.isArray(gate.stacks)) + gate = { passed: false, stacks, output: (gate && gate.output) || 'gate agent failed to report' } + log(`gate: ${gate.passed ? 'PASS' : 'FAIL'} (${gate.stacks.join(', ')})`) +} else { + log('gate: nothing committed - skipped') +} + +return { branch: BRANCH, clusters: publicClusters, excluded, commits, results: allResults, gate } diff --git a/.claude/workflows/bughunt.harness.mjs b/.claude/workflows/bughunt.harness.mjs index 787477e2..244d4cfe 100644 --- a/.claude/workflows/bughunt.harness.mjs +++ b/.claude/workflows/bughunt.harness.mjs @@ -460,6 +460,97 @@ scenarios.s11_drifted_verdict = async () => { assert.equal(result.converged, true) } +// S-known: a finding already in the ledger is suppressed - never verified, never re-confirmed, +// and its text appears in the finder prompt so the model does not spend effort re-deriving it. +scenarios.s_known_ledger_suppresses = async () => { + const known = [ + { file: 'Server/ws/hub.go', line: 140, title: 'distinct bug alpha1 omega1', status: 'declined' }, + ] + const { result, calls } = await run({ + args: { known, maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ + hunt: (round, key, model) => + round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [finding(1)] } : none, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const huntPrompts = calls.filter((c) => /:hunt:/.test(c.opts.label || '')).map((c) => c.prompt) + assert.ok(huntPrompts.length > 0, 'expected at least one finder call') + assert.match(huntPrompts[0], /KNOWN FINDINGS/, 'ledger entries must reach the finder prompt') + assert.match(huntPrompts[0], /\[declined\] distinct bug alpha1 omega1/) + assert.ok( + !calls.some((c) => /:verify:/.test(c.opts.label || '')), + 'a ledger-known candidate must not reach verification', + ) + assert.equal(result.confirmed.length, 0) +} + +// S-lenses: args.lenses replaces the round-1 family entirely, and the round label reflects it. +scenarios.s_custom_lenses = async () => { + const lenses = [ + { key: 'voice-e2ee-keyholder', prompt: 'Hunt the key-holder election.' }, + { key: 'voice-e2ee-rotation', prompt: 'Hunt the rotation paths.' }, + ] + const { result, calls } = await run({ + args: { lenses, maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ hunt: () => none, verify: (r, k, c) => confirmAll(c) }), + }) + const keys = calls + .map((c) => /^r1:hunt:([a-z0-9-]+):(opus|sonnet)$/.exec(c.opts.label || '')) + .filter(Boolean) + .map((m) => m[1]) + assert.deepEqual([...new Set(keys)].sort(), ['voice-e2ee-keyholder', 'voice-e2ee-rotation']) + assert.ok(!keys.includes('ws-hub'), 'the default surface family must not run when lenses are supplied') + assert.equal(result.rounds[0].family, 'custom') + assert.equal(result.rounds[0].lenses, 2) +} + +// S-lenses-default: omitting args.lenses leaves the rotation untouched. +scenarios.s_custom_lenses_absent = async () => { + const { result } = await run({ + args: { maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ hunt: () => none, verify: (r, k, c) => confirmAll(c) }), + }) + assert.equal(result.rounds[0].family, 'surfaces') +} + +// S-ledger-fields: a confirmed record must carry finder detail (why/repro/evidence) as well as +// verifier detail (severity/fix), because the ledger needs both. +scenarios.s_confirmed_carries_finder_detail = async () => { + const cand = { + title: 'distinct bug alpha1 omega1', + file: 'Server/ws/hub.go', + line: 140, + severity: 'low', + why: 'WHY_TEXT', + repro: 'REPRO_TEXT', + evidence: 'EVIDENCE_TEXT', + } + const { result } = await run({ + args: { maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ + hunt: (round, key, model) => + round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [cand] } : none, + verify: (round, key, cands) => ({ + verdicts: cands.map((c) => ({ + title: c.title, file: c.file, line: c.line, + refuted: false, reason: 'confirmed', confidence: 'high', + severity: 'high', fix: 'FIX_TEXT', + })), + }), + }), + }) + assert.equal(result.confirmed.length, 1) + const r = result.confirmed[0] + assert.equal(r.why, 'WHY_TEXT') + assert.equal(r.repro, 'REPRO_TEXT') + assert.equal(r.evidence, 'EVIDENCE_TEXT') + assert.equal(r.severity, 'high', 'verifier severity must win over the finder rating') + assert.equal(r.fix, 'FIX_TEXT') + assert.equal(r.lens, 'ws-hub') + assert.equal(r.round, 1) +} + // ---------- runner ---------- const only = process.argv[2] for (const [name, fn] of Object.entries(scenarios)) { diff --git a/.claude/workflows/bughunt.js b/.claude/workflows/bughunt.js index 6690f758..9d2f6cfb 100644 --- a/.claude/workflows/bughunt.js +++ b/.claude/workflows/bughunt.js @@ -18,12 +18,15 @@ const ARGS = (() => { })() const MAX_ROUNDS = ARGS.maxRounds || 8 const DRY_THRESHOLD = ARGS.dryThreshold || 2 +// A scoped hunt (args.lenses) replaces the round-1 family outright; later rounds still go +// adaptive, so hotspot and fresh-eyes coverage - and therefore convergence - still work. +const CUSTOM_LENSES = Array.isArray(ARGS.lenses) && ARGS.lenses.length ? ARGS.lenses : null // ponytail: rough floor for one round (up to 12 high-effort finders + verifiers); tune after live runs const ROUND_BUDGET_FLOOR = 150000 // The args channel has already been observed delivering something the script // could not read; an unnoticed fallback here is an 8x cost surprise, so say out // loud what the run is actually going to do. -log(`config: maxRounds=${MAX_ROUNDS} dryThreshold=${DRY_THRESHOLD}`) +log(`config: maxRounds=${MAX_ROUNDS} dryThreshold=${DRY_THRESHOLD}${CUSTOM_LENSES ? ` lenses=custom(${CUSTOM_LENSES.length})` : ''}`) // ---------- schemas: copied VERBATIM from the current bughunt.js ---------- const FINDINGS = { @@ -268,12 +271,14 @@ const FLOW_LENSES = [ ] function lensesForRound(round) { + if (CUSTOM_LENSES) return round === 1 ? CUSTOM_LENSES : buildAdaptiveLenses() if (round === 1) return SURFACE_LENSES if (round === 2) return BUGCLASS_LENSES if (round === 3) return FLOW_LENSES return buildAdaptiveLenses() } function familyName(round) { + if (CUSTOM_LENSES) return round === 1 ? 'custom' : 'adaptive' return ['surfaces', 'bug-classes', 'flows'][round - 1] || 'adaptive' } function clusterOf(file) { @@ -403,7 +408,16 @@ const churnFiles = String(recon[0] || '') log('Recon complete - starting converging rounds') // ---------- round loop ---------- -const seen = [] +// Cross-run memory: the calling session passes the findings ledger in as args.known. +// Seeding `seen` is all it takes - finderPrompt() already interpolates seenBlock(seen), +// and each round already dedupes fresh candidates against it, so one assignment buys both +// prompt-level suppression ("do not re-derive this") and mechanical dedupe. +const seen = (ARGS.known || []).map((k) => ({ + file: k.file, + line: k.line, + title: k.title, + status: k.status || 'known', +})) const confirmedAll = [] const unverified = [] const roundStats = [] @@ -499,7 +513,10 @@ while (dry < DRY_THRESHOLD && round < MAX_ROUNDS) { log(`r${round} ${r.lens.key}: verifier verdict "${v.title}" (${v.file}:${v.line}) matched no candidate - dropped`) continue } - unmatched.splice(idx, 1) + // Keep the matched candidate: the verdict schema has no why/repro/evidence, and the + // ledger needs them. Verdict fields are spread last so the verifier's re-rated severity + // and its corrected title/file/line win over the finder's. + const [cand] = unmatched.splice(idx, 1) const rec = { file: v.file, line: v.line, title: v.title, status: v.refuted ? 'refuted' : 'confirmed' } if (seen.some((p) => isDup(rec, p))) continue // cross-lens same-round duplicate seen.push(rec) @@ -507,7 +524,7 @@ while (dry < DRY_THRESHOLD && round < MAX_ROUNDS) { else { newConfirmed++ lensConfirmed++ - confirmedAll.push({ ...v, lens: r.lens.key, round }) + confirmedAll.push({ ...cand, ...v, lens: r.lens.key, round }) } } if (unmatched.length) {