From 34f2e4120753c350024efd7838cb23299433bb04 Mon Sep 17 00:00:00 2001 From: J3vb <192430104+J3vb@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:52:23 +0200 Subject: [PATCH] feat(bughunt): single-finder hunt with graph-fed targeting, telemetry, and defect fixes (#1364) * chore: ignore graphify-out * fix(bughunt): assemble the report in-script - the report agent dropped findings * fix(bughunt): retry only unverified candidates and catch garbage-verdict batches * fix(bughunt): retune the round budget floor and require a budget directive * feat(bughunt): per-round telemetry and a runStats aggregate Co-Authored-By: Claude Opus 4.8 * docs(bughunt-run): record run telemetry and validate coordinates after each hunt * feat(bughunt): drop the sonnet finder slot Co-Authored-By: Claude Opus 4.8 * feat(bughunt): rebuild adaptive targeting - directory clusters, cooldown, graph-fed explore lenses * fix(bughunt): rewind a dead finder's explore draw so unread files are never marked clean * docs(bughunt-run): pre-hunt graph ranking checklist and offline test roster * fix(bughunt): rewind thrown-stage explore draws and correct log/doc wording Co-Authored-By: Claude Opus 4.8 * fix(bughunt): disambiguate absorb() drop log and retire dead sonnet label alternation Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 4.8 --- .claude/skills/bughunt-run/SKILL.md | 58 +++- .claude/workflows/bughunt.harness.mjs | 431 ++++++++++++++++++-------- .claude/workflows/bughunt.js | 335 +++++++++++++------- .gitignore | 1 + 4 files changed, 585 insertions(+), 240 deletions(-) diff --git a/.claude/skills/bughunt-run/SKILL.md b/.claude/skills/bughunt-run/SKILL.md index 5f1e8c2c..476206b7 100644 --- a/.claude/skills/bughunt-run/SKILL.md +++ b/.claude/skills/bughunt-run/SKILL.md @@ -15,14 +15,29 @@ reach a commit, an issue, or a PR body. ## 1. Hunt -Read the ledger and pass every record in as `known`, so the hunt does not -re-derive anything already found, fixed, declined, or refuted: +**Launch the hunt from a turn that carries a token-budget directive** (recommended: +`+25M`, comfortably above a full 8-round run). The workflow's cost ceiling is gated +on `budget.total`, which is null without a directive — a directive-less run has **no +ceiling at all**. The workflow's first log line echoes the state: `budget=25M` means +armed; `budget=NONE - cost ceiling disarmed` means stop the run and relaunch with a +directive. + +Before launching, in order: + +1. **Rebuild the graph** (stale coordinates aim the explore lens at moved code): + `graphify update . --no-cluster` — local tree-sitter, zero LLM cost, ~10.7k nodes. +2. **Rank explore targets**: `node .superpowers/rank-explore.mjs` — writes + `.superpowers/explore-ranking.json`, deprioritizing files recorded clean in + `.superpowers/explored-clean.json` and dropping files that no longer exist. +3. Read the ledger and pass every record in as `known`, so the hunt does not + re-derive anything already found, fixed, declined, or refuted. ``` Workflow({ name: "bughunt", args: { known: , + graph: , lenses: [ {key, prompt}, ... ], // optional: scope the hunt to one subsystem maxRounds: 8, dryThreshold: 2, @@ -30,6 +45,10 @@ Workflow({ }) ``` +If `graph` is omitted or empty the hunt logs +`explore: args.graph absent/empty - falling back to churn-based fresh eyes` and +still runs — degraded targeting, never a smaller lens family. + Omit `lenses` for a general hunt across the rotating families. Each lens object is `{key, prompt}`. `key` must match `^[a-z0-9-]+$` — @@ -40,7 +59,21 @@ validated — it reaches the finder prompt as the literal string `undefined`, silently degrading that lens instead of failing loudly. Check your lens objects before passing them. -When it returns, append each entry of `result.confirmed` to the ledger with +When it returns, first save the raw result verbatim to +`.superpowers/hunts/-raw.json`, then: + +```bash +node .superpowers/render-run-stats.mjs .superpowers/hunts/-raw.json +``` + +This validates the result shape, appends the run's telemetry to +`.superpowers/run-history.json`, updates `.superpowers/explored-clean.json`, and +checks **every** confirmed finding's coordinates against the working tree (file +exists, line within length — the report agent that used to spot-check two findings +is gone). Resolve any `COORD` warnings before appending to the ledger: stale +coordinates poison `bughunt-fix`. + +Then append each entry of `result.confirmed` to the ledger with `status: "open"`, an id from `nextId`, and today's date. Bump `nextId`. The incoming record carries a prose `fix` field (bughunt's suggested remedy) — rename it to `suggestedFix` when appending, so the ledger's `fix` field starts @@ -51,15 +84,12 @@ revertProof}` once something is actually fixed. Then: node .superpowers/render-ledger.mjs ``` -Each confirmed record carries `finder` — which model in the dual-model panel -produced it. The run also logs one `panel:` line with the split. The two finders -are unioned, not voted, so the second model's entire value is what it finds -alone; because duplicates collapse to the opus-slot record, a `sonnet` tag means -opus missed it. Watch that count across a few hunts. Consistently zero is the -evidence for dropping to a single finder — but note that would also weaken -convergence, since a round is only allowed to count as dry when the full panel -reported, so a lone finder having a bad day would read as "clean" instead of -"we didn't fully look". +Each confirmed record carries `finder: "opus"`. The dual-model finder panel was +retired 2026-08-12: attribution over the only measured run priced sonnet's unique +yield (1 high, 4 medium, 9 low) at roughly a third of the run's agents. The known +cost: with one finder, a lazy-but-non-null finder round can read as "clean" where +the panel required both models to agree it was. Watch `runStats` — per-lens +candidate counts make an anomalously empty lens visible after the fact. ## 2. Gate (human) @@ -159,7 +189,9 @@ node .claude/workflows/bughunt.harness.mjs node .claude/workflows/bughunt-fix.harness.mjs node .superpowers/render-ledger.mjs --selftest node .superpowers/verify-fixes.mjs --selftest +node .superpowers/rank-explore.mjs --selftest +node .superpowers/render-run-stats.mjs --selftest ``` -All four run offline with zero API calls. Run them after any edit to the +All six run offline with zero API calls. Run them after any edit to the relevant script. diff --git a/.claude/workflows/bughunt.harness.mjs b/.claude/workflows/bughunt.harness.mjs index 3e90a89f..4144faec 100644 --- a/.claude/workflows/bughunt.harness.mjs +++ b/.claude/workflows/bughunt.harness.mjs @@ -43,18 +43,17 @@ export async function run({ agentStub, args = undefined, budget = undefined }) { } // ---------- stub kit (used from Task 2 onward; harmless now) ---------- -export function makeStub({ hunt, verify, report = () => 'REPORT_MD', recon = defaultRecon }) { +export function makeStub({ hunt, verify, recon = defaultRecon }) { return (prompt, opts) => { const label = opts.label || '' if (label.startsWith('recon:')) return recon(label) - let m = /^r(\d+):hunt:([a-z0-9-]+):(opus|sonnet)$/.exec(label) - if (m) return hunt(Number(m[1]), m[2], m[3], prompt) + let m = /^r(\d+):hunt:([a-z0-9-]+):opus$/.exec(label) + if (m) return hunt(Number(m[1]), m[2], 'opus', prompt) m = /^r(\d+):verify:([a-z0-9-]+?)(:retry)?$/.exec(label) if (m) { const candidates = JSON.parse(prompt.split('--- CANDIDATES ---')[1]) return verify(Number(m[1]), m[2], candidates, Boolean(m[3]), prompt) } - if (label === 'report') return report(prompt) throw new Error(`unexpected agent label: ${label}`) } } @@ -72,6 +71,8 @@ export const finding = (n, over = {}) => ({ evidence: 'e', ...over, }) +export const graphRows = (n) => + Array.from({ length: n }, (_, i) => ({ file: `Server/gen/g${i}.go`, score: 1 - i / (n + 1), degree: 10, cited: 5 })) export const confirmAll = (cands) => ({ verdicts: cands.map((c) => ({ title: c.title, file: c.file, line: c.line, @@ -92,16 +93,11 @@ const scenarios = {} // S1: happy convergence - one bug in round 1, rounds 2-3 dry -> converged. scenarios.s1_convergence = async () => { - const reportPrompts = [] - const { result, calls } = await run({ + const { result, calls, logs } = await run({ agentStub: makeStub({ hunt: (round, key, model) => round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [finding(1)] } : none, verify: (round, key, cands) => confirmAll(cands), - report: (prompt) => { - reportPrompts.push(prompt) - return 'REPORT_MD' - }, }), }) for (const k of ['converged', 'stoppedOnBudget', 'rounds', 'confirmed', 'unverified', 'report']) @@ -111,23 +107,23 @@ scenarios.s1_convergence = async () => { assert.deepEqual(result.rounds.map((r) => r.dryAfter), [0, 1, 2]) assert.deepEqual(result.rounds.map((r) => r.family), ['surfaces', 'bug-classes', 'flows']) assert.equal(result.confirmed.length, 1) - assert.equal(result.report, 'REPORT_MD') assert.ok(!calls.some((c) => (c.opts.label || '').startsWith('r4:')), 'no round 4 after convergence') - assert.match(reportPrompts[0], /CONVERGED after 3 round\(s\)/) - assert.match(reportPrompts[0], /\| 1 \| surfaces \|/) + assert.ok(!calls.some((c) => c.opts.label === 'report'), 'the report is built in-script') + assert.match(result.report, /CONVERGED after 3 round\(s\)/) + assert.match(result.report, /### high - distinct bug alpha1 omega1/) + assert.match(result.report, /\| 1 \| surfaces \|/) + assert.ok(logs.some((l) => /budget=NONE - cost ceiling disarmed/.test(l)), 'a directive-less run must announce the dead ceiling') } -// S2: panel dedupe - opus and sonnet report the same bug -> one candidate, one verify call. -scenarios.s2_panel_dedupe = async () => { +// S2: near-duplicate findings from a single finder collapse - one candidate, one verify call. +scenarios.s2_finder_dedupe = async () => { const verifyBatches = [] - const { result } = await run({ + const { result, calls } = await run({ agentStub: makeStub({ - hunt: (round, key, model) => { - if (round !== 1 || key !== 'ws-hub') return none - return model === 'opus' - ? { findings: [finding(1, { line: 100 })] } - : { findings: [finding(1, { line: 105, title: 'distinct bug alpha1 omega1 variant' })] } - }, + hunt: (round, key) => + round === 1 && key === 'ws-hub' + ? { findings: [finding(1, { line: 100 }), finding(1, { line: 105, title: 'distinct bug alpha1 omega1 variant' })] } + : none, verify: (round, key, cands) => { verifyBatches.push(cands) return confirmAll(cands) @@ -137,6 +133,9 @@ scenarios.s2_panel_dedupe = async () => { assert.equal(verifyBatches.length, 1) assert.equal(verifyBatches[0].length, 1) assert.equal(result.confirmed.length, 1) + const hunts = calls.filter((c) => /^r1:hunt:ws-hub:/.test(c.opts.label || '')) + assert.equal(hunts.length, 1, 'exactly one finder call per lens - the sonnet slot is gone') + assert.match(hunts[0].opts.label, /:opus$/, 'the label keeps the :opus suffix the harness parses') } // S3: refuted findings stay dead - re-reported next round, never re-verified; refutes count toward dry. @@ -237,15 +236,132 @@ scenarios.s6b_verifier_double_failure = async () => { assert.equal(result.unverified.length, 0, 'later-confirmed candidate must leave the unverified list') } -// S7: rounds 1-3 each confirm a bug -> round 4 runs adaptive lenses built from the stats. +// N1 (spec #1): the top-ranked hotspot cluster sits out exactly the next round, then returns. +// The producing cluster keeps running when eligible (the old s7b lock, restated under cooldown). +scenarios.s_cluster_cooldown = async () => { + const { calls } = await run({ + args: { maxRounds: 6, dryThreshold: 9, graph: graphRows(60) }, + agentStub: makeStub({ + hunt: (round, key) => { + if (round === 1 && key === 'ws-hub') + return { findings: [ + finding(1, { file: 'Server/ws/hub.go', title: 'ws bug alpha one' }), + finding(2, { file: 'Server/ws/pubsub.go', line: 300, title: 'ws bug beta two' }), + ] } + if (round === 2 && key === 'concurrency') + return { findings: [ + finding(3, { file: 'Server/ws/emit.go', title: 'ws bug gamma three' }), + finding(4, { file: 'Server/api/user.go', title: 'api bug delta four' }), + ] } + if (round === 4 && key === 'hotspot-server-ws') + return { findings: [finding(9, { file: 'Server/ws/late.go', title: 'late ws bug nine' })] } + return none + }, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const hunted = (rnd, key) => calls.some((c) => (c.opts.label || '') === `r${rnd}:hunt:${key}:opus`) + assert.ok(hunted(4, 'hotspot-server-ws'), 'top cluster hunts in r4') + assert.ok(!hunted(5, 'hotspot-server-ws'), 'the r4 top cluster must sit out r5 even though it produced') + assert.ok(hunted(5, 'hotspot-server-api'), 'the next cluster takes the top slot in r5') + assert.ok(hunted(6, 'hotspot-server-ws'), 'cooldown lasts exactly one round') +} + +// N2 (spec #2): a cooldown gap FREEZES cleanStreak - neither increments nor resets - so +// demotion still means two consecutive clean APPEARANCES. If the gap incremented, ws would +// be demoted before r6; if demotion broke, ws would still run in r8. +scenarios.s_cooldown_freezes_streak = async () => { + const { result, calls } = await run({ + args: { maxRounds: 8, dryThreshold: 9, graph: graphRows(100) }, + agentStub: makeStub({ + hunt: (round, key) => { + if (round === 1 && key === 'ws-hub') + return { findings: [ + finding(1, { file: 'Server/ws/hub.go', title: 'ws bug alpha one' }), + finding(2, { file: 'Server/ws/pubsub.go', line: 300, title: 'ws bug beta two' }), + ] } + if (round === 2 && key === 'concurrency') + return { findings: [finding(3, { file: 'Server/api/user.go', title: 'api bug delta three' })] } + return none + }, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const hunted = (rnd, key) => calls.some((c) => (c.opts.label || '') === `r${rnd}:hunt:${key}:opus`) + assert.ok(hunted(4, 'hotspot-server-ws'), 'clean appearance #1 in r4') + assert.ok(!hunted(5, 'hotspot-server-ws'), 'cooldown in r5') + assert.ok(hunted(6, 'hotspot-server-ws'), 'the gap must freeze the streak at 1, not increment it') + assert.equal(result.rounds.length, 8, 'the run must reach r8 for the demotion assert to mean anything') + assert.ok(!hunted(8, 'hotspot-server-ws'), 'two clean appearances (r4, r6) demote the lens') +} + +// N3 (spec #3): cooldown+demotion emptying the hotspot pool must backfill from explore and +// log it - silent family shrinkage is the exact freshEyesLens() defect this rebuild removes. +scenarios.s_hotspot_backfill = async () => { + const { calls, logs } = await run({ + args: { maxRounds: 5, dryThreshold: 9, graph: graphRows(100) }, + agentStub: makeStub({ + hunt: (round, key) => + round === 1 && key === 'ws-hub' + ? { findings: [finding(1, { file: 'Server/ws/hub.go', title: 'lone ws bug one' })] } + : none, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const r5 = calls.filter((c) => /^r5:hunt:/.test(c.opts.label || '')).map((c) => c.opts.label.split(':')[2]) + assert.ok(!r5.some((k) => k.startsWith('hotspot-')), 'the sole cluster is on cooldown in r5') + assert.deepEqual([...r5].sort(), ['explore-1', 'explore-2', 'explore-3', 'explore-4'], 'the family backfills to full size from explore') + assert.ok(logs.some((l) => /hotspot pool short/.test(l)), 'backfill must be logged, never silent') +} + +// N4 (spec #4): within-run consumption - later rounds draw the NEXT chunk of the ranking, +// never re-offering files already handed to an explore lens this run. +scenarios.s_explore_consumption = async () => { + const { calls } = await run({ + args: { maxRounds: 5, dryThreshold: 9, graph: graphRows(80) }, + agentStub: makeStub({ + hunt: (round, key) => + round === 1 && key === 'ws-hub' + ? { findings: [finding(1, { file: 'Server/ws/hub.go', title: 'seed bug one' })] } + : none, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const promptOf = (rnd, key) => (calls.find((c) => (c.opts.label || '') === `r${rnd}:hunt:${key}:opus`) || {}).prompt || '' + assert.match(promptOf(4, 'explore-1'), /Server\/gen\/g0\.go/) + assert.match(promptOf(4, 'explore-2'), /Server\/gen\/g10\.go/) + assert.match(promptOf(4, 'explore-3'), /Server\/gen\/g20\.go/, 'r4 backfills a third explore lens (single cluster)') + assert.match(promptOf(5, 'explore-1'), /Server\/gen\/g30\.go/, 'r5 draws the next chunk') + assert.doesNotMatch(promptOf(5, 'explore-1'), /Server\/gen\/g0\.go/, 'r5 must not re-offer r4 files') +} + +// N6 (spec #6): args.graph absent -> churn-based fresh-eyes fallback, logged, family intact. +scenarios.s_graph_missing_fallback = async () => { + const { calls, logs } = await run({ + args: { maxRounds: 4, dryThreshold: 9 }, + agentStub: makeStub({ + hunt: (round, key) => + round === 1 && key === 'ws-hub' + ? { findings: [finding(1, { file: 'Server/ws/hub.go', title: 'seed bug one' })] } + : none, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + assert.ok(logs.some((l) => /falling back to churn/.test(l)), 'the fallback must be logged') + const e1 = calls.find((c) => (c.opts.label || '') === 'r4:hunt:explore-1:opus') + assert.ok(e1, 'an explore lens must still run from churn') + assert.match(e1.prompt, /Server\/api\/user\.go/, 'churned file with no findings feeds the fallback') +} + +// S7: rounds 1-3 each confirm a bug -> round 4 runs adaptive lenses: directory-granularity +// hotspots plus explore (churn fallback here - no args.graph is passed). scenarios.s7_adaptive_lenses = async () => { const A = finding(1, { file: 'Server/ws/hub.go', line: 120, title: 'alpha race window one' }) const B = finding(2, { file: 'Server/ws/pubsub.go', line: 60, title: 'beta subscription leak two' }) const C = finding(3, { file: 'Client/tauri-client/src/lib/livekitE2EE.ts', line: 200, title: 'gamma epoch desync three' }) const { result, calls } = await run({ agentStub: makeStub({ - hunt: (round, key, model) => { - if (model !== 'opus') return none + hunt: (round, key) => { if (round === 1 && key === 'ws-hub') return { findings: [A] } if (round === 2 && key === 'concurrency') return { findings: [B] } if (round === 3 && key === 'flow-voice') return { findings: [C] } @@ -257,49 +373,26 @@ scenarios.s7_adaptive_lenses = async () => { assert.equal(result.converged, true) assert.equal(result.rounds.length, 5) // r4, r5 adaptive + dry assert.equal(result.rounds[3].family, 'adaptive') - const r4Hunts = calls.filter((c) => /^r4:hunt:/.test(c.opts.label || '')) - const r4Keys = [...new Set(r4Hunts.map((c) => c.opts.label.split(':')[2]))] + const r4Keys = [...new Set(calls.filter((c) => /^r4:hunt:/.test(c.opts.label || '')).map((c) => c.opts.label.split(':')[2]))] assert.ok(r4Keys.includes('hotspot-server-ws'), `r4 keys: ${r4Keys}`) - assert.ok(r4Keys.includes('fresh-eyes'), `r4 keys: ${r4Keys}`) - const hotspot = r4Hunts.find((c) => c.opts.label.includes('hotspot-server-ws')) + assert.ok(r4Keys.includes('hotspot-client-tauri-client-src-lib'), `r4 keys: ${r4Keys}`) + assert.ok(r4Keys.includes('explore-1'), `r4 keys: ${r4Keys}`) + const hotspot = calls.find((c) => (c.opts.label || '').includes('hotspot-server-ws')) assert.match(hotspot.prompt, /Server\/ws\/hub\.go/) assert.match(hotspot.prompt, /alpha race window one/) - const freshEyes = r4Hunts.find((c) => c.opts.label.includes('fresh-eyes')) - assert.match(freshEyes.prompt, /Server\/api\/user\.go/) // churned, never a finding + const explore = calls.find((c) => (c.opts.label || '') === 'r4:hunt:explore-1:opus') + assert.match(explore.prompt, /Server\/api\/user\.go/) // churned, never a finding assert.equal(result.confirmed.length, 3) } -// S7b: a lens with 2 consecutive clean rounds is demoted from later rounds. -scenarios.s7b_demotion = async () => { - const { result, calls } = await run({ - args: { maxRounds: 6 }, - agentStub: makeStub({ - hunt: (round, key, model) => { - if (model !== 'opus') return none - const src = { 1: 'ws-hub', 2: 'concurrency', 3: 'flow-reconnect', 4: 'hotspot-server-ws', 5: 'hotspot-server-ws' } - if (key === src[round]) - return { findings: [finding(round, { file: `Server/ws/a${round}.go`, title: `unique bug number${round} zeta${round}` })] } - return none - }, - verify: (round, key, cands) => confirmAll(cands), - }), - }) - const labels = calls.map((c) => c.opts.label || '') - assert.ok(labels.some((l) => /^r5:hunt:fresh-eyes:/.test(l)), 'fresh-eyes still runs in r5 (streak 1)') - assert.ok(!labels.some((l) => /^r6:hunt:fresh-eyes:/.test(l)), 'fresh-eyes demoted in r6 (streak 2)') - assert.ok(labels.some((l) => /^r6:hunt:hotspot-server-ws:/.test(l)), 'producing hotspot keeps running') - assert.equal(result.converged, false) - assert.equal(result.confirmed.length, 5) -} - -// S7c: a lens whose VERIFIER died is not demoted; a zero-candidate lens still is. +// S7c: a lens whose VERIFIER died is not demoted (its cluster returns after cooldown); +// a zero-candidate explore lens still accrues streak and demotes. scenarios.s7c_verifier_failure_not_demoted = async () => { const early = { 1: 'ws-hub', 2: 'concurrency', 3: 'flow-reconnect' } const { result, calls } = await run({ - args: { maxRounds: 6 }, + args: { maxRounds: 6, dryThreshold: 9, graph: graphRows(100) }, agentStub: makeStub({ - hunt: (round, key, model) => { - if (model !== 'opus') return none + hunt: (round, key) => { if (round <= 3 && key === early[round]) return { findings: [finding(round, { file: `Server/ws/a${round}.go`, title: `early bug item${round} kappa${round}` })] } if (round >= 4 && key === 'hotspot-server-ws') @@ -310,12 +403,15 @@ scenarios.s7c_verifier_failure_not_demoted = async () => { }), }) const labels = calls.map((c) => c.opts.label || '') - assert.ok(labels.some((l) => /^r6:hunt:hotspot-server-ws:/.test(l)), 'verifier-dead lens must NOT be demoted') - assert.ok(!labels.some((l) => /^r6:hunt:fresh-eyes:/.test(l)), 'zero-candidate lens still accrues streak and demotes') + assert.ok(labels.some((l) => /^r4:hunt:hotspot-server-ws:/.test(l))) + assert.ok(!labels.some((l) => /^r5:hunt:hotspot-server-ws:/.test(l)), 'cooldown after topping r4') + assert.ok(labels.some((l) => /^r6:hunt:hotspot-server-ws:/.test(l)), 'a verifier-dead lens must NOT be demoted') + assert.ok(!labels.some((l) => /^r6:hunt:explore-1:/.test(l)), 'a zero-candidate explore lens still demotes') assert.equal(result.confirmed.length, 3) - assert.equal(result.unverified.length, 3) + assert.equal(result.unverified.length, 2) // b4 and b6, each denied a verdict twice assert.equal(result.converged, false) - assert.ok(result.rounds.slice(3).every((r) => r.dryEligible === false)) + assert.equal(result.rounds[3].dryEligible, false) + assert.equal(result.rounds[5].dryEligible, false) } // S12: empty adaptive family (no confirms, no churn) must break honestly, not count dry rounds. @@ -354,7 +450,7 @@ scenarios.s8_budget_floor = async () => { scenarios.s8b_budget_midrun = async () => { let n = 0 const { result } = await run({ - budget: { total: 1000000, spent: () => 0, remaining: () => (n++ === 0 ? 200000 : 100000) }, + budget: { total: 10000000, spent: () => 0, remaining: () => (n++ === 0 ? 3000000 : 1000000) }, agentStub: makeStub({ hunt: (round, key, model) => round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [finding(1)] } : none, @@ -496,7 +592,7 @@ scenarios.s_custom_lenses = async () => { agentStub: makeStub({ hunt: () => none, verify: (r, k, c) => confirmAll(c) }), }) const keys = calls - .map((c) => /^r1:hunt:([a-z0-9-]+):(opus|sonnet)$/.exec(c.opts.label || '')) + .map((c) => /^r1:hunt:([a-z0-9-]+):opus$/.exec(c.opts.label || '')) .filter(Boolean) .map((m) => m[1]) assert.deepEqual([...new Set(keys)].sort(), ['voice-e2ee-keyholder', 'voice-e2ee-rotation']) @@ -549,50 +645,7 @@ scenarios.s_confirmed_carries_finder_detail = async () => { assert.equal(r.fix, 'FIX_TEXT') assert.equal(r.lens, 'ws-hub') assert.equal(r.round, 1) -} - -// S_FINDER_ATTRIBUTION: confirmed records name which model found them. The panel unions rather -// than votes, so this is the only way to tell whether the second finder earns its cost. Because -// dedupe keeps the first occurrence and opus is index 0, `finder: 'sonnet'` means opus did NOT -// report it - i.e. a sonnet-unique find. `finder: 'opus'` says nothing about sonnet either way. -scenarios.s_finder_attribution = async () => { - const shared = finding(1) - // Different file, not just a different line: isDup treats same-file findings within - // TITLE_MATCH_WINDOW as duplicates on title-word overlap, and the finding() fixtures share - // enough words to collapse into one another. - const sonnetOnly = finding(2, { file: 'Server/api/user.go' }) - const { result } = await run({ - args: { maxRounds: 1, dryThreshold: 9 }, - agentStub: makeStub({ - hunt: (round, key, model) => { - if (round !== 1 || key !== 'ws-hub') return none - return model === 'opus' ? { findings: [shared] } : { findings: [shared, sonnetOnly] } - }, - verify: (round, key, cands) => confirmAll(cands), - }), - }) - const byTitle = Object.fromEntries(result.confirmed.map((r) => [r.title, r.finder])) - assert.equal(byTitle[shared.title], 'opus', 'a find both models made keeps the opus-first record') - assert.equal(byTitle[sonnetOnly.title], 'sonnet', 'a find only sonnet made must be attributed to sonnet') -} - -// S_FINDER_ATTRIBUTION_SURVIVES_DEAD_OPUS: the tag must be taken from the panel slot, not from the -// position in the surviving list. Filtering the nulls out BEFORE reading the index shifts sonnet -// into slot 0 and mislabels every one of its finds as opus - exactly when attribution matters most. -scenarios.s_finder_attribution_survives_dead_opus = async () => { - const only = finding(3) - const { result } = await run({ - args: { maxRounds: 1, dryThreshold: 9 }, - agentStub: makeStub({ - hunt: (round, key, model) => { - if (round !== 1 || key !== 'ws-hub') return none - return model === 'opus' ? null : { findings: [only] } - }, - verify: (round, key, cands) => confirmAll(cands), - }), - }) - assert.equal(result.confirmed.length, 1) - assert.equal(result.confirmed[0].finder, 'sonnet', 'a dead opus must not relabel sonnet finds as opus') + assert.equal(r.finder, 'opus', 'the finder tag is constant now but the ledger still expects it') } // S_VERIFIER_IS_NOT_TOLD_THE_FINDER: the verifier prompt deliberately says "another model" and @@ -619,26 +672,162 @@ scenarios.s_verifier_is_not_told_the_finder = async () => { } } -// S_PANEL_SPLIT_IS_REPORTED: a tag nobody reads is not a measurement. The run must say out loud -// how many confirmed findings only the second finder produced, which is the number that decides -// whether the second model is worth its cost. -scenarios.s_panel_split_is_reported = async () => { - const { logs } = await run({ +// New (spec Testing #8): the report is built in-script. Section count must equal the confirmed +// count at 82 (the agent version emitted 79 for 82), and the unverified section must survive +// the agent's removal - it used to exist only inside the report agent's prompt. +scenarios.s_report_deterministic = async () => { + const many = Array.from({ length: 82 }, (_, i) => + finding(i, { file: `Server/ws/f${i}.go`, line: 10, title: `unique bug row${i} tag${i}` })) + const stuck = finding(999, { file: 'Server/api/stuck.go', line: 40, title: 'stuck bug never verified' }) + const { result, calls } = await run({ args: { maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ + hunt: (round, key, model) => + round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [...many, stuck] } : none, + verify: (round, key, cands) => confirmAll(cands.filter((c) => c.file !== 'Server/api/stuck.go')), + }), + }) + assert.equal(result.confirmed.length, 82) + assert.equal(result.unverified.length, 1) + assert.ok(!calls.some((c) => c.opts.label === 'report'), 'no report agent may run') + const sections = (result.report.match(/^### /gm) || []).length + assert.equal(sections, 82, 'one section per confirmed finding, none dropped') + assert.match(result.report, /## Unverified - re-run/) + assert.match(result.report, /stuck bug never verified/) + assert.match(result.report, /## Convergence/) +} + +// New (spec Testing #7): the retry re-sends ONLY unmatched candidates, and N garbage verdicts +// (count == candidate count, zero of them matching) must still trigger it - the hole S10 misses +// because S10's verdict list is empty rather than full of junk. +scenarios.s_targeted_retry = async () => { + const a = finding(1, { file: 'Server/ws/a.go', title: 'alpha bug one paired' }) + const b = finding(2, { file: 'Server/api/b.go', title: 'beta bug two orphaned' }) + const retryBatches = [] + const { result } = await run({ + args: { maxRounds: 1, dryThreshold: 9 }, + agentStub: makeStub({ + hunt: (round, key, model) => + round === 1 && key === 'ws-hub' && model === 'opus' ? { findings: [a, b] } : none, + verify: (round, key, cands, isRetry) => { + if (isRetry) { + retryBatches.push(cands) + return confirmAll(cands) + } + // one real verdict for a, one garbage verdict pointing nowhere: count matches, content doesn't + return { + verdicts: [ + { title: a.title, file: a.file, line: a.line, refuted: false, reason: 'ok', confidence: 'high', severity: 'high', fix: 'f' }, + { title: 'hallucinated', file: 'Server/nowhere.go', line: 1, refuted: false, reason: 'x', confidence: 'low', severity: 'low' }, + ], + } + }, + }), + }) + assert.equal(retryBatches.length, 1, 'retry must fire despite verdict count == candidate count') + assert.deepEqual(retryBatches[0].map((c) => c.file), ['Server/api/b.go'], 'only the unmatched candidate is re-sent') + assert.equal(result.confirmed.length, 2) + assert.equal(result.unverified.length, 0) +} + +// New (spec Testing #9): the retuned floor must stop a run the old 150k floor let through. +// 1M remaining is under the ~2M measured per-round cost - starting a round would overshoot. +scenarios.s9_budget_ceiling_retuned = async () => { + const { result, logs } = await run({ + budget: { total: 10000000, spent: () => 9000000, remaining: () => 1000000 }, + agentStub: makeStub({ hunt: () => none, verify: (r, k, c) => confirmAll(c) }), + }) + assert.equal(result.rounds.length, 0, '1M remaining must not start a ~2M round') + assert.equal(result.stoppedOnBudget, true) + assert.ok(logs.some((l) => /Budget floor/.test(l))) +} + +// New: telemetry. Per-round suppression split (ledger vs same-run), spend sampling, file +// coverage, severity mix, per-lens precision, and the top-level runStats aggregate. Without +// this every cost figure from a run is eyewitness-only - the 2026-08-12 problem. +scenarios.s_telemetry = async () => { + let spent = 0 + const known = [{ file: 'Server/ws/hub.go', line: 100, title: 'known bug from ledger prior', status: 'fixed' }] + const { result } = await run({ + args: { maxRounds: 1, dryThreshold: 9, known }, + budget: { total: 50000000, spent: () => (spent += 500000), remaining: () => 40000000 }, agentStub: makeStub({ hunt: (round, key, model) => { - if (round !== 1 || key !== 'ws-hub') return none - return model === 'opus' - ? { findings: [finding(1)] } - : { findings: [finding(1), finding(2, { file: 'Server/api/user.go' })] } + if (round !== 1 || key !== 'ws-hub' || model !== 'opus') return none + return { findings: [ + finding(1, { file: 'Server/ws/hub.go', line: 102, title: 'known bug from ledger prior' }), + finding(2, { file: 'Server/api/fresh.go', severity: 'medium', title: 'fresh bug beta gamma' }), + ] } }, verify: (round, key, cands) => confirmAll(cands), }), }) - const line = logs.find((l) => /panel:/.test(l)) - assert.ok(line, 'the run must report the panel split') - assert.match(line, /sonnet-only/, 'the split must name the sonnet-only count explicitly') - assert.match(line, /\b1\b/, 'exactly one confirmed finding here was sonnet-only') + const r1 = result.rounds[0] + assert.equal(r1.suppressedLedger, 1, 'the ledger-known duplicate must be counted as ledger suppression') + assert.equal(r1.suppressedRun, 0) + assert.ok(r1.spentAfter > r1.spentBefore, 'per-round spend must be sampled') + assert.equal(r1.filesTouched, 1) + assert.equal(r1.filesNew, 1) + assert.deepEqual(r1.severity, { critical: 0, high: 0, medium: 1, low: 0 }) + assert.equal(r1.perLens['ws-hub'].confirmed, 1) + assert.equal(r1.perLens['ws-hub'].fresh, 1) + assert.ok(result.runStats, 'runStats missing from the result') + assert.equal(result.runStats.confirmed, 1) + assert.equal(result.runStats.suppressedLedger, 1) + assert.equal(result.runStats.config.maxRounds, 1) + assert.match(result.report, /## Run stats/) +} + +// N7 (Task 9 review finding): a dead finder on an explore lens read nothing - its draw is +// rewound so the files never reach exploredFiles, where the session would record them clean +// and deprioritize them in every future hunt. maxRounds caps at 4 on purpose: a live round-5 +// lens would legitimately re-read the rewound files and they would CORRECTLY re-enter +// exploredFiles - the poison-prevention property is only assertable when the run ends here. +// Re-offering in later rounds follows from the same exploreConsumed state drawExploreFiles +// filters on, so this one scenario locks the mechanism. +scenarios.s_explore_rewind_on_dead_finder = async () => { + const { result, calls } = await run({ + args: { maxRounds: 4, dryThreshold: 9, graph: graphRows(80) }, + agentStub: makeStub({ + hunt: (round, key) => { + if (round === 1 && key === 'ws-hub') + return { findings: [finding(1, { file: 'Server/ws/hub.go', title: 'seed bug one' })] } + if (round === 4 && key === 'explore-1') return null // dead finder: read nothing + return none + }, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const promptOf = (rnd, key) => (calls.find((c) => (c.opts.label || '') === `r${rnd}:hunt:${key}:opus`) || {}).prompt || '' + assert.match(promptOf(4, 'explore-1'), /Server\/gen\/g0\.go/, 'r4 explore-1 drew the head of the ranking') + for (let i = 0; i < 10; i++) + assert.ok(!result.exploredFiles.includes(`Server/gen/g${i}.go`), `g${i} was never read - must not be reported explored`) + assert.ok(result.exploredFiles.includes('Server/gen/g10.go'), 'files a LIVE lens drew stay reported') + assert.ok(result.exploredFiles.includes('Server/gen/g20.go'), 'backfilled live lens files stay reported too') +} + +// N8 (final-review finding): a THROWN stage nulls the whole lens result - the second +// finder-failure mode the code documents. Its explore draw must rewind exactly like the +// null-finder case, or never-read files reach exploredFiles and poison explored-clean. +scenarios.s_explore_rewind_on_thrown_stage = async () => { + const { result, calls } = await run({ + args: { maxRounds: 4, dryThreshold: 9, graph: graphRows(80) }, + agentStub: makeStub({ + hunt: (round, key) => { + if (round === 1 && key === 'ws-hub') + return { findings: [finding(1, { file: 'Server/ws/hub.go', title: 'seed bug one' })] } + if (round === 4 && key === 'explore-1') throw new Error('finder infrastructure blew up') + return none + }, + verify: (round, key, cands) => confirmAll(cands), + }), + }) + const promptOf = (rnd, key) => (calls.find((c) => (c.opts.label || '') === `r${rnd}:hunt:${key}:opus`) || {}).prompt || '' + assert.match(promptOf(4, 'explore-1'), /Server\/gen\/g0\.go/, 'r4 explore-1 drew the head of the ranking') + for (let i = 0; i < 10; i++) + assert.ok(!result.exploredFiles.includes(`Server/gen/g${i}.go`), `g${i} was never read - must not be reported explored`) + assert.ok(result.exploredFiles.includes('Server/gen/g10.go'), 'files a LIVE lens drew stay reported') + assert.equal(result.rounds[3].dryEligible, false, 'a nulled lens result still makes the round ineligible') } // ---------- runner ---------- diff --git a/.claude/workflows/bughunt.js b/.claude/workflows/bughunt.js index 0b5a03c0..f39a996e 100644 --- a/.claude/workflows/bughunt.js +++ b/.claude/workflows/bughunt.js @@ -1,10 +1,9 @@ export const meta = { name: 'bughunt', - description: 'Converging multi-round bug hunt: rotating lens families, dual-model panels, fable refute-by-default verification, dry-threshold stop', + description: 'Converging multi-round bug hunt: rotating lens families, single opus finder, fable refute-by-default verification, dry-threshold stop', whenToUse: 'Hunting real bugs across the Go server, Tauri Rust backend, and TS client until consecutive rounds go dry. Not a security-only scan.', phases: [ { title: 'Recon', detail: 'haiku: churn + concurrency-surface inventory' }, - { title: 'Report', detail: 'fable: ranked findings + convergence table' }, ], } @@ -19,14 +18,15 @@ const ARGS = (() => { const MAX_ROUNDS = ARGS.maxRounds || 8 const DRY_THRESHOLD = ARGS.dryThreshold || 2 // A scoped hunt (args.lenses) replaces the round-1 family outright; later rounds still go -// adaptive, so hotspot and fresh-eyes coverage - and therefore convergence - still work. +// adaptive, so hotspot and explore coverage - and therefore convergence - still work. const CUSTOM_LENSES = Array.isArray(ARGS.lenses) && ARGS.lenses.length ? ARGS.lenses : null -// ponytail: rough floor for one round (up to 12 high-effort finders + verifiers); tune after live runs -const ROUND_BUDGET_FLOOR = 150000 +// Floor for one round, tuned from the 2026-08-12 run: ~2.6M output tokens per round measured. +// The old 150k floor would overshoot the ceiling by nearly a full round. +const ROUND_BUDGET_FLOOR = 2000000 // The args channel has already been observed delivering something the script // could not read; an unnoticed fallback here is an 8x cost surprise, so say out // loud what the run is actually going to do. -log(`config: maxRounds=${MAX_ROUNDS} dryThreshold=${DRY_THRESHOLD}${CUSTOM_LENSES ? ` lenses=custom(${CUSTOM_LENSES.length})` : ''}`) +log(`config: maxRounds=${MAX_ROUNDS} dryThreshold=${DRY_THRESHOLD}${CUSTOM_LENSES ? ` lenses=custom(${CUSTOM_LENSES.length})` : ''} budget=${budget.total ? Math.round(budget.total / 1e6) + 'M' : 'NONE - cost ceiling disarmed'}`) // ---------- schemas: copied VERBATIM from the current bughunt.js ---------- const FINDINGS = { @@ -93,14 +93,16 @@ Out of scope, do not report: naming, formatting, missing tests, "consider adding performance that is not a hang, anything you cannot point at specific lines for. Method: - 1. Read the actual files. Never report from a filename or a grep hit alone. + 1. Read the actual files. Never report from a filename, a grep hit, or a graph edge alone - a + graphify edge is structural evidence of coupling, not of a bug; open the cited file and confirm. 2. For every candidate, grep for ALL callers before judging - a guard may already live upstream. 3. Check whether an existing test already locks the behavior you think is wrong. If a test asserts it, it is intended behavior, not a bug. Test files are *_test.go and tests/unit/*.test.ts. 4. Report EVERY finding you can prove - there is no cap. The quality bar stays: zero findings is a valid, respectable answer, and each finding needs file, line, and a concrete repro. -You may run read-only shell commands (grep, git log, go doc). Do not modify any file. Do not run the test suite. +You may run read-only shell commands (grep, git log, go doc, graphify path, graphify explain). +Do not modify any file. Do not run the test suite. ` // ---------- lens catalog ---------- @@ -271,46 +273,83 @@ const FLOW_LENSES = [ ] function lensesForRound(round) { - if (CUSTOM_LENSES) return round === 1 ? CUSTOM_LENSES : buildAdaptiveLenses() + if (CUSTOM_LENSES) return round === 1 ? CUSTOM_LENSES : buildAdaptiveLenses(round) if (round === 1) return SURFACE_LENSES if (round === 2) return BUGCLASS_LENSES if (round === 3) return FLOW_LENSES - return buildAdaptiveLenses() + return buildAdaptiveLenses(round) } function familyName(round) { if (CUSTOM_LENSES) return round === 1 ? 'custom' : 'adaptive' return ['surfaces', 'bug-classes', 'flows'][round - 1] || 'adaptive' } +// Directory granularity: the old two/three-segment cluster collapsed the whole TS client into +// one bucket (35 of 82 findings), so the "top cluster" never changed for five straight rounds. function clusterOf(file) { const parts = String(file).split('/') - return parts.slice(0, parts[0] === 'Client' ? 3 : 2).join('/') + return parts.length > 1 ? parts.slice(0, -1).join('/') : parts[0] } -function freshEyesLens() { - const files = churnFiles.filter((f) => !seen.some((s) => s.file === f)).slice(0, 10) - if (!files.length) return [] - return [ - { - key: 'fresh-eyes', - prompt: - `These files churned heavily in the last 8 weeks, yet no hunt round has confirmed or refuted a ` + - `single finding in them - either they are clean or every lens so far walked past them. Read each ` + - `one IN FULL with fresh eyes and hunt for real bugs of any class:\n` + - files.map((f) => ` - ${f}`).join('\n'), - }, - ] + +// ---------- explore targeting ---------- +// args.graph: session-computed coupling ranking (rank-explore.mjs). The workflow only reads +// .file - scoring already happened outside, where the filesystem is. +const GRAPH_ROWS = (Array.isArray(ARGS.graph) ? ARGS.graph : []).filter((r) => r && typeof r.file === 'string') +const EXPLORE_FILES_PER_LENS = 10 +const exploreConsumed = new Set() // within-run consumption: never re-offer a file to a later round +let exploreFallbackLogged = false +function drawExploreFiles() { + let pool + if (GRAPH_ROWS.length) pool = GRAPH_ROWS.map((r) => r.file) + else { + if (!exploreFallbackLogged) { + log('explore: args.graph absent/empty - falling back to churn-based fresh eyes') + exploreFallbackLogged = true + } + pool = churnFiles + } + const files = pool + .filter((f) => !exploreConsumed.has(f) && !seen.some((s) => s.file === f)) + .slice(0, EXPLORE_FILES_PER_LENS) + for (const f of files) exploreConsumed.add(f) + return files } -function buildAdaptiveLenses() { +function exploreLens(i) { + const files = drawExploreFiles() + if (!files.length) return null + const src = GRAPH_ROWS.length + ? `These files are heavily coupled (per the code graph) to files where confirmed bugs live, yet no ` + + `hunt has confirmed or refuted a single finding in them - either they are clean or every lens so ` + + `far walked past them.` + : `These files churned heavily in the last 8 weeks, yet no hunt round has confirmed or refuted a ` + + `single finding in them - either they are clean or every lens so far walked past them.` + return { + key: `explore-${i}`, + prompt: `${src} Read each one IN FULL with fresh eyes and hunt for real bugs of any class:\n` + + files.map((f) => ` - ${f}`).join('\n'), + files, + } +} + +let cooldownCluster = null // the top-ranked cluster hunted in round N sits out round N+1 +function buildAdaptiveLenses(round) { const byCluster = {} for (const c of confirmedAll) { const cl = clusterOf(c.file) if (!byCluster[cl]) byCluster[cl] = [] byCluster[cl].push(c) } - const top = Object.entries(byCluster) + // Explore-heavy schedule: measured hotspot yield flattened to 0.25 high+med/agent by round 6. + const hotspotQuota = round <= 5 ? 2 : 1 + const exploreQuota = round <= 5 ? 2 : 3 + const hotKey = (cl) => ('hotspot ' + cl).toLowerCase().replace(/[^a-z0-9]+/g, '-') + const picked = Object.entries(byCluster) .sort((a, b) => b[1].length - a[1].length) - .slice(0, 3) - const hotspots = top.map(([cluster, items]) => ({ - key: ('hotspot ' + cluster).toLowerCase().replace(/[^a-z0-9]+/g, '-'), + .filter(([cl]) => cl !== cooldownCluster) + .filter(([cl]) => (cleanStreak[hotKey(cl)] || 0) < 2) // pre-filter so backfill sees the real shortfall + .slice(0, hotspotQuota) + cooldownCluster = picked.length ? picked[0][0] : null + const hotspots = picked.map(([cluster, items]) => ({ + key: hotKey(cluster), prompt: `Bugs cluster. Confirmed findings so far in ${cluster}:\n` + items.map((i) => ` - ${i.file}:${i.line} ${i.title}`).join('\n') + @@ -318,7 +357,19 @@ function buildAdaptiveLenses() { `(subscribe/unsubscribe, open/close, register/transfer, acquire/release), and the paths a past fix ` + `here did NOT cover. Do not re-report the findings listed above - they are already known.`, })) - return [...hotspots, ...freshEyesLens()] + const shortfall = hotspotQuota - hotspots.length + if (shortfall > 0) log(`adaptive: hotspot pool short by ${shortfall} - trying explore backfill`) + const explores = [] + for (let i = 1; i <= exploreQuota + shortfall; i++) { + if ((cleanStreak[`explore-${i}`] || 0) >= 2) continue // demoted slot: no substitution, that IS demotion + const lens = exploreLens(i) + if (!lens) { + log(`adaptive: explore pool exhausted after ${explores.length} lens(es)`) + break + } + explores.push(lens) + } + return [...hotspots, ...explores] } // ---------- dedupe + ledger helpers ---------- @@ -343,10 +394,18 @@ function isDup(a, b) { const hits = aw.filter((w) => bw.has(w)).length return hits * 2 >= aw.length } -function dedupe(cands, priors) { +function dedupe(cands, priors, counts) { const kept = [] for (const c of cands) { - if (priors.some((p) => isDup(c, p)) || kept.some((k) => isDup(c, k))) continue + const prior = priors.find((p) => isDup(c, p)) + if (prior) { + if (counts) counts[prior.fromLedger ? 'suppressedLedger' : 'suppressedRun']++ + continue + } + if (kept.some((k) => isDup(c, k))) { + if (counts) counts.suppressedRun++ + continue + } kept.push(c) } return kept @@ -417,6 +476,7 @@ const seen = (ARGS.known || []).map((k) => ({ line: k.line, title: k.title, status: k.status || 'known', + fromLedger: true, // telemetry: distinguishes ledger suppression from same-run suppression })) const confirmedAll = [] const unverified = [] @@ -464,10 +524,12 @@ while (dry < DRY_THRESHOLD && round < MAX_ROUNDS) { const family = lensesForRound(round + 1) if (!family || !family.length) break // nothing to hunt != everything demoted round++ + const spentBefore = budget.spent() + const counts = { suppressedLedger: 0, suppressedRun: 0, finderNull: 0, finderEmpty: 0, verifierNull: 0 } const lenses = family.filter((l) => (cleanStreak[l.key] || 0) < 2) if (!lenses.length) { dry++ - roundStats.push({ round, family: familyName(round), lenses: 0, candidates: 0, fresh: 0, confirmed: 0, refuted: 0, dryEligible: true, dryAfter: dry }) + roundStats.push({ round, family: familyName(round), lenses: 0, candidates: 0, fresh: 0, confirmed: 0, refuted: 0, dryEligible: true, dryAfter: dry, severity: { critical: 0, high: 0, medium: 0, low: 0 }, perLens: {}, filesTouched: 0, filesNew: 0, ...counts, spentBefore, spentAfter: budget.spent() }) log(`Round ${round}: every lens demoted - counts as a dry round (dry=${dry})`) continue } @@ -476,123 +538,184 @@ while (dry < DRY_THRESHOLD && round < MAX_ROUNDS) { const lensResults = await pipeline( lenses, (lens) => - parallel([ - () => agent(finderPrompt(lens, rnd), { label: `r${rnd}:hunt:${lens.key}:opus`, phase: `Round ${rnd}`, model: 'opus', effort: 'high', schema: FINDINGS }), - () => agent(finderPrompt(lens, rnd), { label: `r${rnd}:hunt:${lens.key}:sonnet`, phase: `Round ${rnd}`, model: 'sonnet', effort: 'high', schema: FINDINGS }), - ]).then((pair) => ({ lens, pair })), + agent(finderPrompt(lens, rnd), { label: `r${rnd}:hunt:${lens.key}:opus`, phase: `Round ${rnd}`, model: 'opus', effort: 'high', schema: FINDINGS }) + .then((res) => ({ lens, res })), async (r) => { - const { lens, pair } = r - const finderFailed = pair.some((p) => p === null) - // Tag by panel slot, not by position in the surviving list: filtering the nulls out first - // would shift sonnet into slot 0 whenever opus dies and mislabel its finds. The panel unions - // rather than votes, so this is the only signal for whether the second finder earns its cost - // - and since dedupe keeps the first occurrence and opus is slot 0, `finder: 'sonnet'` means - // opus did not report it. `finder: 'opus'` says nothing about sonnet either way. - const union = pair.flatMap((p, i) => - p ? (p.findings || []).map((f) => ({ ...f, finder: i ? 'sonnet' : 'opus' })) : [], - ) - const fresh = dedupe(union, seenAtStart) - if (!fresh.length) return { lens, finderFailed, unionCount: union.length, fresh: [], verdicts: [] } + const { lens, res } = r + // agent() returns null on failure; a thrown stage instead nulls the whole lens result, + // which the eligibility check catches separately. Both checks are needed. + const finderFailed = res === null + if (finderFailed) counts.finderNull++ + else if (!(res.findings || []).length) counts.finderEmpty++ + // finder is constant now; kept on the record for ledger continuity across hunts + const union = res ? (res.findings || []).map((f) => ({ ...f, finder: 'opus' })) : [] + const fresh = dedupe(union, seenAtStart, counts) + if (!fresh.length) return { lens, finderFailed, unionCount: union.length, fresh: [], matched: [], unmatched: [] } log(`r${rnd} ${lens.key}: ${fresh.length} fresh candidate(s) -> verification`) const vopts = { phase: `Round ${rnd}`, model: 'fable', effort: 'high', schema: VERDICTS } - let v = await agent(verifyPrompt(lens.key, fresh), { ...vopts, label: `r${rnd}:verify:${lens.key}` }) - if (!v || (v.verdicts || []).length < fresh.length) { - const retry = await agent(verifyPrompt(lens.key, fresh), { ...vopts, label: `r${rnd}:verify:${lens.key}:retry` }) - if (((retry && retry.verdicts) || []).length > ((v && v.verdicts) || []).length) v = retry + // Pair verdicts to candidates as they arrive, then retry ONLY what got no usable verdict. + // Retrying the whole batch re-burned every verdict on a partial return, and the old + // count-based trigger let N unmatched garbage verdicts skip the retry entirely. + const matched = [] + const unmatched = fresh.slice() + const absorb = (vs) => { + for (const v of vs || []) { + const vRec = { file: v.file, line: v.line, title: v.title } + const idx = unmatched.findIndex((f) => isDup(vRec, f) || isDup(f, vRec)) + if (idx === -1) { + const claimed = matched.some(({ cand }) => isDup(vRec, cand) || isDup(cand, vRec)) + log(`r${rnd} ${lens.key}: verifier verdict "${v.title}" (${v.file}:${v.line}) ${claimed ? 'duplicates an already-claimed candidate' : 'matched no candidate'} - dropped`) + continue + } + const [cand] = unmatched.splice(idx, 1) + matched.push({ v, cand }) + } } - return { lens, finderFailed, unionCount: union.length, fresh, verdicts: (v && v.verdicts) || [] } + const v1 = await agent(verifyPrompt(lens.key, fresh), { ...vopts, label: `r${rnd}:verify:${lens.key}` }) + absorb(v1 && v1.verdicts) + if (!v1) counts.verifierNull++ + if (unmatched.length) { + const v2 = await agent(verifyPrompt(lens.key, unmatched.slice()), { ...vopts, label: `r${rnd}:verify:${lens.key}:retry` }) + absorb(v2 && v2.verdicts) + if (!v2) counts.verifierNull++ + } + return { lens, finderFailed, unionCount: union.length, fresh, matched, unmatched } }, ) + // a thrown stage nulls the whole lens result - rewind its explore draw too, or the + // session records never-read files as explored-clean (the same poison as a null finder) + lensResults.forEach((r, i) => { + if (!r && lenses[i].files) for (const f of lenses[i].files) exploreConsumed.delete(f) + }) + let eligible = !lensResults.some((r) => !r) let newConfirmed = 0 let newRefuted = 0 let candCount = 0 let freshCount = 0 + const perLens = {} + const sevMix = { critical: 0, high: 0, medium: 0, low: 0 } + const filesTouched = new Set() + const filesNew = new Set() for (const r of lensResults.filter(Boolean)) { candCount += r.unionCount freshCount += r.fresh.length if (r.finderFailed) eligible = false + if (r.finderFailed && r.lens.files) { + // a dead finder read nothing: un-consume its draw so later rounds can re-offer the + // files and the session does not record never-examined files as explored-clean + for (const f of r.lens.files) exploreConsumed.delete(f) + } let lensConfirmed = 0 - const unmatched = r.fresh.slice() - for (const v of r.verdicts) { - const vRec = { file: v.file, line: v.line, title: v.title } - const idx = unmatched.findIndex((f) => isDup(vRec, f) || isDup(f, vRec)) - if (idx === -1) { - log(`r${round} ${r.lens.key}: verifier verdict "${v.title}" (${v.file}:${v.line}) matched no candidate - dropped`) - continue - } + let lensRefuted = 0 + for (const { v, cand } of r.matched) { // Keep the matched candidate: the verdict schema has no why/repro/evidence, and the // ledger needs them. Verdict fields are spread last so the verifier's re-rated severity // and its corrected title/file/line win over the finder's. - const [cand] = unmatched.splice(idx, 1) const rec = { file: v.file, line: v.line, title: v.title, status: v.refuted ? 'refuted' : 'confirmed' } - if (seen.some((p) => isDup(rec, p))) continue // cross-lens same-round duplicate + if (seen.some((p) => isDup(rec, p))) { counts.suppressedRun++; continue } // cross-lens same-round duplicate seen.push(rec) - if (v.refuted) newRefuted++ + if (v.refuted) { newRefuted++; lensRefuted++ } else { newConfirmed++ lensConfirmed++ + sevMix[v.severity] = (sevMix[v.severity] || 0) + 1 confirmedAll.push({ ...cand, ...v, lens: r.lens.key, round }) } } - if (unmatched.length) { + if (r.unmatched.length) { eligible = false // partial verifier failure: some candidates got no verdict at all - for (const f of unmatched) unverified.push({ ...f, lens: r.lens.key, round }) + for (const f of r.unmatched) unverified.push({ ...f, lens: r.lens.key, round }) } // a lens hunted at partial panel strength, or whose candidates never got a verdict, is not evidence of cleanliness - if (!r.finderFailed && !unmatched.length) cleanStreak[r.lens.key] = lensConfirmed > 0 ? 0 : (cleanStreak[r.lens.key] || 0) + 1 + if (!r.finderFailed && !r.unmatched.length) cleanStreak[r.lens.key] = lensConfirmed > 0 ? 0 : (cleanStreak[r.lens.key] || 0) + 1 + perLens[r.lens.key] = { candidates: r.unionCount, fresh: r.fresh.length, confirmed: lensConfirmed, refuted: lensRefuted, unverified: r.unmatched.length } + // coverage proxy: files that produced fresh candidates this round (finder reading is unobservable) + for (const f of r.fresh) { + filesTouched.add(f.file) + if (!seenAtStart.some((s) => s.file === f.file)) filesNew.add(f.file) + } } if (newConfirmed > 0) dry = 0 else if (eligible) dry++ // ineligible zero-confirm round: dry unchanged - "we didn't fully look" is not "it's clean" - roundStats.push({ round, family: familyName(round), lenses: lenses.length, candidates: candCount, fresh: freshCount, confirmed: newConfirmed, refuted: newRefuted, dryEligible: eligible, dryAfter: dry }) + roundStats.push({ round, family: familyName(round), lenses: lenses.length, candidates: candCount, fresh: freshCount, confirmed: newConfirmed, refuted: newRefuted, dryEligible: eligible, dryAfter: dry, severity: sevMix, perLens, filesTouched: filesTouched.size, filesNew: filesNew.size, ...counts, spentBefore, spentAfter: budget.spent() }) log(`Round ${round} (${familyName(round)}): ${newConfirmed} confirmed, ${newRefuted} refuted, dry=${dry}${eligible ? '' : ' (ineligible)'}`) } const converged = dry >= DRY_THRESHOLD -// The panel unions rather than votes, so the second finder's whole value is what it finds alone. -// dedupe keeps the opus-slot record when both report the same bug, so a confirmed finding tagged -// sonnet is one opus missed. A run where that count is 0 is the evidence for dropping the second -// model; anything above 0 is what it bought. -const sonnetOnly = confirmedAll.filter((f) => f.finder === 'sonnet').length -log(`panel: ${confirmedAll.length} confirmed, ${sonnetOnly} sonnet-only (opus missed), ${confirmedAll.length - sonnetOnly} found by opus`) - -// ---------- report ---------- -phase('Report') +// ---------- report (deterministic) ---------- +// A report agent silently dropped findings (79 sections for 82 confirmed on 2026-08-12), so the +// markdown is assembled in-script from confirmedSorted. Coordinate spot-checking moved to the +// calling session, which validates EVERY finding's file/line after return (see bughunt-run). const RANK = { critical: 0, high: 1, medium: 2, low: 3 } const confirmedSorted = confirmedAll.slice().sort((a, b) => RANK[a.severity] - RANK[b.severity]) const unverifiedFinal = unverified.filter((u) => !seen.some((p) => isDup(u, p))) const table = convergenceTable(roundStats, converged, stoppedOnBudget) - -let report -if (!confirmedSorted.length && !unverifiedFinal.length) { - const outcome = converged ? 'Converged' : stoppedOnBudget ? 'Stopped on budget - NOT converged' : 'Hit the round backstop - NOT converged' - report = `${outcome} after ${round} round(s) with zero confirmed findings.\n\n${table}` -} else { - report = await agent( - `You are writing the final bug-hunt report for OwnCord (D:/Local-Lab/Repos/OwnCord).\n\n` + - `The findings below already survived adversarial verification - do NOT re-litigate them, and do NOT add new ` + - `ones. Your job is presentation and prioritization for a maintainer who will fix these today.\n\n` + - `Spot-check the two highest-severity findings against the real files to make sure file paths and line numbers ` + - `are accurate; correct them silently if they drifted.\n\n` + - `Write markdown:\n` + - ` - Open with one paragraph: how many real bugs, in which subsystems, whether the hunt CONVERGED, and which ` + - `finding to fix first and why.\n` + - ` - Then one section per finding, ordered by severity: a "### - " heading, the ` + - `\`file:line\` reference, what breaks and under exactly what conditions, and the smallest correct fix.\n` + - (unverifiedFinal.length - ? ` - Then an "## Unverified - re-run" section listing these candidates whose verification failed twice: ` + - `${JSON.stringify(unverifiedFinal)}\n` - : '') + - ` - End with the convergence table below, VERBATIM.\n` + - `Write in complete sentences. No emoji, no "consider" hedging.\n\n` + - `--- CONFIRMED FINDINGS ---\n${JSON.stringify(confirmedSorted, null, 2)}\n\n` + - `--- CONVERGENCE TABLE ---\n${table}`, - { label: 'report', phase: 'Report', model: 'fable', effort: 'high' }, - ) +const sum = (k) => roundStats.reduce((n, r) => n + (r[k] || 0), 0) +const runStats = { + config: { maxRounds: MAX_ROUNDS, dryThreshold: DRY_THRESHOLD, customLenses: !!CUSTOM_LENSES, knownCount: (ARGS.known || []).length, graphRows: GRAPH_ROWS.length, budgetTotal: budget.total }, + spentTotal: budget.spent(), + rounds: roundStats.length, + converged, + stoppedOnBudget, + confirmed: confirmedSorted.length, + refuted: sum('refuted'), + unverified: unverifiedFinal.length, + suppressedLedger: sum('suppressedLedger'), + suppressedRun: sum('suppressedRun'), + finderNull: sum('finderNull'), + finderEmpty: sum('finderEmpty'), + verifierNull: sum('verifierNull'), } -return { converged, stoppedOnBudget, rounds: roundStats, confirmed: confirmedSorted, unverified: unverifiedFinal, report } +function buildReport() { + const outcome = converged + ? `CONVERGED after ${round} round(s).` + : stoppedOnBudget + ? `NOT converged - stopped on budget after ${round} round(s).` + : `NOT converged - hit the round backstop after ${round} round(s).` + const sev = { critical: 0, high: 0, medium: 0, low: 0 } + for (const f of confirmedSorted) sev[f.severity] = (sev[f.severity] || 0) + 1 + const lines = ['# Bug hunt report', ''] + lines.push( + `${confirmedSorted.length} confirmed finding(s) - ${sev.critical} critical, ${sev.high} high, ` + + `${sev.medium} medium, ${sev.low} low. ${outcome}` + + (confirmedSorted.length + ? ` Fix first: ${confirmedSorted[0].title} (\`${confirmedSorted[0].file}:${confirmedSorted[0].line}\`).` + : ''), + '', + ) + for (const f of confirmedSorted) { + lines.push(`### ${f.severity} - ${f.title}`, '') + lines.push(`\`${f.file}:${f.line}\` - lens \`${f.lens}\`, round ${f.round}, confidence ${f.confidence}`, '') + if (f.why) lines.push(f.why, '') + if (f.repro) lines.push(`**Repro:** ${f.repro}`, '') + if (f.evidence) lines.push(`**Evidence:** ${f.evidence}`, '') + if (f.fix) lines.push(`**Fix:** ${f.fix}`, '') + } + if (unverifiedFinal.length) { + lines.push('## Unverified - re-run', '') + for (const u of unverifiedFinal) lines.push(`- \`${u.file}:${u.line}\` ${u.title} (lens \`${u.lens}\`, round ${u.round})`) + lines.push('') + } + lines.push(table) + lines.push('', '## Run stats', '') + lines.push( + `Total spent: ${runStats.spentTotal} output tokens across ${runStats.rounds} round(s). ` + + `Suppressed by dedupe: ${runStats.suppressedLedger} ledger-known, ${runStats.suppressedRun} same-run. ` + + `Agent failures: ${runStats.finderNull} finder null, ${runStats.finderEmpty} finder empty, ${runStats.verifierNull} verifier null.`, + '', + ) + lines.push('| round | spent | files (new) | suppressed ledger/run | finder null/empty | verifier null |') + lines.push('|---|---|---|---|---|---|') + for (const s of roundStats) + lines.push(`| ${s.round} | ${s.spentAfter - s.spentBefore} | ${s.filesTouched} (${s.filesNew}) | ${s.suppressedLedger}/${s.suppressedRun} | ${s.finderNull}/${s.finderEmpty} | ${s.verifierNull} |`) + return lines.join('\n') +} +const report = buildReport() + +return { converged, stoppedOnBudget, rounds: roundStats, confirmed: confirmedSorted, unverified: unverifiedFinal, runStats, exploredFiles: [...exploreConsumed], report } diff --git a/.gitignore b/.gitignore index cc3c0bcd..808e4c01 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,4 @@ Client/tauri-client/.env # local server run logs server.log +graphify-out/