diff --git a/.taskfiles/frontend.yml b/.taskfiles/frontend.yml index 8c32efc5e3..3c2113bf96 100644 --- a/.taskfiles/frontend.yml +++ b/.taskfiles/frontend.yml @@ -184,13 +184,13 @@ tasks: storybook: desc: "Start Storybook dev server" - deps: [install] + deps: [prepare] cmds: - npx storybook dev -p 6006 {{.CLI_ARGS}} storybook:build: desc: "Build static Storybook" - deps: [install] + deps: [prepare] cmds: - npx storybook build {{.CLI_ARGS}} @@ -204,7 +204,7 @@ tasks: storybook:test: desc: "Scan every story in real Chromium: it must render and pass axe" - deps: [install, storybook:browser] + deps: [prepare, storybook:browser] cmds: # Runs each story as a browser test. Pass a filter through, e.g. # task frontend:storybook:test -- Button @@ -212,7 +212,7 @@ tasks: storybook:a11y: desc: "a11y regression gate over every story: fail only on NEW axe violations" - deps: [install, storybook:browser] + deps: [prepare, storybook:browser] cmds: - node .storybook/a11y-scan.mjs - node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt @@ -231,14 +231,14 @@ tasks: Pass a base ref through CLI_ARGS, e.g. task frontend:storybook:a11y:changed -- origin/release - deps: [install, storybook:browser] + deps: [prepare, storybook:browser] vars: BASE: '{{.CLI_ARGS | default "origin/main"}}' CHANGED: sh: node .storybook/a11y-changed.mjs {{.CLI_ARGS | default "origin/main"}} cmds: - cmd: | - if [ -z "{{.CHANGED}}" ]; then + if [ -z '{{.CHANGED}}' ]; then echo "a11y: no story files affected vs {{.BASE}} — nothing to check" exit 0 fi @@ -247,7 +247,7 @@ tasks: storybook:a11y:record: desc: "Re-record the a11y baseline (run after intentionally fixing/adding violations)" - deps: [install, storybook:browser] + deps: [prepare, storybook:browser] cmds: - node .storybook/a11y-scan.mjs - node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt --record diff --git a/frontend/.storybook/a11y-check.mjs b/frontend/.storybook/a11y-check.mjs index 7eef5e079e..43092812ff 100644 --- a/frontend/.storybook/a11y-check.mjs +++ b/frontend/.storybook/a11y-check.mjs @@ -52,6 +52,7 @@ const RULE_URL = /dequeuniversity\.com\/rules\/axe\/[\d.]+\/([a-z0-9-]+)/g; function collect(dir) { const rules = {}; // storyKey -> Set(ruleId) const crashed = []; // storyKey[] — failed for a non-a11y reason + const unloadable = []; // storyFile[] — the file itself never ran const seenFiles = new Set(); let scanned = 0; @@ -68,6 +69,14 @@ function collect(dir) { const idx = norm.search(/editor\/src\//); const file = idx >= 0 ? norm.slice(idx) : norm; seenFiles.add(file); + // A story file that fails to import produces a failed suite with no + // assertions at all. Every other check here reads assertions, so such a + // file satisfies the manifest and contributes nothing — its stories go + // unscanned while the run still reports clean. + if ((tf.assertionResults || []).length === 0 && tf.status !== "passed") { + unloadable.push(file); + continue; + } for (const a of tf.assertionResults || []) { scanned++; if (a.status === "passed") continue; @@ -88,14 +97,14 @@ function collect(dir) { } } } - return { rules, crashed, seenFiles, scanned }; + return { rules, crashed, unloadable, seenFiles, scanned }; } if (!existsSync(inDir)) { console.error(`a11y-check: scan dir not found: ${inDir}`); process.exit(2); } -const { rules, crashed, seenFiles, scanned } = collect(inDir); +const { rules, crashed, unloadable, seenFiles, scanned } = collect(inDir); const observed = {}; for (const [k, set] of Object.entries(rules)) observed[k] = [...set].sort(); @@ -129,6 +138,14 @@ if (record || merge) { merge && existsSync(baselineFile) ? JSON.parse(readFileSync(baselineFile, "utf8")) : {}; + if (unloadable.length) { + console.error( + `a11y-check: refusing to record — ${unloadable.length} story file(s) failed to load:`, + ); + unloadable.slice(0, 20).forEach((f) => console.error(` ${f}`)); + console.error("Their stories never ran, so the baseline would lose them."); + process.exit(2); + } if (crashed.length) { console.error( `a11y-check: refusing to record — ${crashed.length} story(ies) failed for a non-a11y reason:`, @@ -176,6 +193,19 @@ console.log( `${pairs} story-rule pairs (baselined).`, ); +if (unloadable.length) { + console.error( + `\n✖ ${unloadable.length} story file(s) failed to load, so their stories never ran:`, + ); + unloadable.slice(0, 50).forEach((f) => console.error(` ${f}`)); + if (unloadable.length > 50) + console.error(` … and ${unloadable.length - 50} more`); + console.error( + "\nA file that cannot be imported reports no violations at all. The resolve " + + "or transform error is in the scan log (.a11y-scan/scan.log, uploaded as a " + + "run artifact); a missing generated asset is the usual cause.", + ); +} if (crashed.length) { console.error(`\n✖ ${crashed.length} story(ies) failed to render:`); crashed.slice(0, 50).forEach((k) => console.error(` ${k}`)); @@ -191,7 +221,7 @@ if (regressions.length) { "baseline key no longer matches — re-record: task frontend:storybook:a11y:record", ); } -if (crashed.length || regressions.length) process.exit(1); +if (unloadable.length || crashed.length || regressions.length) process.exit(1); if (fixed.length) console.log(