diff --git a/.github/workflows/frontend-a11y.yml b/.github/workflows/frontend-a11y.yml index d763447048..f4d3e3b93a 100644 --- a/.github/workflows/frontend-a11y.yml +++ b/.github/workflows/frontend-a11y.yml @@ -55,3 +55,6 @@ jobs: path: frontend/.a11y-scan/ retention-days: 7 if-no-files-found: ignore + # The reports live in a dot-directory, which upload-artifact treats as + # hidden and silently skips by default. + include-hidden-files: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 301a1fc8df..50346a6a39 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -92,6 +92,9 @@ jobs: path: frontend/.a11y-scan/ retention-days: 14 if-no-files-found: ignore + # The reports live in a dot-directory, which upload-artifact treats as + # hidden and silently skips by default. + include-hidden-files: true # Builds all desktop platforms on a schedule so the Rust dependency cache is # written on main, where PR and merge-queue tauri builds can restore it. diff --git a/.taskfiles/frontend.yml b/.taskfiles/frontend.yml index 1338aae8e1..8c32efc5e3 100644 --- a/.taskfiles/frontend.yml +++ b/.taskfiles/frontend.yml @@ -214,42 +214,42 @@ tasks: desc: "a11y regression gate over every story: fail only on NEW axe violations" deps: [install, storybook:browser] cmds: - - bash .storybook/a11y-scan.sh + - node .storybook/a11y-scan.mjs - node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt storybook:a11y:changed: - desc: "a11y gate over stories changed vs a base ref (default origin/main)" + desc: "a11y gate over the stories this branch affects (default base origin/main)" summary: | - Scans only the stories this branch touches, which is what pull requests - run — a full scan takes ~30 minutes, far too long to sit in front of every - merge. The nightly job covers the rest of the suite. + Scans the stories a branch affects, which is what pull requests run — a + full scan takes ~30 minutes, far too long to sit in front of every merge. + A story is affected if its file changed, or if a same-named sibling + source file changed (editing Button.tsx or Button.css re-scans + Button.stories.tsx — the story renders the live component, so a component + edit changes what the story shows without touching the story file). + Changes that ripple further than a component's own stories are covered by + the nightly full sweep. Pass a base ref through CLI_ARGS, e.g. task frontend:storybook:a11y:changed -- origin/release deps: [install, storybook:browser] vars: BASE: '{{.CLI_ARGS | default "origin/main"}}' - # Stories touched by this branch, plus any not yet committed. CHANGED: - sh: | - { git diff --name-only --diff-filter=d {{.CLI_ARGS | default "origin/main"}}...HEAD -- '*.stories.ts' '*.stories.tsx'; - git diff --name-only --diff-filter=d -- '*.stories.ts' '*.stories.tsx'; - git ls-files --others --exclude-standard -- '*.stories.ts' '*.stories.tsx'; } \ - | sed 's|^frontend/||' | sort -u | tr '\n' ' ' + sh: node .storybook/a11y-changed.mjs {{.CLI_ARGS | default "origin/main"}} cmds: - cmd: | if [ -z "{{.CHANGED}}" ]; then - echo "a11y: no story files changed vs {{.BASE}} — nothing to check" + echo "a11y: no story files affected vs {{.BASE}} — nothing to check" exit 0 fi - bash .storybook/a11y-scan.sh {{.CHANGED}} + node .storybook/a11y-scan.mjs {{.CHANGED}} node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt storybook:a11y:record: desc: "Re-record the a11y baseline (run after intentionally fixing/adding violations)" deps: [install, storybook:browser] cmds: - - bash .storybook/a11y-scan.sh + - node .storybook/a11y-scan.mjs - node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt --record # ============================================================ diff --git a/frontend/.storybook/a11y-changed.mjs b/frontend/.storybook/a11y-changed.mjs new file mode 100644 index 0000000000..6aacf6274c --- /dev/null +++ b/frontend/.storybook/a11y-changed.mjs @@ -0,0 +1,48 @@ +// Prints the story files a branch affects, one per line — the scan set for the +// pull-request a11y gate. A story is affected if its file changed against the +// base ref (or is uncommitted/untracked), or if a same-named sibling source +// file changed: stories render the live component, so editing Button.tsx or +// Button.css changes what Button.stories.tsx shows without touching it. +// +// node a11y-changed.mjs [base-ref] (default origin/main) +// +// Node rather than shell so the task works no matter what invokes it — Task's +// embedded interpreter runs on Windows, but sed/grep/sort do not exist for +// developers calling tasks from PowerShell. +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +const base = process.argv[2] || "origin/main"; + +const git = (...args) => + execFileSync("git", args, { encoding: "utf8" }) + .split("\n") + .map((l) => l.trim().replace(/^frontend\//, "")) + .filter(Boolean); + +const STORY = /\.stories\.tsx?$/; +const TEST = /\.test\.tsx?$/; +const SOURCE = /\.(ts|tsx|css)$/; + +// Committed changes vs the base, plus working-tree changes, plus untracked +// files — so the gate covers exactly what the branch would merge and what a +// developer is about to commit. +const changed = [ + ...git("diff", "--name-only", "--diff-filter=d", `${base}...HEAD`), + ...git("diff", "--name-only", "--diff-filter=d"), + ...git("ls-files", "--others", "--exclude-standard"), +]; + +const stories = new Set(); +for (const f of changed) { + if (STORY.test(f)) { + stories.add(f); + continue; + } + if (TEST.test(f) || !SOURCE.test(f)) continue; + const sibling = f.replace(SOURCE, ""); + for (const s of [`${sibling}.stories.tsx`, `${sibling}.stories.ts`]) + if (existsSync(s)) stories.add(s); +} + +process.stdout.write([...stories].sort().join("\n")); diff --git a/frontend/.storybook/a11y-scan.mjs b/frontend/.storybook/a11y-scan.mjs new file mode 100644 index 0000000000..77aed79d70 --- /dev/null +++ b/frontend/.storybook/a11y-scan.mjs @@ -0,0 +1,197 @@ +// Runs the Storybook Vitest scan in batches and emits one JSON report per batch +// into .a11y-scan/, plus a manifest of every story file the run was supposed to +// cover and a log of the raw output. Consumed by a11y-check.mjs, which fails if +// any manifest entry produced no results. Run from frontend/. +// +// node a11y-scan.mjs scan every story +// node a11y-scan.mjs [file…] scan only these story files +// +// Batching keeps each browser session small: a single run over the whole story +// set holds one Chromium context open for the entire scan, so one crash in it +// costs every story after that point. +// +// Node rather than shell so the task works no matter what invokes it — Task's +// embedded interpreter runs on Windows, but bash/sed/sort do not exist for +// developers calling tasks from PowerShell. +import { execFileSync, spawn } from "node:child_process"; +import { + existsSync, + mkdirSync, + openSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +const OUT = ".a11y-scan"; +const CHUNK = 20; +const BATCH_TIMEOUT_MS = 300_000; + +const git = (...args) => + execFileSync("git", args, { encoding: "utf8" }) + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + +rmSync(OUT, { recursive: true, force: true }); +mkdirSync(OUT, { recursive: true }); +const manifestFile = join(OUT, "manifest.txt"); +const logFile = join(OUT, "scan.log"); + +const args = process.argv.slice(2); +let files; +if (args.length > 0) { + // Explicit list (the pull-request path passes just the stories a branch + // affects). Anything that no longer exists is dropped, so a deleted story + // doesn't fail the manifest check. + files = [...new Set(args.filter((f) => existsSync(f)))].sort(); + if (files.length === 0) { + console.log( + "a11y-scan: no existing story files in the given list — nothing to scan", + ); + writeFileSync(manifestFile, ""); + process.exit(0); + } +} else { + // Tracked story files plus any not yet committed, so a new story can be + // checked before it is added to the index. + files = [ + ...new Set([ + ...git( + "ls-files", + "--", + "editor/src/**/*.stories.ts", + "editor/src/**/*.stories.tsx", + ), + ...git( + "ls-files", + "--others", + "--exclude-standard", + "--", + "editor/src/**/*.stories.ts", + "editor/src/**/*.stories.tsx", + ), + ]), + ].sort(); + if (files.length === 0) { + console.error("a11y-scan: no story files found — check the glob"); + process.exit(2); + } +} +writeFileSync(manifestFile, files.join("\n") + "\n"); + +/** Failures carrying no axe rule — a throw, a timeout, a dropped browser page. */ +function crashCount(reportFile) { + try { + const report = JSON.parse(readFileSync(reportFile, "utf8")); + let crashes = 0; + for (const tf of report.testResults ?? []) + for (const a of tf.assertionResults ?? []) { + if (a.status === "passed") continue; + const msg = (a.failureMessages ?? []).join("\n"); + if (!/dequeuniversity\.com\/rules\/axe\//.test(msg)) crashes++; + } + return crashes; + } catch { + return -1; // unreadable report counts as a failed attempt + } +} + +/** Runs one vitest batch, tee'd to the log, killed (whole tree) on timeout. */ +function runBatch(filters, outputFile) { + return new Promise((resolve) => { + const cmd = + `npx vitest run --config .storybook/vitest.config.ts ` + + `--reporter=json --outputFile=${outputFile} ` + + filters.map((f) => `"${f}"`).join(" "); + const log = openSync(logFile, "a"); + const child = spawn(cmd, { + shell: true, + detached: process.platform !== "win32", + stdio: ["ignore", log, log], + }); + const timer = setTimeout(() => { + if (process.platform === "win32") { + try { + execFileSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { + stdio: "ignore", + }); + } catch { + /* already gone */ + } + } else { + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + /* already gone */ + } + } + }, BATCH_TIMEOUT_MS); + child.on("exit", () => { + clearTimeout(timer); + resolve(); + }); + }); +} + +const hasReport = (f) => existsSync(f) && statSync(f).size > 0; + +const batches = []; +for (let i = 0; i < files.length; i += CHUNK) + batches.push(files.slice(i, i + CHUNK)); +console.log( + `a11y-scan: ${files.length} story files, ${batches.length} batches of ${CHUNK}`, +); + +const failed = []; +for (let bi = 0; bi < batches.length; bi++) { + const n = bi + 1; + const out = join(OUT, `chunk-${n}.json`); + // The scan exits non-zero whenever a story has a violation — expected here, + // so the report is what matters, not the status. + // + // A batch is retried once when it produced no report, or when its report + // contains crash-class failures. A one-off infrastructure death — the + // browser page dropping, a Vite dep re-optimize reloading mid-run — passes + // on the retry; a story that genuinely cannot render fails both attempts. + const filters = batches[bi].map((f) => f.replace(/\.tsx$/, "")); + for (let attempt = 1; attempt <= 2; attempt++) { + await runBatch(filters, out); + if (!hasReport(out)) { + console.error( + `a11y-scan: batch ${n} produced no report (attempt ${attempt})`, + ); + continue; + } + if (attempt === 1) { + const crashes = crashCount(out); + if (crashes !== 0) { + console.error( + `a11y-scan: batch ${n} has ${crashes} crash-class failure(s) — retrying once`, + ); + rmSync(out, { force: true }); + continue; + } + } + break; + } + if (hasReport(out)) console.log(` batch ${n}/${batches.length} done`); + else { + failed.push(n); + console.error(` batch ${n}/${batches.length} FAILED — no report`); + } +} + +const present = batches.filter((_, i) => + hasReport(join(OUT, `chunk-${i + 1}.json`)), +).length; +console.log(`a11y-scan: ${present}/${batches.length} batches produced reports`); +if (failed.length > 0) { + console.error( + `a11y-scan: ${failed.length} batch(es) produced no report: ${failed.join(" ")}`, + ); + console.error(`a11y-scan: see ${logFile}. Not reporting on a partial scan.`); + process.exit(2); +} diff --git a/frontend/.storybook/a11y-scan.sh b/frontend/.storybook/a11y-scan.sh deleted file mode 100644 index 855a4449dc..0000000000 --- a/frontend/.storybook/a11y-scan.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# Run the Storybook Vitest scan in batches and emit one JSON report per batch -# into .a11y-scan/, plus a manifest of every story file the run was supposed to -# cover and a log of the raw output. Consumed by a11y-check.mjs, which fails if -# any manifest entry produced no results. Run from frontend/. -# -# a11y-scan.sh scan every story -# a11y-scan.sh [file…] scan only these story files -# -# Batching keeps each browser session small: a single run over the whole story -# set holds one Chromium context open for the entire scan, so one crash in it -# costs every story after that point. -set -uo pipefail -cd "$(dirname "$0")/.." || exit 1 - -OUT=".a11y-scan" -rm -rf "$OUT" -mkdir -p "$OUT" -MANIFEST="$OUT/manifest.txt" -LOG="$OUT/scan.log" - -if [ "$#" -gt 0 ]; then - # Explicit list (the pull-request path passes just the stories a branch - # touched). Anything that no longer exists is dropped, so a deleted story - # doesn't fail the manifest check. - for f in "$@"; do [ -f "$f" ] && printf '%s\n' "$f"; done | sort -u >"$MANIFEST" -else - # Tracked story files plus any not yet committed, so a new story can be - # checked before it is added to the index. - { - git ls-files 'editor/src/**/*.stories.ts' 'editor/src/**/*.stories.tsx' - git ls-files --others --exclude-standard 'editor/src/**/*.stories.ts' \ - 'editor/src/**/*.stories.tsx' - } | sort -u >"$MANIFEST" -fi - -mapfile -t FILES <"$MANIFEST" -TOTAL=${#FILES[@]} -if [ "$TOTAL" -eq 0 ]; then - if [ "$#" -gt 0 ]; then - echo "a11y-scan: no existing story files in the given list — nothing to scan" - exit 0 - fi - echo "a11y-scan: no story files found — check the glob" >&2 - exit 2 -fi - -CHUNK=20 -NB=$(((TOTAL + CHUNK - 1) / CHUNK)) -echo "a11y-scan: $TOTAL story files, $NB batches of $CHUNK" - -failed=() -i=0 -ci=0 -while [ "$i" -lt "$TOTAL" ]; do - ci=$((ci + 1)) - batch=("${FILES[@]:i:CHUNK}") - i=$((i + CHUNK)) - out="$OUT/chunk-$ci.json" - filters=() - for f in "${batch[@]}"; do filters+=("${f%.tsx}"); done - # The scan exits non-zero whenever a story has a violation — expected here, so - # the report is what matters, not the status. Output is teed to the log so a red - # CI run still has the offending selectors and help text to work from. - for attempt in 1 2; do - timeout 300 npx vitest run --config .storybook/vitest.config.ts \ - --reporter=json --outputFile="$out" "${filters[@]}" >>"$LOG" 2>&1 - [ -s "$out" ] && break - echo "a11y-scan: batch $ci produced no report (attempt $attempt)" >&2 - done - if [ -s "$out" ]; then - echo " batch $ci/$NB done" - else - failed+=("$ci") - echo " batch $ci/$NB FAILED — no report" >&2 - fi -done - -echo "a11y-scan: $(ls "$OUT"/chunk-*.json 2>/dev/null | wc -l)/$NB batches produced reports" -if [ ${#failed[@]} -gt 0 ]; then - echo "a11y-scan: ${#failed[@]} batch(es) produced no report: ${failed[*]}" >&2 - echo "a11y-scan: see $LOG. Not reporting on a partial scan." >&2 - exit 2 -fi diff --git a/frontend/.storybook/vitest.config.ts b/frontend/.storybook/vitest.config.ts index 8c0d0e81f2..8519e50267 100644 --- a/frontend/.storybook/vitest.config.ts +++ b/frontend/.storybook/vitest.config.ts @@ -18,18 +18,32 @@ export default defineConfig({ // Pre-scan every story + the preview so Vite discovers the story set's large // dep surface (embedpdf plugins, @mui icons, …) in one pass up front. entries: ["editor/src/**/*.stories.@(ts|tsx)", ".storybook/preview.tsx"], - // `entries` alone does not catch deps reached only through a transformed - // JSX runtime import, so Vite optimizes them lazily mid-run and emits - // "optimized dependencies changed, reloading". That reload tears down the - // browser worker and whichever stories were mid-load fail with a bogus - // "Failed to fetch dynamically imported module" — a scan that then looks - // like a real result. Naming them here keeps a run deterministic. + // `entries` alone does not catch deps reached through a transformed JSX + // runtime import, nor the preview's own dependency graph (the test plugin + // injects the preview in a way the entry scanner doesn't crawl). Vite then + // optimizes them lazily mid-run and emits "optimized dependencies changed, + // reloading" — the reload tears down the browser worker and whichever + // stories were mid-load fail with a bogus "Failed to fetch dynamically + // imported module" that reads like a real crash. Only a cold dep cache + // hits this, which is every CI run. Naming them keeps a run deterministic. include: [ "react", "react/jsx-runtime", "react/jsx-dev-runtime", "react-dom", "react-dom/client", + "@storybook/react-vite", + "@storybook/addon-a11y/preview", + "@storybook/addon-themes", + "msw-storybook-addon", + "react-router-dom", + "@tanstack/react-query", + "i18next", + "react-i18next", + "smol-toml", + "@mantine/core", + "@supabase/supabase-js", + "axios", ], }, test: {