diff --git a/.taskfiles/pre-commit.yml b/.taskfiles/pre-commit.yml index 63fdd9ea01..136f852a5b 100644 --- a/.taskfiles/pre-commit.yml +++ b/.taskfiles/pre-commit.yml @@ -4,8 +4,6 @@ version: '3' # pre-commit hook (.pre-commit-config.yaml) and CI (pre_commit.yml) both call. vars: - GITLEAKS: '8.30.0' - # File selections as git pathspecs: git does the include/exclude matching, so # there is no grep/xargs and it behaves identically on every platform. PY_FILES: >- @@ -43,7 +41,9 @@ vars: ':(exclude).github/workflows/*' LOCALE_TOML: 'frontend/editor/public/locales/*/translation.toml' - GITLEAKS_BIN: '.task/bin/gitleaks-{{.GITLEAKS}}{{if eq OS "windows"}}.exe{{end}}' + # gitleaks is pinned + checksum-verified by scripts/pre-commit/install_gitleaks.py, + # which owns the version and caches the binary here. + GITLEAKS_BIN: '.task/bin/gitleaks{{if eq OS "windows"}}.exe{{end}}' tasks: default: @@ -84,22 +84,13 @@ tasks: - test -d scripts/pre-commit/.venv clean: - desc: "Remove the cache/build artifacts" + desc: "Remove the cached gitleaks binary and the tool virtualenv" cmds: - - task: '{{if eq OS "windows"}}clean-windows{{else}}clean-unix{{end}}' - - clean-unix: - internal: true - cmds: - - rm -rf scripts/pre-commit/.venv .task/bin/gitleaks-* - - # On Windows, use PowerShell so it matches the same paths and tolerates absent - # files without erroring. - clean-windows: - internal: true - ignore_error: true - cmds: - - powershell -NoProfile -Command "Remove-Item -Recurse -Force -ErrorAction SilentlyContinue scripts/pre-commit/.venv, .task/bin/gitleaks-*" + - cmd: rm -rf scripts/pre-commit/.venv .task/bin/gitleaks + platforms: [linux, darwin] + - cmd: cmd /c "rmdir /s /q scripts\pre-commit\.venv & del /q .task\bin\gitleaks.exe" + platforms: [windows] + ignore_error: true # Individual checks (hidden from `task --list`, but callable, e.g. # `task pre-commit:toml-sort FIX=1`). Pass FIX=1 to auto-fix where supported. @@ -125,7 +116,7 @@ tasks: whitespace: cmds: - - uv run --no-project python scripts/pre-commit/whitespace.py {{if .FIX}}--fix {{end}}$(git ls-files {{.WS_FILES}}) + - uv run --no-project python scripts/pre-commit/whitespace.py {{if .FIX}}--fix {{end}}{{.WS_FILES}} gitleaks: deps: [gitleaks-bin] @@ -137,23 +128,6 @@ tasks: gitleaks-bin: internal: true - desc: "Ensure the pinned gitleaks binary is cached in .task/bin" - status: - - test -f {{.GITLEAKS_BIN}} - vars: - GL_ARCH: '{{if eq ARCH "amd64"}}x64{{else if eq ARCH "arm64"}}arm64{{else if eq ARCH "386"}}x32{{else}}{{ARCH}}{{end}}' - GL_PLATFORM: '{{OS}}_{{.GL_ARCH}}' - GL_URL: 'https://github.com/gitleaks/gitleaks/releases/download/v{{.GITLEAKS}}/gitleaks_{{.GITLEAKS}}_{{.GL_PLATFORM}}' - # SHA-256 of each release asset, from gitleaks_{{.GITLEAKS}}_checksums.txt. - GL_SHA: >- - {{if eq .GL_PLATFORM "linux_x64"}}79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e - {{- else if eq .GL_PLATFORM "linux_arm64"}}b4cbbb6ddf7d1b2a603088cd03a4e3f7ce48ee7fd449b51f7de6ee2906f5fa2f - {{- else if eq .GL_PLATFORM "darwin_x64"}}ca221d012d247080c2f6f61f4b7a83bffa2453806b0c195c795bbe9a8c775ed5 - {{- else if eq .GL_PLATFORM "darwin_arm64"}}b251ab2bcd4cd8ba9e56ff37698c033ebf38582b477d21ebd86586d927cf87e7 - {{- else if eq .GL_PLATFORM "windows_x64"}}54fe94f644b832dd08e8c3a5915efb3bfa862386d59fb27ca0792cb687a83573 - {{- end}} + desc: "Ensure the pinned, checksum-verified gitleaks binary is cached in .task/bin" cmds: - - cmd: bash scripts/pre-commit/install-gitleaks.sh "{{.GL_URL}}.tar.gz" "{{.GL_SHA}}" "{{.GITLEAKS_BIN}}" - platforms: [linux, darwin] - - cmd: powershell -NoProfile -File scripts/pre-commit/install-gitleaks.ps1 -Url "{{.GL_URL}}.zip" -Sha "{{.GL_SHA}}" -Dest "{{.GITLEAKS_BIN}}" - platforms: [windows] + - uv run --no-project python scripts/pre-commit/install_gitleaks.py diff --git a/scripts/pre-commit/install-gitleaks.ps1 b/scripts/pre-commit/install-gitleaks.ps1 deleted file mode 100644 index ec554bb6cb..0000000000 --- a/scripts/pre-commit/install-gitleaks.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -# Download, checksum-verify and extract the gitleaks binary. -# -# Usage: install-gitleaks.ps1 -Url -Sha -Dest -# -# Called by the pre-commit:gitleaks-bin Task target, which owns the pinned -# version and per-platform checksums and passes the resolved values in. -param( - [Parameter(Mandatory)] [string]$Url, - [Parameter(Mandatory)] [string]$Sha, - [Parameter(Mandatory)] [string]$Dest -) -$ErrorActionPreference = 'Stop' - -if (-not $Sha) { - throw 'No pinned gitleaks checksum for this platform' -} - -New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Dest) | Out-Null - -$archive = New-TemporaryFile -$extract = Join-Path $env:TEMP 'gitleaks-extract' -try { - Invoke-WebRequest -Uri $Url -OutFile $archive - if ((Get-FileHash $archive -Algorithm SHA256).Hash -ne $Sha) { - throw 'gitleaks checksum mismatch' - } - Expand-Archive -Force -Path $archive -DestinationPath $extract - Move-Item -Force -Path (Join-Path $extract 'gitleaks.exe') -Destination $Dest -} finally { - Remove-Item -Force -ErrorAction SilentlyContinue $archive, $extract -Recurse -} diff --git a/scripts/pre-commit/install-gitleaks.sh b/scripts/pre-commit/install-gitleaks.sh deleted file mode 100644 index efafc88f56..0000000000 --- a/scripts/pre-commit/install-gitleaks.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Download, checksum-verify and extract the gitleaks binary. -# -# Usage: install-gitleaks.sh -# -# Called by the pre-commit:gitleaks-bin Task target, which owns the pinned -# version and per-platform checksums and passes the resolved values in. -set -euo pipefail - -url=$1 -sha=$2 -dest=$3 - -if [ -z "$sha" ]; then - echo "No pinned gitleaks checksum for this platform" >&2 - exit 1 -fi - -mkdir -p "$(dirname "$dest")" -archive=$(mktemp) -trap 'rm -f "$archive"' EXIT - -curl -fsSL "$url" -o "$archive" -actual=$(shasum -a 256 "$archive" | awk '{print $1}') -if [ "$actual" != "$sha" ]; then - echo "gitleaks checksum mismatch: expected $sha, got $actual" >&2 - exit 1 -fi -tar -xzO -f "$archive" gitleaks > "$dest" -chmod +x "$dest" diff --git a/scripts/pre-commit/install_gitleaks.py b/scripts/pre-commit/install_gitleaks.py new file mode 100644 index 0000000000..c624132196 --- /dev/null +++ b/scripts/pre-commit/install_gitleaks.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Download the pinned gitleaks binary into .task/bin, verifying its checksum. + +gitleaks is a Go binary with no PyPI package, so it can't be locked like the +other tools (ruff/codespell/toml-sort live in scripts/pre-commit/pyproject.toml). +This script is the single source of truth for the gitleaks version and the +SHA-256 of each release asset. It is cross-platform (stdlib only) and idempotent: +if the cached binary already reports the pinned version it does nothing, so +`task pre-commit` can call it every run. + +Bump the version by editing VERSION and the SHA256 map (values come from the +release's gitleaks__checksums.txt). +""" + +from __future__ import annotations + +import hashlib +import platform +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +VERSION = "8.30.0" + +# SHA-256 of each release asset, keyed by "_" (gitleaks' own naming). +SHA256 = { + "linux_x64": "79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e", + "linux_arm64": "b4cbbb6ddf7d1b2a603088cd03a4e3f7ce48ee7fd449b51f7de6ee2906f5fa2f", + "darwin_x64": "ca221d012d247080c2f6f61f4b7a83bffa2453806b0c195c795bbe9a8c775ed5", + "darwin_arm64": "b251ab2bcd4cd8ba9e56ff37698c033ebf38582b477d21ebd86586d927cf87e7", + "windows_x64": "54fe94f644b832dd08e8c3a5915efb3bfa862386d59fb27ca0792cb687a83573", +} + +REPO_ROOT = Path(__file__).resolve().parents[2] +IS_WINDOWS = platform.system() == "Windows" +BIN = REPO_ROOT / ".task" / "bin" / ("gitleaks.exe" if IS_WINDOWS else "gitleaks") + + +def platform_key() -> str: + os_name = {"Linux": "linux", "Darwin": "darwin", "Windows": "windows"}.get( + platform.system() + ) + arch = { + "x86_64": "x64", + "amd64": "x64", + "arm64": "arm64", + "aarch64": "arm64", + "i386": "x32", + "i686": "x32", + "x86": "x32", + "armv7l": "armv7", + "armv6l": "armv6", + }.get(platform.machine().lower()) + if not os_name or not arch: + raise SystemExit( + f"Unsupported platform for gitleaks: {platform.system()}/{platform.machine()}" + ) + return f"{os_name}_{arch}" + + +def cached_version() -> str | None: + if not BIN.exists(): + return None + try: + return subprocess.run( + [str(BIN), "version"], capture_output=True, text=True + ).stdout.strip() + except OSError: + return None + + +def main() -> int: + if cached_version() == VERSION: + return 0 + + key = platform_key() + expected = SHA256.get(key) + if expected is None: + raise SystemExit(f"No pinned gitleaks checksum for {key}") + + suffix = "zip" if key.startswith("windows") else "tar.gz" + asset = f"gitleaks_{VERSION}_{key}.{suffix}" + url = f"https://github.com/gitleaks/gitleaks/releases/download/v{VERSION}/{asset}" + print(f"Downloading gitleaks {VERSION} ({asset})", flush=True) + + BIN.parent.mkdir(parents=True, exist_ok=True) + archive, _ = urllib.request.urlretrieve(url) + digest = hashlib.sha256(Path(archive).read_bytes()).hexdigest() + if digest != expected: + raise SystemExit( + f"gitleaks checksum mismatch: expected {expected}, got {digest}" + ) + + member = "gitleaks.exe" if IS_WINDOWS else "gitleaks" + if suffix == "zip": + with zipfile.ZipFile(archive) as zf: + data = zf.read(member) + else: + with tarfile.open(archive) as tf: + extracted = tf.extractfile(member) + if extracted is None: + raise SystemExit(f"{member} not found in {asset}") + data = extracted.read() + BIN.write_bytes(data) + BIN.chmod(0o755) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pre-commit/whitespace.py b/scripts/pre-commit/whitespace.py index df77d3728d..e3f27c41c3 100644 --- a/scripts/pre-commit/whitespace.py +++ b/scripts/pre-commit/whitespace.py @@ -3,11 +3,14 @@ Replaces the end-of-file-fixer / trailing-whitespace pre-commit hooks, which have no read-only mode. Run via `task pre-commit` (check) and `task -pre-commit:fix`; Task selects the files (with `git ls-files`) and passes them -as arguments. +pre-commit:fix`. - python scripts/whitespace.py ... # check: report, exit 1 if any need fixing - python scripts/whitespace.py --fix ... # fix: rewrite in place +Takes git pathspecs (not a file list) and runs `git ls-files` itself, so the +matched files never hit the command line - on Windows that list can be ~66KB +and exceed the ~32KB CreateProcess argv limit. + + python whitespace.py ... # check: report, exit 1 if any need fixing + python whitespace.py --fix ... # fix: rewrite in place Operates on bytes and only ever touches trailing spaces/tabs and the final newline, so it never mangles content or line endings. Binary files (those with @@ -16,10 +19,21 @@ a NUL byte) are skipped. from __future__ import annotations +import subprocess import sys from pathlib import Path +def tracked_files(pathspecs: list[str]) -> list[str]: + result = subprocess.run( + ["git", "ls-files", "-z", *pathspecs], + check=True, + capture_output=True, + text=True, + ) + return [path for path in result.stdout.split("\0") if path] + + def normalise(data: bytes) -> bytes: # Strip trailing spaces/tabs from each line (leave \r so CRLF survives). lines = [line.rstrip(b" \t") for line in data.split(b"\n")] @@ -32,10 +46,10 @@ def normalise(data: bytes) -> bytes: def main() -> int: args = sys.argv[1:] fix = "--fix" in args - paths = [a for a in args if a != "--fix"] + pathspecs = [a for a in args if a != "--fix"] offenders: list[str] = [] - for path in paths: + for path in tracked_files(pathspecs): data = Path(path).read_bytes() if b"\0" in data: continue