diff --git a/.claude/skills/ci-check/SKILL.md b/.claude/skills/ci-check/SKILL.md index 72b6ed63..ab397b68 100644 --- a/.claude/skills/ci-check/SKILL.md +++ b/.claude/skills/ci-check/SKILL.md @@ -27,8 +27,12 @@ Add `-tags wazero` to `go vet`/`go test` when you touched `plugin/`. A `windows-latest` `-race` failure inside `ws` that matches `runtime.scanstack` or `runtime.(*unwinder).next` is a Go 1.26.5 runtime GC fault, not your change. -Rerun the job (`gh run rerun --job `); a job cannot be rerun while its -parent run is still in progress. +The Go 1.26.6 toolchain shows a variant signature: `unexpected fault address +0xffffffffffffffff` / `fatal error: fault` (signal 0xc0000005) inside ordinary +stdlib frames such as `log/slog.(*Logger).Enabled` — same spurious runtime +fault, same verdict, especially when the diff touches no Go code. Rerun the +job (`gh run rerun --job `); a job cannot be rerun while its parent run is +still in progress. ## Client (from `Client/tauri-client/`) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06bc0cf..517c56d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,9 +109,14 @@ jobs: working-directory: Server/ verify: false + # ubuntu-latest deliberately: the client TS code has zero win32-conditional + # paths (no process.platform / path.sep branches in src or the unit suites), + # prettier pins endOfLine: lf and .gitattributes forces eol=lf, so a Windows + # runner adds queue time without adding coverage. Windows-specific behavior + # is covered where it exists: rust-tests and the tauri-build matrix. client-check: name: Client Static Checks - runs-on: windows-latest + runs-on: ubuntu-latest defaults: run: working-directory: Client/tauri-client/ @@ -165,9 +170,11 @@ jobs: # Unit tests live in their own job so a suite failure is visible as exactly one # failing check instead of masking the static gates above. The suite is GREEN # and must stay green — never "fix" a failing test by editing its assertions. + # ubuntu-latest for the same reason as client-check above: jsdom-only vitest + # with no platform-conditional code under test. client-tests: name: Client Unit Tests - runs-on: windows-latest + runs-on: ubuntu-latest defaults: run: working-directory: Client/tauri-client/ @@ -292,6 +299,12 @@ jobs: # channel CRUD, audit log and re-login — the one DC-04 surface the mocked # suites cannot reach. Non-blocking while it earns its soak, same # graduation convention client-e2e followed. + # GRADUATION CRITERION (recorded 2026-08-15): flip continue-on-error to + # false once the job has ~30 consecutive green runs on main with no + # infra-flake reruns — the same evidence bar client-e2e cleared (270+ green + # runs cited in docs/audit-2026-08-04-docs-and-coverage.md) scaled to this + # job's lower traffic. Check with: gh run list -w CI -b main --json + # conclusion | jq '[.[] | .conclusion] | index("failure")'. admin-e2e: name: Admin Panel E2E (real server, non-blocking) runs-on: ubuntu-latest diff --git a/.github/workflows/load-baseline.yml b/.github/workflows/load-baseline.yml new file mode 100644 index 00000000..ed58cf49 --- /dev/null +++ b/.github/workflows/load-baseline.yml @@ -0,0 +1,131 @@ +# Manual WebSocket load baseline against a locally booted server. +# +# workflow_dispatch ONLY, and deliberately not part of the blocking CI matrix: +# a perf run on shared runners is a flake source and a wall-clock tax the +# 10-job/~15-min pipeline doesn't need. Run it before/after changes to the +# hub, the write path, or the replay budget, and compare the uploaded +# k6-summary.json + metrics snapshot between runs. Runner-grade hardware is +# NOT a capacity promise for real deployments — treat results as relative +# (before vs after), not absolute. +name: Load Baseline + +on: + workflow_dispatch: + inputs: + users: + description: "Load-test users to register (max VUs in the script is 100)" + required: false + default: "100" + +permissions: + contents: read + +jobs: + k6-baseline: + name: k6 WebSocket baseline + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version: "1.26" + cache-dependency-path: Server/go.sum + + - name: Build server + working-directory: Server + env: + CGO_ENABLED: "0" + run: go build -o chatserver . + + - name: Boot server + working-directory: Server + env: + # Registration/login are per-IP rate limited (3/min and 5/min by + # default) and every VU logs in from 127.0.0.1 — scale the auth + # limits up with the knob that exists for shared-IP scenarios. + OWNCORD_SECURITY_AUTH_RATE_LIMIT_MULTIPLIER: "100" + run: | + mkdir -p "$RUNNER_TEMP/loadtest" + cp chatserver "$RUNNER_TEMP/loadtest/" + cd "$RUNNER_TEMP/loadtest" + ./chatserver > server.log 2>&1 & + echo $! > server.pid + for _ in $(seq 1 30); do + sleep 1 + if ./chatserver healthcheck; then exit 0; fi + done + echo "::error::server never became healthy" + tail -50 server.log + exit 1 + + - name: Seed owner, channel, and load-test users + working-directory: Server + run: | + BASE=https://127.0.0.1:8443 + TOKEN=$(curl -sk -X POST "$BASE/admin/api/setup" \ + -H 'Content-Type: application/json' \ + -d '{"username":"loadadmin","password":"LoadTest123!Admin"}' | jq -r .token) + [ -n "$TOKEN" ] && [ "$TOKEN" != "null" ] || { echo "::error::setup failed"; exit 1; } + + CHANNEL_ID=$(curl -sk -X POST "$BASE/admin/api/channels" \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d '{"name":"loadtest","type":"text"}' | jq -r .id) + [ -n "$CHANNEL_ID" ] && [ "$CHANNEL_ID" != "null" ] || { echo "::error::channel create failed"; exit 1; } + echo "CHANNEL_ID=$CHANNEL_ID" >> "$GITHUB_ENV" + echo "ADMIN_TOKEN=$TOKEN" >> "$GITHUB_ENV" + + INVITE=$(curl -sk -X POST "$BASE/api/v1/invites" \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d '{"max_uses":0}' | jq -r .code) + [ -n "$INVITE" ] && [ "$INVITE" != "null" ] || { echo "::error::invite create failed"; exit 1; } + + USERS="${{ inputs.users }}" + for i in $(seq 1 "${USERS:-100}"); do + code=$(curl -sk -o /dev/null -w '%{http_code}' -X POST "$BASE/api/v1/auth/register" \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"loadtest$i\",\"password\":\"LoadTest123!\",\"invite_code\":\"$INVITE\"}") + if [ "$code" != "200" ] && [ "$code" != "201" ]; then + echo "::error::registering loadtest$i failed with $code" + exit 1 + fi + done + + - name: Install k6 + run: | + curl -fsSL https://dl.k6.io/key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/k6-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update -qq && sudo apt-get install -y k6 + + - name: Run k6 baseline + working-directory: Server/scripts/k6 + env: + K6_WS_URL: wss://127.0.0.1:8443/api/v1/ws + K6_HTTP_URL: https://127.0.0.1:8443 + K6_CHANNEL_ID: ${{ env.CHANNEL_ID }} + run: | + mkdir -p reports + k6 run --insecure-skip-tls-verify ws-load.js + + - name: Snapshot server metrics + if: always() + run: | + curl -sk https://127.0.0.1:8443/api/v1/metrics | tee "$RUNNER_TEMP/loadtest/metrics-after.json" || true + + - name: Stop server + if: always() + run: | + kill "$(cat "$RUNNER_TEMP/loadtest/server.pid")" 2>/dev/null || true + sleep 3 + + - name: Upload results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: k6-baseline + path: | + Server/scripts/k6/reports/k6-summary.json + ${{ runner.temp }}/loadtest/metrics-after.json + ${{ runner.temp }}/loadtest/server.log + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7201c448..56546acd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,15 @@ on: tags: - "v*" +# A deleted-and-re-pushed tag (it has happened — see the checksum note in the +# publish job) must not race two publish runs: `gh release create` fails +# loudly on the second run, but the ghcr :latest push does not, and which run +# wins it would be arbitrary. Queue, never cancel — a half-cancelled release +# is worse than a slow one. +concurrency: + group: release-${{ github.ref_name }} + cancel-in-progress: false + jobs: # The v1.1.0-alpha.4 release shipped clients still versioned 1.1.0-alpha.3 # because the client manifests weren't bumped before tagging — deployed @@ -223,6 +232,40 @@ jobs: CGO_ENABLED: "0" run: go build -o chatserver -ldflags "-s -w -X main.version=$VERSION" . + # Boot-smoke the EXACT artifact that ships: this feed drives the signed + # self-update, so a binary that compiles but dies on boot would deploy + # itself to every auto-updating instance. CI's tests exercise the same + # commit but never this build (release ldflags, CGO_ENABLED=0) and never + # execute the produced binary. First run writes config.yaml, generates a + # self-signed cert, migrates a fresh SQLite DB — a real cold boot. + - name: Boot-smoke server binary + shell: bash + working-directory: Server + run: | + SMOKE_DIR="$RUNNER_TEMP/owncord-smoke" + mkdir -p "$SMOKE_DIR" + cd "$SMOKE_DIR" + BIN="$GITHUB_WORKSPACE/Server/chatserver" + [ -f "$GITHUB_WORKSPACE/Server/chatserver.exe" ] && BIN="$GITHUB_WORKSPACE/Server/chatserver.exe" + "$BIN" & + SERVER_PID=$! + ok=0 + for _ in $(seq 1 30); do + sleep 1 + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "::error::server process exited during boot smoke" + exit 1 + fi + if "$BIN" healthcheck; then ok=1; break; fi + done + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + if [ "$ok" != "1" ]; then + echo "::error::server never reported healthy within 30s" + exit 1 + fi + echo "boot smoke passed" + - name: Create tar.gz (Linux) if: matrix.os == 'ubuntu-latest' working-directory: Server @@ -378,6 +421,39 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=raw,value=latest + # Build locally first so the image can be boot-smoked BEFORE anything + # is pushed — a pushed :latest that dies on boot deploys itself to every + # `docker compose pull` upgrade. + - name: Build image (local, for smoke test) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: Server/ + load: true + build-args: VERSION=${{ env.VERSION }} + tags: owncord-smoke:candidate + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Boot-smoke Docker image + run: | + docker run -d --name owncord-smoke owncord-smoke:candidate + ok=0 + for _ in $(seq 1 30); do + sleep 1 + if [ "$(docker inspect -f '{{.State.Running}}' owncord-smoke)" != "true" ]; then + echo "::error::container exited during boot smoke" + docker logs owncord-smoke + exit 1 + fi + if docker exec owncord-smoke /chatserver healthcheck; then ok=1; break; fi + done + docker logs owncord-smoke + docker rm -f owncord-smoke + if [ "$ok" != "1" ]; then + echo "::error::container never reported healthy within 30s" + exit 1 + fi + - name: Build and push uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: diff --git a/Server/admin/backup_maintenance.go b/Server/admin/backup_maintenance.go new file mode 100644 index 00000000..393cfb09 --- /dev/null +++ b/Server/admin/backup_maintenance.go @@ -0,0 +1,189 @@ +package admin + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/owncord/server/db" +) + +// Scheduled-backup intervals for the backup_schedule setting values the admin +// UI offers. "off" (or anything unrecognised) disables scheduling. +const ( + backupIntervalDaily = 24 * time.Hour + backupIntervalWeekly = 7 * 24 * time.Hour +) + +// MaintainBackups implements the backup_schedule / backup_retention settings +// the admin panel has always offered. It is driven by main.go's 15-minute +// maintenance loop, mirroring the expired-session sweep: read the settings +// each tick, take a scheduled backup when the newest backup on disk is older +// than the schedule interval, and prune backups past the retention window. +// +// Freshness is judged by the newest *.db file's mtime, manual backups +// included — an operator who clicked "Backup now" this morning does not need +// a second copy tonight. Retention prunes by mtime in whole days, but never +// removes the newest backup, so a long-dead schedule cannot delete the last +// copy in the directory. +// +// The returned error feeds the maintenance loop's circuit breaker; settings +// simply not existing (fresh DB mid-migration) is not an error. +func MaintainBackups(ctx context.Context, database *db.DB) error { + schedule, err := database.GetSetting(ctx, "backup_schedule") + if err != nil { + if errors.Is(err, db.ErrNotFound) { + return nil + } + return fmt.Errorf("MaintainBackups: reading backup_schedule: %w", err) + } + + var interval time.Duration + switch strings.ToLower(strings.TrimSpace(schedule)) { + case "daily": + interval = backupIntervalDaily + case "weekly": + interval = backupIntervalWeekly + } + + var firstErr error + if interval > 0 { + if err := runScheduledBackup(ctx, database, interval); err != nil { + slog.Warn("scheduled backup failed", "error", err) + firstErr = err + } + } + + if err := pruneExpiredBackups(ctx, database); err != nil { + slog.Warn("backup retention pruning failed", "error", err) + if firstErr == nil { + firstErr = err + } + } + return firstErr +} + +// runScheduledBackup takes a backup when the newest existing one is older +// than interval (or none exists). +func runScheduledBackup(ctx context.Context, database *db.DB, interval time.Duration) error { + newest, _, err := scanBackups() + if err != nil { + return err + } + if !newest.IsZero() && time.Since(newest) < interval { + return nil + } + + if err := os.MkdirAll(backupBaseDir, 0o750); err != nil { + return fmt.Errorf("creating backup dir: %w", err) + } + // VACUUM INTO refuses an existing destination, and the timestamp only has + // second resolution — suffix on collision instead of failing the tick. + base := "scheduled_" + time.Now().UTC().Format("20060102_150405") + name := base + ".db" + path := filepath.Join(backupBaseDir, name) + for i := 2; ; i++ { + if _, err := os.Stat(path); err != nil { + // ENOENT is the free-slot case. Any OTHER stat error (EACCES on + // the dir, an unreadable mount) cannot be fixed by trying more + // suffixes — stop probing and let VACUUM INTO surface the real + // failure with a legible error instead of spinning this loop. + break + } + if i > 100 { + return fmt.Errorf("scheduled backup: no free filename after %s (tried 100 suffixes)", base) + } + name = fmt.Sprintf("%s_%d.db", base, i) + path = filepath.Join(backupBaseDir, name) + } + if err := database.BackupToSafe(ctx, path, backupBaseDir); err != nil { + return err + } + if err := db.CheckBackupIntegrity(ctx, path); err != nil { + _ = os.Remove(path) + return fmt.Errorf("scheduled backup failed verification: %w", err) + } + + slog.Info("scheduled backup created", "name", name) + // Actor 0 = system, same audit action the manual handler writes. + db.WriteAudit(ctx, database, 0, "backup_create", "server", 0, "scheduled backup saved: "+name) + return nil +} + +// pruneExpiredBackups deletes *.db backups whose mtime is older than the +// backup_retention window (in days), always keeping the newest one. +func pruneExpiredBackups(ctx context.Context, database *db.DB) error { + retStr, err := database.GetSetting(ctx, "backup_retention") + if err != nil { + if errors.Is(err, db.ErrNotFound) { + return nil + } + return fmt.Errorf("reading backup_retention: %w", err) + } + // Malformed values parse to 0; zero-or-below means retention is off, so a + // typo disables pruning rather than failing the maintenance tick. + days, _ := strconv.Atoi(strings.TrimSpace(retStr)) + if days <= 0 { + return nil + } + + newest, entries, err := scanBackups() + if err != nil || len(entries) == 0 { + return err + } + cutoff := time.Now().Add(-time.Duration(days) * 24 * time.Hour) + pruned := 0 + for _, e := range entries { + if e.mtime.Before(cutoff) && !e.mtime.Equal(newest) { + if rmErr := os.Remove(e.path); rmErr != nil { + slog.Warn("backup retention: failed to remove expired backup", "path", e.path, "error", rmErr) + continue + } + pruned++ + } + } + if pruned > 0 { + slog.Info("backup retention: pruned expired backups", "count", pruned, "retention_days", days) + db.WriteAudit(ctx, database, 0, "backup_delete", "server", 0, + fmt.Sprintf("retention pruned %d backup(s) older than %d days", pruned, days)) + } + return nil +} + +type backupFile struct { + path string + mtime time.Time +} + +// scanBackups lists *.db files in the backup dir, returning the newest mtime +// and all entries. A missing directory is "no backups", not an error. +func scanBackups() (newest time.Time, files []backupFile, err error) { + entries, err := os.ReadDir(backupBaseDir) + if err != nil { + if os.IsNotExist(err) { + return time.Time{}, nil, nil + } + return time.Time{}, nil, fmt.Errorf("reading backup dir: %w", err) + } + for _, e := range entries { + if e.IsDir() || filepath.Ext(e.Name()) != ".db" { + continue + } + info, infoErr := e.Info() + if infoErr != nil { + continue + } + mt := info.ModTime() + files = append(files, backupFile{path: filepath.Join(backupBaseDir, e.Name()), mtime: mt}) + if mt.After(newest) { + newest = mt + } + } + return newest, files, nil +} diff --git a/Server/admin/backup_maintenance_test.go b/Server/admin/backup_maintenance_test.go new file mode 100644 index 00000000..359855a6 --- /dev/null +++ b/Server/admin/backup_maintenance_test.go @@ -0,0 +1,152 @@ +package admin_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/owncord/server/admin" +) + +func listBackupFiles(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + t.Fatalf("ReadDir: %v", err) + } + var names []string + for _, e := range entries { + if filepath.Ext(e.Name()) == ".db" { + names = append(names, e.Name()) + } + } + return names +} + +func backdate(t *testing.T, path string, age time.Duration) { + t.Helper() + old := time.Now().Add(-age) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatalf("Chtimes: %v", err) + } +} + +// TestMaintainBackups_ScheduleAndRetention exercises the full settings-driven +// lifecycle: off is a no-op, daily creates one backup and only one, staleness +// triggers the next, and retention prunes expired backups while always +// keeping the newest. +func TestMaintainBackups_ScheduleAndRetention(t *testing.T) { + database := openAdminTestDB(t) + dir := t.TempDir() + admin.SetBackupBaseDir(dir) + t.Cleanup(func() { admin.SetBackupBaseDir(filepath.Join("data", "backups")) }) + ctx := context.Background() + + // Settings absent → no-op, no error. + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups with no settings: %v", err) + } + if got := listBackupFiles(t, dir); len(got) != 0 { + t.Fatalf("no-settings tick created files: %v", got) + } + + // Schedule off → still a no-op. + mustSetSetting(t, database, "backup_schedule", "off") + mustSetSetting(t, database, "backup_retention", "7") + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups with schedule=off: %v", err) + } + if got := listBackupFiles(t, dir); len(got) != 0 { + t.Fatalf("schedule=off created files: %v", got) + } + + // Daily → first tick creates exactly one scheduled backup. + mustSetSetting(t, database, "backup_schedule", "daily") + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups daily #1: %v", err) + } + files := listBackupFiles(t, dir) + if len(files) != 1 || !strings.HasPrefix(files[0], "scheduled_") { + t.Fatalf("after first daily tick files = %v, want one scheduled_*.db", files) + } + first := filepath.Join(dir, files[0]) + + // Fresh backup on disk → next tick is a no-op. + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups daily #2: %v", err) + } + if got := listBackupFiles(t, dir); len(got) != 1 { + t.Fatalf("fresh-backup tick changed files: %v", got) + } + + // Backup older than a day (but inside retention) → a new one is taken and + // the old one is kept. + backdate(t, first, 25*time.Hour) + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups daily #3: %v", err) + } + if got := listBackupFiles(t, dir); len(got) != 2 { + t.Fatalf("stale-backup tick files = %v, want 2", got) + } + + // Old backup past the 7-day retention window → pruned; the fresh one stays. + backdate(t, first, 8*24*time.Hour) + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups daily #4: %v", err) + } + got := listBackupFiles(t, dir) + if len(got) != 1 { + t.Fatalf("retention tick files = %v, want 1", got) + } + if filepath.Join(dir, got[0]) == first { + t.Fatalf("retention pruned the newest backup instead of the expired one") + } +} + +// TestMaintainBackups_RetentionNeverDeletesNewest locks the safety rule: even +// when every backup is past retention, the newest survives. +func TestMaintainBackups_RetentionNeverDeletesNewest(t *testing.T) { + database := openAdminTestDB(t) + dir := t.TempDir() + admin.SetBackupBaseDir(dir) + t.Cleanup(func() { admin.SetBackupBaseDir(filepath.Join("data", "backups")) }) + ctx := context.Background() + + mustSetSetting(t, database, "backup_schedule", "off") + mustSetSetting(t, database, "backup_retention", "7") + + // Two ancient backups, one slightly newer than the other. + older := filepath.Join(dir, "chatserver_a.db") + newer := filepath.Join(dir, "chatserver_b.db") + for _, p := range []string{older, newer} { + if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + backdate(t, older, 30*24*time.Hour) + backdate(t, newer, 20*24*time.Hour) + + if err := admin.MaintainBackups(ctx, database); err != nil { + t.Fatalf("MaintainBackups: %v", err) + } + got := listBackupFiles(t, dir) + if len(got) != 1 || got[0] != "chatserver_b.db" { + t.Fatalf("files = %v, want only chatserver_b.db (newest kept)", got) + } +} + +func mustSetSetting(t *testing.T, database interface { + SetSetting(ctx context.Context, key, value string) error +}, key, value string, +) { + t.Helper() + if err := database.SetSetting(context.Background(), key, value); err != nil { + t.Fatalf("SetSetting(%s): %v", key, err) + } +} diff --git a/Server/admin/export_test.go b/Server/admin/export_test.go index 84098395..5e013739 100644 --- a/Server/admin/export_test.go +++ b/Server/admin/export_test.go @@ -36,6 +36,18 @@ func SetSetupLimiterReapTiming(interval, maxWindow time.Duration) (restore func( // at a temp dir. Lives here so it stays out of the production binary. func SetBackupBaseDir(dir string) { backupBaseDir = dir } +// StubCopyBackup swaps the restore path's file-copy hook so tests can inject +// mid-copy failures that pass the pre-copy integrity gate. CopyBackupForTest +// is the real implementation, for stubs that only want to fail once. +func StubCopyBackup(fn func(src, dst string) error) (restore func()) { + prev := copyBackupFile + copyBackupFile = fn + return func() { copyBackupFile = prev } +} + +// CopyBackupForTest exposes the real copyFile for StubCopyBackup delegates. +var CopyBackupForTest = copyFile + // StubCloseError makes the next handleRestoreBackup call's database.Close() // return err instead of actually closing the pools, so tests can exercise the // Close-failure branch without a genuine driver-level close error (see diff --git a/Server/admin/handlers_backup.go b/Server/admin/handlers_backup.go index 6a973816..90f55f44 100644 --- a/Server/admin/handlers_backup.go +++ b/Server/admin/handlers_backup.go @@ -30,15 +30,30 @@ const ( // backupBaseDir is the directory for backup files, resolved to an absolute // path at package init time so handlers don't depend on the process CWD (L14). +// Overridden at startup via SetBackupDir with cfg.Backup.Dir. var backupBaseDir string func init() { - abs, err := filepath.Abs(filepath.Join("data", "backups")) - if err == nil { - backupBaseDir = abs - } else { - backupBaseDir = filepath.Join("data", "backups") + backupBaseDir = absOrRaw(filepath.Join("data", "backups")) +} + +// SetBackupDir points every backup handler and the scheduled-backup +// maintenance at the operator-configured directory. Call once at startup with +// cfg.Backup.Dir (main.go, next to SetDatabasePath); tests use it to isolate +// a temp dir. Mirrors SetDatabasePath: without it, a configured backup.dir +// would be ignored while backups keep landing in the default location. +func SetBackupDir(dir string) { + if dir == "" { + return } + backupBaseDir = absOrRaw(dir) +} + +func absOrRaw(p string) string { + if abs, err := filepath.Abs(p); err == nil { + return abs + } + return p } // dbFilePath is the live SQLite database file that "Restore backup" @@ -73,12 +88,22 @@ func handleBackup(database *db.DB) http.Handler { // Detached like the restore path's safety backup: an interrupted // VACUUM INTO leaves a truncated .db that handleListBackups would - // present as restorable. - if err := database.BackupTo(context.WithoutCancel(r.Context()), backupPath); err != nil { + // present as restorable. BackupToSafe is rooted at the configured + // backup dir (SetBackupDir), not the historical hardcoded default. + if err := database.BackupToSafe(context.WithoutCancel(r.Context()), backupPath, backupDir); err != nil { writeErr(w, http.StatusInternalServerError, "INTERNAL_ERROR", "backup failed") return } + // Verify before reporting success: a backup that fails integrity_check + // is worse than no backup, because the operator believes they have one. + if err := db.CheckBackupIntegrity(context.WithoutCancel(r.Context()), backupPath); err != nil { + slog.Error("backup failed integrity check — removing", "path", backupPath, "err", err) + _ = os.Remove(backupPath) + writeErr(w, http.StatusInternalServerError, "INTERNAL_ERROR", "backup failed verification") + return + } + actor := actorFromContext(r) backupName := filepath.Base(backupPath) slog.Info("database backup created", "actor_id", actor, "name", backupName) @@ -191,6 +216,16 @@ func handleRestoreBackup(database *db.DB, hub HubBroadcaster) http.Handler { return } + // Refuse to overwrite the live database with a file SQLite itself + // rejects — a truncated pre-crash backup, a stray non-database .db. + // The pre-restore safety copy would make this survivable, but "restore + // succeeded" followed by a broken server is still the worst UX here. + if err := db.CheckBackupIntegrity(context.WithoutCancel(r.Context()), target); err != nil { + slog.Error("restore refused: backup failed integrity check", "backup", name, "err", err) + writeErr(w, http.StatusBadRequest, "BAD_REQUEST", "backup file failed integrity verification") + return + } + dbPath := dbFilePath actor := actorFromContext(r) @@ -215,7 +250,7 @@ func handleRestoreBackup(database *db.DB, hub HubBroadcaster) http.Handler { // a server started from another working directory writes it somewhere // the operator will never find it. preRestore := filepath.Join(backupBaseDir, "pre_restore_"+time.Now().UTC().Format("20060102_150405")+".db") - if err := database.BackupTo(context.WithoutCancel(r.Context()), preRestore); err != nil { + if err := database.BackupToSafe(context.WithoutCancel(r.Context()), preRestore, backupBaseDir); err != nil { // Fail closed. The admin panel promises "a pre-restore backup will // be created" before an irreversible overwrite; proceeding without // one takes away the safety net the operator was shown, exactly @@ -254,7 +289,7 @@ func handleRestoreBackup(database *db.DB, hub HubBroadcaster) http.Handler { // Stream the backup file over the (now closed) database to avoid loading // the entire DB into memory (could be hundreds of MiB). - if err := copyFile(target, dbPath); err != nil { + if err := copyBackupFile(target, dbPath); err != nil { // copyFile truncates the destination with os.Create before it can know // whether the read will succeed, so the live database file is already // destroyed by the time we get here — and the DB is closed, so nothing @@ -262,7 +297,7 @@ func handleRestoreBackup(database *db.DB, hub HubBroadcaster) http.Handler { // leaving the operator with a zero-byte database. slog.Error("restore copy failed — rolling back to the pre-restore safety copy", "backup", name, "err", err) msg := "failed to restore database file — the pre-restore safety copy was put back, server restarting" - if rbErr := copyFile(preRestore, dbPath); rbErr != nil { + if rbErr := copyBackupFile(preRestore, dbPath); rbErr != nil { slog.Error("rollback from the pre-restore safety copy failed — recover manually", "safety_copy", preRestore, "err", rbErr) msg = "failed to restore database file AND failed to roll back — recover manually from " + filepath.Base(preRestore) @@ -356,6 +391,11 @@ func restartProcess(reason string) { os.Exit(0) //nolint:gocritic // backstop if the SIGTERM handler didn't exit } +// copyBackupFile is the restore path's file-copy hook. It exists as a var so +// tests can inject the hard-to-simulate mid-copy failure (truncate-then-fail) +// the rollback branch exists for; production never swaps it. +var copyBackupFile = copyFile + // copyFile streams src to dst without loading the entire file into memory. func copyFile(src, dst string) error { in, err := os.Open(src) //nolint:gosec // G703: src is from sanitized backup path diff --git a/Server/admin/handlers_backup_test.go b/Server/admin/handlers_backup_test.go index de48e7dc..8344f606 100644 --- a/Server/admin/handlers_backup_test.go +++ b/Server/admin/handlers_backup_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "os" "path/filepath" @@ -267,12 +268,13 @@ func TestHandleRestoreBackup_Success(t *testing.T) { t.Fatalf("MkdirAll data: %v", err) } - // Write content as the "backup" to restore from. + // A real SQLite backup to restore from — the handler now verifies backups + // with integrity_check before touching the live database, so a text + // fixture would be (correctly) refused. backupName := "chatserver_20240101_120000.db" backupPath := filepath.Join(backupDir, backupName) - fakeContent := []byte("fake sqlite db content") - if err := os.WriteFile(backupPath, fakeContent, 0o644); err != nil { - t.Fatalf("WriteFile backup: %v", err) + if err := database.BackupToSafe(context.Background(), backupPath, backupDir); err != nil { + t.Fatalf("BackupToSafe fixture: %v", err) } restarted, restoreHook := admin.StubRestart() @@ -369,11 +371,31 @@ func TestHandleRestoreBackup_RollsBackWhenCopyFails(t *testing.T) { t.Fatalf("WriteFile live db: %v", err) } + // A valid backup (it must pass the pre-copy integrity gate); the mid-copy + // failure is injected through the copy hook below, reproducing the exact + // failure mode the rollback exists for: os.Create truncates the live DB, + // then the copy dies. backupName := "chatserver_20240102_120000.db" - if err := os.MkdirAll(filepath.Join(backupDir, backupName), 0o750); err != nil { - t.Fatalf("MkdirAll fake backup: %v", err) + if err := database.BackupToSafe(context.Background(), filepath.Join(backupDir, backupName), backupDir); err != nil { + t.Fatalf("BackupToSafe fixture: %v", err) } + failedOnce := false + restoreCopy := admin.StubCopyBackup(func(src, dst string) error { + if !failedOnce { + failedOnce = true + // Truncate the destination the way the real copy's os.Create + // does, then fail — the state the rollback must repair. + f, createErr := os.Create(dst) + if createErr == nil { + _ = f.Close() + } + return fmt.Errorf("injected copy failure") + } + return admin.CopyBackupForTest(src, dst) + }) + defer restoreCopy() + restarted, restoreHook := admin.StubRestart() defer restoreHook() @@ -442,9 +464,13 @@ func TestHandleRestoreBackup_RestartsWhenCloseFails(t *testing.T) { if err := os.WriteFile(dbPath, []byte("original live contents"), 0o600); err != nil { t.Fatalf("WriteFile live db: %v", err) } + // A real SQLite backup — the restore handler verifies backups with + // integrity_check before touching the live database, so a text fixture + // would be (correctly) refused with 400 before the Close-failure branch + // under test is ever reached. backupName := "chatserver_20240103_120000.db" - if err := os.WriteFile(filepath.Join(backupDir, backupName), []byte("replacement contents"), 0o644); err != nil { - t.Fatalf("WriteFile backup: %v", err) + if err := database.BackupToSafe(context.Background(), filepath.Join(backupDir, backupName), backupDir); err != nil { + t.Fatalf("BackupToSafe fixture: %v", err) } restarted, restoreRestartHook := admin.StubRestart() @@ -483,8 +509,8 @@ func TestHandleRestoreBackup_AbortsWithoutSafetyBackup(t *testing.T) { } backupName := "chatserver_20240101_120000.db" dbFile := filepath.Join(tmpDir, "data", "chatserver.db") - if err := os.WriteFile(filepath.Join(backupDir, backupName), []byte("replacement"), 0o644); err != nil { - t.Fatalf("WriteFile backup: %v", err) + if err := database.BackupToSafe(context.Background(), filepath.Join(backupDir, backupName), backupDir); err != nil { + t.Fatalf("BackupToSafe fixture: %v", err) } if err := os.WriteFile(dbFile, []byte("original"), 0o644); err != nil { t.Fatalf("WriteFile db: %v", err) @@ -551,9 +577,13 @@ func TestHandleRestoreBackup_UsesConfiguredDatabasePath(t *testing.T) { t.Cleanup(func() { admin.SetDatabasePath(filepath.Join("data", "chatserver.db")) }) backupName := "chatserver_20240101_120000.db" - backupContent := []byte("restored contents") - if err := os.WriteFile(filepath.Join(backupDir, backupName), backupContent, 0o644); err != nil { - t.Fatalf("WriteFile backup: %v", err) + backupPath := filepath.Join(backupDir, backupName) + if err := database.BackupToSafe(context.Background(), backupPath, backupDir); err != nil { + t.Fatalf("BackupToSafe fixture: %v", err) + } + backupContent, err := os.ReadFile(backupPath) + if err != nil { + t.Fatalf("ReadFile fixture: %v", err) } restarted, restoreHook := admin.StubRestart() diff --git a/Server/admin/handlers_channel_perms.go b/Server/admin/handlers_channel_perms.go index b7c25e5a..bdcee704 100644 --- a/Server/admin/handlers_channel_perms.go +++ b/Server/admin/handlers_channel_perms.go @@ -160,8 +160,20 @@ func handlePutChannelPermission(database *db.DB, hub HubBroadcaster, permInvalid db.WriteAudit(context.WithoutCancel(r.Context()), database, actor, "channel_perms_update", "channel", ch.ID, fmt.Sprintf("set overrides for role %s on #%s (allow=%#x deny=%#x)", role.Name, ch.Name, allow, deny)) + // Narrow the eviction to users actually holding this role: a + // role-scoped override cannot change any other user's verdict, and + // InvalidateAll here made every connected user repopulate (2 reads + // each) synchronously inside RefreshChannelVisibility below — a + // whole-cache stampede that grows with total population, not with the + // role's size. Same rationale (and same fail-safe) as the role-perms + // handler: an unreadable member list falls back to the full flush, + // because a missed eviction is a stale grant. if permInvalidator != nil { - permInvalidator.InvalidateAll() + if affected, listErr := database.ListUserIDsByRole(r.Context(), roleID); listErr == nil { + invalidateUsers(permInvalidator, affected) + } else { + permInvalidator.InvalidateAll() + } } if hub != nil { hub.RefreshChannelVisibility(ch) @@ -222,8 +234,20 @@ func handleDeleteChannelPermission(database *db.DB, hub HubBroadcaster, permInva db.WriteAudit(context.WithoutCancel(r.Context()), database, actor, "channel_perms_clear", "channel", ch.ID, fmt.Sprintf("cleared overrides for role %s on #%s", role.Name, ch.Name)) + // Narrow the eviction to users actually holding this role: a + // role-scoped override cannot change any other user's verdict, and + // InvalidateAll here made every connected user repopulate (2 reads + // each) synchronously inside RefreshChannelVisibility below — a + // whole-cache stampede that grows with total population, not with the + // role's size. Same rationale (and same fail-safe) as the role-perms + // handler: an unreadable member list falls back to the full flush, + // because a missed eviction is a stale grant. if permInvalidator != nil { - permInvalidator.InvalidateAll() + if affected, listErr := database.ListUserIDsByRole(r.Context(), roleID); listErr == nil { + invalidateUsers(permInvalidator, affected) + } else { + permInvalidator.InvalidateAll() + } } if hub != nil { hub.RefreshChannelVisibility(ch) diff --git a/Server/admin/handlers_channel_perms_test.go b/Server/admin/handlers_channel_perms_test.go index 6f70cce5..2a257427 100644 --- a/Server/admin/handlers_channel_perms_test.go +++ b/Server/admin/handlers_channel_perms_test.go @@ -107,6 +107,13 @@ func TestPutChannelPermission_PersistsAndPropagates(t *testing.T) { t.Fatalf("CreateChannel: %v", err) } + // A member of the targeted role, so the narrowed invalidation has someone + // to evict. Users of other roles must NOT be evicted. + memberID, err := database.CreateUser(context.Background(), "role3member", "hash", 3) + if err != nil { + t.Fatalf("CreateUser: %v", err) + } + denyPrivate := permissions.ReadMessages | permissions.ConnectVoice body := map[string]any{"allow": 0, "deny": denyPrivate} w := doRequest(t, handler, http.MethodPut, @@ -123,8 +130,14 @@ func TestPutChannelPermission_PersistsAndPropagates(t *testing.T) { t.Errorf("persisted override = (%#x, %#x), want (0, %#x)", allow, deny, denyPrivate) } - if inv.invalidateAllN != 1 { - t.Errorf("InvalidateAll calls = %d, want 1", inv.invalidateAllN) + // The eviction is narrowed to the targeted role's members — a role-scoped + // override cannot change any other user's verdict, so the whole-cache + // flush (and its repopulate stampede) is reserved for the fail-safe path. + if inv.invalidateAllN != 0 { + t.Errorf("InvalidateAll calls = %d, want 0 (narrowed invalidation)", inv.invalidateAllN) + } + if len(inv.invalidateUserIDs) != 1 || inv.invalidateUserIDs[0] != memberID { + t.Errorf("InvalidateUser calls = %v, want exactly [%d]", inv.invalidateUserIDs, memberID) } if len(hub.visibilityRefreshes) != 1 || hub.visibilityRefreshes[0].ID != chID { t.Errorf("RefreshChannelVisibility not called for channel %d", chID) @@ -317,6 +330,10 @@ func TestDeleteChannelPermission_ClearsOverride(t *testing.T) { if err := database.UpsertChannelOverride(context.Background(), chID, 3, 0, permissions.ReadMessages); err != nil { t.Fatalf("UpsertChannelOverride: %v", err) } + memberID, err := database.CreateUser(context.Background(), "role3clear", "hash", 3) + if err != nil { + t.Fatalf("CreateUser: %v", err) + } w := doRequest(t, handler, http.MethodDelete, "/channels/"+itoa(chID)+"/permissions/3", token, nil) @@ -331,8 +348,12 @@ func TestDeleteChannelPermission_ClearsOverride(t *testing.T) { if allow != 0 || deny != 0 { t.Errorf("override still present: (%#x, %#x)", allow, deny) } - if inv.invalidateAllN != 1 { - t.Errorf("InvalidateAll calls = %d, want 1", inv.invalidateAllN) + // Narrowed invalidation: only the targeted role's members are evicted. + if inv.invalidateAllN != 0 { + t.Errorf("InvalidateAll calls = %d, want 0 (narrowed invalidation)", inv.invalidateAllN) + } + if len(inv.invalidateUserIDs) != 1 || inv.invalidateUserIDs[0] != memberID { + t.Errorf("InvalidateUser calls = %v, want exactly [%d]", inv.invalidateUserIDs, memberID) } if len(hub.visibilityRefreshes) != 1 { t.Errorf("RefreshChannelVisibility calls = %d, want 1", len(hub.visibilityRefreshes)) diff --git a/Server/admin/static/index.html b/Server/admin/static/index.html index 5afa8b01..cdff39e6 100644 --- a/Server/admin/static/index.html +++ b/Server/admin/static/index.html @@ -1568,12 +1568,12 @@ async function renderSettings(){ let html='
Server Settings
Configure your OwnCord server
'; html+='

General

'; html+='
Server Name
'; - html+='
Server Icon URL
'; + html+='
Server Icon URL
Not used by the server or client yet — stored for a future release
'; html+='
Message of the Day
Shown to users when they connect
'; html+='
'; html+='

Limits

'; - html+='
Max Upload Size (bytes)
'; - html+='
Voice Quality
'; + html+='
Max Upload Size (bytes)
Controlled by upload.max_size_mb in config.yaml (requires restart) — this display value has no effect
'; + html+='
Voice Quality
Controlled by voice.quality in config.yaml (requires restart) — this display value has no effect
'; html+='
'; html+='

Security

'; html+='
Require 2FA
Require all users to enable two-factor authentication
'; diff --git a/Server/api/auth_handler.go b/Server/api/auth_handler.go index ee3f1b2c..eb47a7c0 100644 --- a/Server/api/auth_handler.go +++ b/Server/api/auth_handler.go @@ -108,13 +108,13 @@ func MountAuthRoutes(r chi.Router, database *db.DB, limiter *auth.RateLimiter, t usedTOTPCodes := auth.NewUsedTOTPCodeStore() r.Route("/api/v1/auth", func(r chi.Router) { - r.With(RateLimitMiddleware(registerLimiter, "register:", registerRateLimitPerMinute, time.Minute, trustedProxies)). + r.With(RateLimitMiddleware(registerLimiter, "register:", scaledAuthLimit(registerRateLimitPerMinute), time.Minute, trustedProxies)). Post("/register", handleRegister(database, trustedProxies)) - r.With(RateLimitMiddleware(loginLimiter, "login:", loginRateLimitPerMinute, time.Minute, trustedProxies)). + r.With(RateLimitMiddleware(loginLimiter, "login:", scaledAuthLimit(loginRateLimitPerMinute), time.Minute, trustedProxies)). Post("/login", handleLogin(database, limiter, partialStore, trustedProxies)) - r.With(RateLimitMiddleware(limiter, "totp_verify:", verifyTOTPRateLimitPerMinute, time.Minute, trustedProxies)). + r.With(RateLimitMiddleware(limiter, "totp_verify:", scaledAuthLimit(verifyTOTPRateLimitPerMinute), time.Minute, trustedProxies)). Post("/verify-totp", handleVerifyTOTP(database, partialStore, limiter, usedTOTPCodes, totpKey)) r.With(AuthMiddleware(database)). @@ -124,20 +124,20 @@ func MountAuthRoutes(r chi.Router, database *db.DB, limiter *auth.RateLimiter, t Get("/me", handleMe()) r.With(AuthMiddleware(database), - RateLimitMiddleware(limiter, "del_account:", sensitiveEndpointRateLimitPerMinute, time.Minute, trustedProxies)). + RateLimitMiddleware(limiter, "del_account:", scaledAuthLimit(sensitiveEndpointRateLimitPerMinute), time.Minute, trustedProxies)). Delete("/account", handleDeleteAccount(database, limiter, ab)) }) r.With(AuthMiddleware(database), - RateLimitMiddleware(limiter, "totp:", sensitiveEndpointRateLimitPerMinute, time.Minute, trustedProxies)). + RateLimitMiddleware(limiter, "totp:", scaledAuthLimit(sensitiveEndpointRateLimitPerMinute), time.Minute, trustedProxies)). Post("/api/v1/users/me/totp/enable", handleEnableTOTP(pendingTOTPStore, limiter)) r.With(AuthMiddleware(database), - RateLimitMiddleware(limiter, "totp:", sensitiveEndpointRateLimitPerMinute, time.Minute, trustedProxies)). + RateLimitMiddleware(limiter, "totp:", scaledAuthLimit(sensitiveEndpointRateLimitPerMinute), time.Minute, trustedProxies)). Post("/api/v1/users/me/totp/confirm", handleConfirmTOTP(database, pendingTOTPStore, usedTOTPCodes, limiter, totpKey)) r.With(AuthMiddleware(database), - RateLimitMiddleware(limiter, "totp:", sensitiveEndpointRateLimitPerMinute, time.Minute, trustedProxies)). + RateLimitMiddleware(limiter, "totp:", scaledAuthLimit(sensitiveEndpointRateLimitPerMinute), time.Minute, trustedProxies)). Delete("/api/v1/users/me/totp", handleDisableTOTP(database, pendingTOTPStore, limiter)) } @@ -393,7 +393,7 @@ func handleLogin(database *db.DB, limiter *auth.RateLimiter, partialStore *auth. // password on attempt 10 still succeeds — successful logins reset // both counters. The reservation sits after the DB-error return above // so a transient DB outage still does not consume attempts. - if !limiter.Allow(failKey, loginFailureThreshold+1, loginFailureWindow) || + if !limiter.Allow(failKey, scaledAuthLimit(loginFailureThreshold)+1, loginFailureWindow) || !limiter.Allow(userFailKey, loginUserFailureThreshold+1, loginUserFailureWindow) { writeJSON(w, http.StatusTooManyRequests, errorResponse{ Error: "RATE_LIMITED", @@ -416,7 +416,7 @@ func handleLogin(database *db.DB, limiter *auth.RateLimiter, partialStore *auth. // only decide the lockouts, at the same boundary as before: the // 10th in-window failure locks the key. Check is read-only, so // the reservation is not double-counted. - if !limiter.Check(failKey, loginFailureThreshold+1, loginFailureWindow) { + if !limiter.Check(failKey, scaledAuthLimit(loginFailureThreshold)+1, loginFailureWindow) { limiter.Lockout(r.Context(), lockKey, loginLockoutDuration) } // BUG-110: per-username lockout on threshold. diff --git a/Server/api/avatar_handler_test.go b/Server/api/avatar_handler_test.go index c662bdb2..d1875baf 100644 --- a/Server/api/avatar_handler_test.go +++ b/Server/api/avatar_handler_test.go @@ -195,8 +195,10 @@ func TestUploadAvatar_StorageErrorDoesNotLeakPath(t *testing.T) { } rr := doAvatarUpload(t, router, token, "me.png", makePNGBytes(t, 32, 32)) - if rr.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400; body: %s", rr.Code, rr.Body.String()) + // Server-side filesystem failures are 507 (storage.ErrIO) so they are + // distinguishable from bad uploads; the no-leak contract is unchanged. + if rr.Code != http.StatusInsufficientStorage { + t.Fatalf("status = %d, want 507; body: %s", rr.Code, rr.Body.String()) } var resp map[string]any diff --git a/Server/api/constants.go b/Server/api/constants.go index 36ebd638..2375717f 100644 --- a/Server/api/constants.go +++ b/Server/api/constants.go @@ -1,6 +1,8 @@ package api import ( + "math" + "sync/atomic" "time" "github.com/owncord/server/config" @@ -11,6 +13,37 @@ import ( // Each constant defines either a request cap or a sliding-window duration used // by the per-endpoint rate limiters. +// authRateScaleBits holds the security.auth_rate_limit_multiplier as float +// bits. It scales the per-IP auth request caps and failure thresholds for +// deployments where many users share one IP (office/school NAT) — the +// compiled-in constants below assume roughly one person per address. Atomic +// because tests construct multiple routers concurrently. Set via +// setAuthRateScale in NewRouter; reads happen at mount time and on the login +// failure-count path. +var authRateScaleBits atomic.Uint64 + +func init() { authRateScaleBits.Store(math.Float64bits(1.0)) } + +// setAuthRateScale clamps and installs the auth rate multiplier. Zero or +// negative (unset config) means 1.0. +func setAuthRateScale(m float64) { + if m <= 0 { + m = 1.0 + } + m = math.Min(math.Max(m, 0.1), 100) + authRateScaleBits.Store(math.Float64bits(m)) +} + +// scaledAuthLimit applies the auth rate multiplier to a compiled-in limit, +// never returning less than 1. +func scaledAuthLimit(n int) int { + scaled := int(math.Round(float64(n) * math.Float64frombits(authRateScaleBits.Load()))) + if scaled < 1 { + return 1 + } + return scaled +} + const ( // registerRateLimitPerMinute is the maximum registration attempts per IP per minute. registerRateLimitPerMinute = 3 diff --git a/Server/api/emoji_handler.go b/Server/api/emoji_handler.go index 499274eb..23eaf1f8 100644 --- a/Server/api/emoji_handler.go +++ b/Server/api/emoji_handler.go @@ -21,7 +21,6 @@ import ( "github.com/owncord/server/auth" "github.com/owncord/server/db" "github.com/owncord/server/service" - "github.com/owncord/server/storage" ) // EmojiBroadcaster is the slice of the hub the emoji routes need: after every @@ -74,7 +73,7 @@ var allowedEmojiMIME = map[string]bool{ // are gated on MANAGE_SERVER inside EmojiService. The image route is // authenticated rather than public so an emoji cannot be used as an // unauthenticated tracking pixel hosted on someone else's server. -func MountEmojiRoutes(r chi.Router, database *db.DB, svc *service.Services, store *storage.Storage, limiter *auth.RateLimiter, broadcaster EmojiBroadcaster) { +func MountEmojiRoutes(r chi.Router, database *db.DB, svc *service.Services, store FileStore, limiter *auth.RateLimiter, broadcaster EmojiBroadcaster) { r.Route("/api/v1/emoji", func(r chi.Router) { r.Use(AuthMiddleware(database)) r.Get("/", handleListEmoji(svc)) @@ -102,7 +101,7 @@ func handleListEmoji(svc *service.Services) http.HandlerFunc { // member without MANAGE_SERVER cannot make the server spool a body to disk; the // shortcode is validated next, so a malformed name costs nothing either; only // then are the bytes read, sniffed, measured and stored. -func handleCreateEmoji(svc *service.Services, store *storage.Storage, limiter *auth.RateLimiter, broadcaster EmojiBroadcaster) http.HandlerFunc { +func handleCreateEmoji(svc *service.Services, store FileStore, limiter *auth.RateLimiter, broadcaster EmojiBroadcaster) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { user, ok := r.Context().Value(UserKey).(*db.User) if !ok || user == nil { @@ -197,10 +196,9 @@ func handleCreateEmoji(svc *service.Services, store *storage.Storage, limiter *a storedAs := uuid.New().String() if _, saveErr := store.Save(storedAs, bytes.NewReader(raw)); saveErr != nil { - slog.Warn("emoji upload rejected by storage", "error", saveErr) - writeJSON(w, http.StatusBadRequest, errorResponse{ - Error: "BAD_REQUEST", Message: fmt.Sprintf("upload rejected: %s", saveErr), - }) + // writeStorageSaveError also stops echoing raw storage errors + // (which embed absolute paths) into the response body. + writeStorageSaveError(w, saveErr, "emoji upload") return } @@ -219,7 +217,7 @@ func handleCreateEmoji(svc *service.Services, store *storage.Storage, limiter *a } } -func handleDeleteEmoji(svc *service.Services, store *storage.Storage, broadcaster EmojiBroadcaster) http.HandlerFunc { +func handleDeleteEmoji(svc *service.Services, store FileStore, broadcaster EmojiBroadcaster) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { user, ok := r.Context().Value(UserKey).(*db.User) if !ok || user == nil { @@ -278,7 +276,7 @@ func broadcastEmojiSet(ctx context.Context, svc *service.Services, broadcaster E // server-wide by construction, so authentication is the whole check. The // response is immutable for the id's lifetime (an emoji's bytes never change // — a replacement is a new row), which is what lets it be cached hard. -func handleServeEmojiImage(svc *service.Services, store *storage.Storage) http.HandlerFunc { +func handleServeEmojiImage(svc *service.Services, store FileStore) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64) if err != nil || id <= 0 { diff --git a/Server/api/filestore.go b/Server/api/filestore.go new file mode 100644 index 00000000..9b79b0d4 --- /dev/null +++ b/Server/api/filestore.go @@ -0,0 +1,31 @@ +package api + +import ( + "io" + + "github.com/owncord/server/storage" +) + +// FileStore is the consumer-side seam over blob storage, following the repo's +// interface-at-the-consumer pattern (see service.Store, D3): the api package +// declares exactly the three operations its handlers use, and the concrete +// *storage.Storage satisfies it. This is deliberately an interface carve-out, +// NOT an alternative-backend implementation — multi-backend storage (S3 and +// friends) is out of scope today. What the seam buys now is that the +// contract an alternative backend would have to meet is written down where +// it is consumed; note in particular that Open must return a seekable file +// (storage.File), because the serve paths use http.ServeContent for range +// requests. +type FileStore interface { + // Save writes r to a file named by uuid, validating content type by + // magic bytes and enforcing the size limit. Filesystem-level failures + // carry storage.ErrIO. + Save(uuid string, r io.Reader) (int64, error) + // Delete removes the stored file named uuid. + Delete(uuid string) error + // Open opens the stored file named uuid for seekable reading. + Open(uuid string) (storage.File, error) +} + +// compile-time proof the disk implementation satisfies the seam. +var _ FileStore = (*storage.Storage)(nil) diff --git a/Server/api/health_test.go b/Server/api/health_test.go new file mode 100644 index 00000000..71d3bc5c --- /dev/null +++ b/Server/api/health_test.go @@ -0,0 +1,120 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRunHealthChecks_AllHealthy(t *testing.T) { + status, reason := runHealthChecks(context.Background(), healthDeps{ + dbPing: func(context.Context) error { return nil }, + dispatchAlive: func() bool { return true }, + freeDiskBytes: func() (uint64, error) { return 10 << 30, nil }, + }) + if status != "ok" || reason != "" { + t.Fatalf("got (%q, %q), want (ok, \"\")", status, reason) + } +} + +func TestRunHealthChecks_HubDeadWinsFirst(t *testing.T) { + status, reason := runHealthChecks(context.Background(), healthDeps{ + dbPing: func(context.Context) error { return errors.New("also down") }, + dispatchAlive: func() bool { return false }, + }) + if status != "degraded" || reason != "hub" { + t.Fatalf("got (%q, %q), want (degraded, hub)", status, reason) + } +} + +func TestRunHealthChecks_DBError(t *testing.T) { + status, reason := runHealthChecks(context.Background(), healthDeps{ + dbPing: func(context.Context) error { return errors.New("locked") }, + dispatchAlive: func() bool { return true }, + }) + if status != "degraded" || reason != "database" { + t.Fatalf("got (%q, %q), want (degraded, database)", status, reason) + } +} + +func TestRunHealthChecks_LowDisk(t *testing.T) { + status, reason := runHealthChecks(context.Background(), healthDeps{ + freeDiskBytes: func() (uint64, error) { return 1 << 20, nil }, // 1 MiB + }) + if status != "degraded" || reason != "disk" { + t.Fatalf("got (%q, %q), want (degraded, disk)", status, reason) + } +} + +// TestRunHealthChecks_UnknownProbesCountHealthy locks the "unknown ≠ full" +// rule: a probe error (unsupported platform, missing dir) must not degrade. +func TestRunHealthChecks_UnknownProbesCountHealthy(t *testing.T) { + status, _ := runHealthChecks(context.Background(), healthDeps{ + freeDiskBytes: func() (uint64, error) { return 0, errors.New("no statfs here") }, + }) + if status != "ok" { + t.Fatalf("probe error degraded health: got %q, want ok", status) + } +} + +func TestHandleHealth_DegradedReturns503WithReason(t *testing.T) { + h := handleHealth(healthDeps{ + onlineUsers: func() int { return 3 }, + dispatchAlive: func() bool { return false }, + }) + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest(http.MethodGet, "/health", nil)) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rec.Code) + } + var resp healthResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.Status != "degraded" || resp.Reason != "hub" { + t.Errorf("body = %+v, want status=degraded reason=hub", resp) + } + if resp.OnlineUsers != 3 { + t.Errorf("online_users = %d, want 3", resp.OnlineUsers) + } +} + +// TestHandleHealth_CanceledRequestDoesNotPoisonCache locks the WithoutCancel +// guard: a probe that disconnects mid-request must not stamp a false +// "degraded/database" verdict into the shared 5s cache. +func TestHandleHealth_CanceledRequestDoesNotPoisonCache(t *testing.T) { + h := handleHealth(healthDeps{ + dbPing: func(ctx context.Context) error { return ctx.Err() }, + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // client already gone + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest(http.MethodGet, "/health", nil).WithContext(ctx)) + if rec.Code != http.StatusOK { + t.Fatalf("canceled request produced %d; its cancellation leaked into the cached checks", rec.Code) + } +} + +// TestHandleHealth_ChecksAreCached locks the amplification guard: the endpoint +// is unauthenticated and rate-limit-exempt, so the real checks must run at +// most once per healthCacheTTL, not per request. +func TestHandleHealth_ChecksAreCached(t *testing.T) { + pings := 0 + h := handleHealth(healthDeps{ + dbPing: func(context.Context) error { pings++; return nil }, + }) + for range 5 { + rec := httptest.NewRecorder() + h(rec, httptest.NewRequest(http.MethodGet, "/health", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + } + if pings != 1 { + t.Fatalf("dbPing ran %d times across 5 requests, want 1 (cached)", pings) + } +} diff --git a/Server/api/metrics_handler.go b/Server/api/metrics_handler.go index 221c854c..d6593428 100644 --- a/Server/api/metrics_handler.go +++ b/Server/api/metrics_handler.go @@ -2,12 +2,23 @@ package api import ( "context" + "database/sql" "net/http" "runtime" "time" ) +// EventPersisterMetrics is the nested event-persistence block of ServerMetrics. +// Present only when event persistence is enabled. +type EventPersisterMetrics struct { + Persisted uint64 `json:"persisted"` + Dropped uint64 `json:"dropped"` + Flushes uint64 `json:"flushes"` + Errors uint64 `json:"errors"` +} + // ServerMetrics holds runtime metrics for the /api/v1/metrics endpoint. +// The shape is documented in docs/deployment.md — keep the two in sync. type ServerMetrics struct { Uptime string `json:"uptime"` UptimeSeconds float64 `json:"uptime_seconds"` @@ -17,36 +28,122 @@ type ServerMetrics struct { NumGC uint32 `json:"num_gc"` ConnectedUsers int `json:"connected_users"` VoiceSessions int `json:"voice_sessions"` - BroadcastDrops uint64 `json:"broadcast_drops"` - LiveKitHealthy *bool `json:"livekit_healthy,omitempty"` + // BroadcastDrops counts messages dropped because the hub-wide broadcast + // queue was full — a hub-level overload signal, distinct from the + // per-client backpressure counters below. Any nonzero growth here means + // sequenced events were lost before delivery and is worth alerting on. + BroadcastDrops uint64 `json:"broadcast_drops"` + LiveKitHealthy *bool `json:"livekit_healthy,omitempty"` + + // Reconnect replay tier hits. A rising full-resync share means the replay + // budget (ring size / cold cap) is too small for observed disconnect gaps. + ReconnectTierBuffer uint64 `json:"reconnect_tier_buffer"` + ReconnectTierDB uint64 `json:"reconnect_tier_db"` + ReconnectTierFull uint64 `json:"reconnect_tier_full"` + + // Per-client send-queue backpressure totals. + BackpressureQueueDisconnects uint64 `json:"backpressure_queue_disconnects"` + BackpressureHighFallbacks uint64 `json:"backpressure_high_fallbacks"` + BackpressureLowDrops uint64 `json:"backpressure_low_drops"` + + // WSConnRejects counts upgrades refused by the max_ws_connections cap. + WSConnRejects uint64 `json:"ws_conn_rejects"` + + // DiskFreeMB is free space on the data volume; omitted when unknown. + DiskFreeMB *float64 `json:"disk_free_mb,omitempty"` + + // SQLite writer-pool saturation: time spent queueing for the single write + // connection. The most direct signal for the documented single-writer + // bottleneck. + DBWriterWaitCount int64 `json:"db_writer_wait_count"` + DBWriterWaitSeconds float64 `json:"db_writer_wait_seconds"` + + // Permission cache effectiveness. + PermCacheHits uint64 `json:"perm_cache_hits"` + PermCacheMisses uint64 `json:"perm_cache_misses"` + + EventPersister *EventPersisterMetrics `json:"event_persister,omitempty"` +} + +// MetricsSources provides the live data feeds for handleMetrics. Any nil +// field is skipped, leaving that metric at its zero value (or absent for +// pointer-typed output), so tests and partial wirings stay cheap. +type MetricsSources struct { + ConnectedUsers func() int + VoiceSessions func() int + BroadcastDrops func() uint64 + LiveKitHealth func(context.Context) (bool, error) + ReconnectTiers func() (buffer, db, full uint64) + Backpressure func() (queueDisconnects, highFallbacks, lowDrops uint64) + ConnRejects func() uint64 + PersisterStats func() (persisted, dropped, flushes, errs uint64, ok bool) + DBStats func() sql.DBStats // writer pool + PermCache func() (hits, misses uint64) + DiskFree func() (uint64, error) } // handleMetrics returns an HTTP handler that reports runtime server metrics. -// getConnectedUsers is a callback to retrieve the current WebSocket client count. -// getBroadcastDrops is a callback to retrieve the cumulative broadcast drop counter. -// livekitHealthCheck is optional — if non-nil, it probes the LiveKit companion process. -func handleMetrics(getConnectedUsers func() int, getVoiceSessions func() int, getBroadcastDrops func() uint64, livekitHealthCheck func(context.Context) (bool, error)) http.HandlerFunc { +func handleMetrics(src MetricsSources) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { var m runtime.MemStats runtime.ReadMemStats(&m) uptime := time.Since(serverStartTime) metrics := ServerMetrics{ - Uptime: uptime.Truncate(time.Second).String(), - UptimeSeconds: uptime.Seconds(), - GoRoutines: runtime.NumGoroutine(), - HeapAllocMB: float64(m.HeapAlloc) / 1024 / 1024, - HeapSysMB: float64(m.HeapSys) / 1024 / 1024, - NumGC: m.NumGC, - ConnectedUsers: getConnectedUsers(), - VoiceSessions: getVoiceSessions(), - BroadcastDrops: getBroadcastDrops(), + Uptime: uptime.Truncate(time.Second).String(), + UptimeSeconds: uptime.Seconds(), + GoRoutines: runtime.NumGoroutine(), + HeapAllocMB: float64(m.HeapAlloc) / 1024 / 1024, + HeapSysMB: float64(m.HeapSys) / 1024 / 1024, + NumGC: m.NumGC, } - if livekitHealthCheck != nil { - healthy, _ := livekitHealthCheck(r.Context()) + if src.ConnectedUsers != nil { + metrics.ConnectedUsers = src.ConnectedUsers() + } + if src.VoiceSessions != nil { + metrics.VoiceSessions = src.VoiceSessions() + } + if src.BroadcastDrops != nil { + metrics.BroadcastDrops = src.BroadcastDrops() + } + if src.LiveKitHealth != nil { + healthy, _ := src.LiveKitHealth(r.Context()) metrics.LiveKitHealthy = &healthy } + if src.ReconnectTiers != nil { + metrics.ReconnectTierBuffer, metrics.ReconnectTierDB, metrics.ReconnectTierFull = src.ReconnectTiers() + } + if src.Backpressure != nil { + metrics.BackpressureQueueDisconnects, metrics.BackpressureHighFallbacks, metrics.BackpressureLowDrops = src.Backpressure() + } + if src.PersisterStats != nil { + if persisted, dropped, flushes, errs, ok := src.PersisterStats(); ok { + metrics.EventPersister = &EventPersisterMetrics{ + Persisted: persisted, + Dropped: dropped, + Flushes: flushes, + Errors: errs, + } + } + } + if src.DBStats != nil { + st := src.DBStats() + metrics.DBWriterWaitCount = st.WaitCount + metrics.DBWriterWaitSeconds = st.WaitDuration.Seconds() + } + if src.PermCache != nil { + metrics.PermCacheHits, metrics.PermCacheMisses = src.PermCache() + } + if src.ConnRejects != nil { + metrics.WSConnRejects = src.ConnRejects() + } + if src.DiskFree != nil { + if free, err := src.DiskFree(); err == nil { + mb := float64(free) / 1024 / 1024 + metrics.DiskFreeMB = &mb + } + } writeJSON(w, http.StatusOK, metrics) } diff --git a/Server/api/metrics_handler_test.go b/Server/api/metrics_handler_test.go index df8ff281..79f49cc3 100644 --- a/Server/api/metrics_handler_test.go +++ b/Server/api/metrics_handler_test.go @@ -2,10 +2,12 @@ package api_test import ( "context" + "database/sql" "encoding/json" "net/http" "net/http/httptest" "testing" + "time" "github.com/go-chi/chi/v5" "github.com/owncord/server/api" @@ -15,12 +17,17 @@ import ( func buildMetricsRouter(allowedCIDRs []string) http.Handler { r := chi.NewRouter() r.With(api.AdminIPRestrict(allowedCIDRs, nil)). - Get("/api/v1/metrics", api.HandleMetricsForTest( - func() int { return 5 }, - func() int { return 2 }, - func() uint64 { return 0 }, - func(_ context.Context) (bool, error) { return true, nil }, - )) + Get("/api/v1/metrics", api.HandleMetricsForTest(api.MetricsSources{ + ConnectedUsers: func() int { return 5 }, + VoiceSessions: func() int { return 2 }, + BroadcastDrops: func() uint64 { return 0 }, + LiveKitHealth: func(_ context.Context) (bool, error) { return true, nil }, + ReconnectTiers: func() (uint64, uint64, uint64) { return 7, 3, 1 }, + Backpressure: func() (uint64, uint64, uint64) { return 4, 9, 11 }, + PersisterStats: func() (uint64, uint64, uint64, uint64, bool) { return 100, 2, 10, 1, true }, + DBStats: func() sql.DBStats { return sql.DBStats{WaitCount: 6, WaitDuration: 1500 * time.Millisecond} }, + PermCache: func() (uint64, uint64) { return 42, 8 }, + })) return r } @@ -45,6 +52,11 @@ func TestHandleMetrics_ReturnsExpectedFields(t *testing.T) { "uptime", "uptime_seconds", "goroutines", "heap_alloc_mb", "heap_sys_mb", "num_gc", "connected_users", "voice_sessions", "broadcast_drops", "livekit_healthy", + "reconnect_tier_buffer", "reconnect_tier_db", "reconnect_tier_full", + "backpressure_queue_disconnects", "backpressure_high_fallbacks", "backpressure_low_drops", + "db_writer_wait_count", "db_writer_wait_seconds", + "perm_cache_hits", "perm_cache_misses", + "event_persister", } for _, f := range requiredFields { if _, ok := resp[f]; !ok { @@ -62,6 +74,25 @@ func TestHandleMetrics_ReturnsExpectedFields(t *testing.T) { if resp["livekit_healthy"] != true { t.Errorf("livekit_healthy = %v, want true", resp["livekit_healthy"]) } + if int(resp["reconnect_tier_buffer"].(float64)) != 7 { + t.Errorf("reconnect_tier_buffer = %v, want 7", resp["reconnect_tier_buffer"]) + } + if int(resp["backpressure_low_drops"].(float64)) != 11 { + t.Errorf("backpressure_low_drops = %v, want 11", resp["backpressure_low_drops"]) + } + if got := resp["db_writer_wait_seconds"].(float64); got != 1.5 { + t.Errorf("db_writer_wait_seconds = %v, want 1.5", got) + } + if int(resp["perm_cache_hits"].(float64)) != 42 { + t.Errorf("perm_cache_hits = %v, want 42", resp["perm_cache_hits"]) + } + ep, ok := resp["event_persister"].(map[string]any) + if !ok { + t.Fatalf("event_persister = %v, want object", resp["event_persister"]) + } + if int(ep["persisted"].(float64)) != 100 { + t.Errorf("event_persister.persisted = %v, want 100", ep["persisted"]) + } } func TestHandleMetrics_AdminIPRestrict_BlocksNonAdmin(t *testing.T) { @@ -92,12 +123,14 @@ func TestHandleMetrics_AdminIPRestrict_AllowsAdmin(t *testing.T) { func TestHandleMetrics_WithoutLiveKitHealthCheck(t *testing.T) { r := chi.NewRouter() - r.Get("/api/v1/metrics", api.HandleMetricsForTest( - func() int { return 0 }, - func() int { return 0 }, - func() uint64 { return 0 }, - nil, // no livekit - )) + r.Get("/api/v1/metrics", api.HandleMetricsForTest(api.MetricsSources{ + ConnectedUsers: func() int { return 0 }, + VoiceSessions: func() int { return 0 }, + BroadcastDrops: func() uint64 { return 0 }, + // LiveKitHealth nil — no livekit wired. + // PersisterStats returning ok=false must omit event_persister. + PersisterStats: func() (uint64, uint64, uint64, uint64, bool) { return 0, 0, 0, 0, false }, + })) req := httptest.NewRequest(http.MethodGet, "/api/v1/metrics", nil) req.RemoteAddr = "127.0.0.1:9999" @@ -115,4 +148,8 @@ func TestHandleMetrics_WithoutLiveKitHealthCheck(t *testing.T) { if _, ok := resp["livekit_healthy"]; ok { t.Errorf("livekit_healthy should be omitted when health check is nil, got %v", resp["livekit_healthy"]) } + // event_persister should be absent when the persister reports ok=false. + if _, ok := resp["event_persister"]; ok { + t.Errorf("event_persister should be omitted when persistence is disabled, got %v", resp["event_persister"]) + } } diff --git a/Server/api/profile_handler.go b/Server/api/profile_handler.go index 9613d347..241ca00b 100644 --- a/Server/api/profile_handler.go +++ b/Server/api/profile_handler.go @@ -18,7 +18,6 @@ import ( "github.com/owncord/server/auth" "github.com/owncord/server/db" "github.com/owncord/server/service" - "github.com/owncord/server/storage" "github.com/owncord/server/ws" ) @@ -73,7 +72,7 @@ type ProfileBroadcaster interface { // store may be nil, in which case the avatar-upload route is not registered — // a server with no storage backend has nowhere to put the bytes, and a route // that 500s on every call is worse than one that 404s. -func MountProfileRoutes(r chi.Router, database *db.DB, svc *service.Services, store *storage.Storage, limiter *auth.RateLimiter, trustedProxies []string, broadcaster ProfileBroadcaster) { +func MountProfileRoutes(r chi.Router, database *db.DB, svc *service.Services, store FileStore, limiter *auth.RateLimiter, trustedProxies []string, broadcaster ProfileBroadcaster) { r.Route("/api/v1/users/me", func(r chi.Router) { r.Use(AuthMiddleware(database)) @@ -469,7 +468,7 @@ func handleRevokeSession(svc *service.Services) http.HandlerFunc { func handleUploadAvatar( database *db.DB, svc *service.Services, - store *storage.Storage, + store FileStore, limiter *auth.RateLimiter, broadcaster ProfileBroadcaster, ) http.HandlerFunc { @@ -558,10 +557,7 @@ func handleUploadAvatar( fileID := uuid.New().String() written, saveErr := store.Save(fileID, bytes.NewReader(raw)) if saveErr != nil { - slog.Warn("avatar upload rejected by storage", "error", saveErr) - writeJSON(w, http.StatusBadRequest, errorResponse{ - Error: "BAD_REQUEST", Message: safeStorageErrorMessage(saveErr), - }) + writeStorageSaveError(w, saveErr, "avatar upload") return } diff --git a/Server/api/router.go b/Server/api/router.go index f61a8b2f..bea474c7 100644 --- a/Server/api/router.go +++ b/Server/api/router.go @@ -3,6 +3,7 @@ package api import ( "context" + "database/sql" "encoding/json" "fmt" "log/slog" @@ -17,11 +18,13 @@ import ( "github.com/owncord/server/auth" "github.com/owncord/server/config" "github.com/owncord/server/db" + "github.com/owncord/server/diskutil" "github.com/owncord/server/permissions" "github.com/owncord/server/plugin" "github.com/owncord/server/service" "github.com/owncord/server/stackutil" "github.com/owncord/server/storage" + "github.com/owncord/server/syncutil" "github.com/owncord/server/telemetry" "github.com/owncord/server/updater" "github.com/owncord/server/ws" @@ -34,6 +37,9 @@ import ( // pluginRegistry may be nil — in that case the plugin admin endpoints respond // with 503 on lifecycle calls and an empty list on read. func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.RingBuffer, pluginRegistry *plugin.Registry) (http.Handler, *ws.Hub, func()) { + // Install the auth rate multiplier before any route mounts read it. + setAuthRateScale(cfg.Security.AuthRateLimitMultiplier) + // Load (or auto-generate) the AES-256 key for TOTP secret encryption // (M1). Done first, before any other setup, so a fatal failure here // (below) doesn't leave background goroutines or partially-mounted @@ -86,14 +92,39 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri } // Health check — unauthenticated, no versioning prefix. - // The online user count callback is set after hub creation below. + // The hub-backed callbacks are set after hub creation below (late-bound + // closures, same pattern the old online-user counter used). One shared + // handler instance backs both /health mounts so they share the check cache. var getOnlineUsers func() int - r.Get("/health", handleHealth(func() int { - if getOnlineUsers != nil { - return getOnlineUsers() - } - return 0 - })) + var hubAlive func() bool + healthHandler := handleHealth(healthDeps{ + onlineUsers: func() int { + if getOnlineUsers != nil { + return getOnlineUsers() + } + return 0 + }, + dbPing: func(ctx context.Context) error { + if database == nil { + return nil + } + // Reader pool, not the writer: a scheduled backup's VACUUM INTO + // holds the sole writer connection for its whole duration, and + // the server keeps serving reads throughout — /health must not + // call that outage (see db.PingRead). + return database.PingRead(ctx) + }, + dispatchAlive: func() bool { + if hubAlive != nil { + return hubAlive() + } + return true + }, + freeDiskBytes: func() (uint64, error) { + return diskutil.FreeBytes(cfg.Server.DataDir) + }, + }) + r.Get("/health", healthHandler) // Shared rate limiter for auth endpoints. Lockouts are persisted to the // database so they survive server restarts (M2 security hardening). @@ -106,12 +137,7 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri // Versioned API routes. r.Route("/api/v1", func(r chi.Router) { - r.Get("/health", handleHealth(func() int { - if getOnlineUsers != nil { - return getOnlineUsers() - } - return 0 - })) + r.Get("/health", healthHandler) r.Get("/info", handleInfo(cfg)) }) @@ -156,7 +182,10 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri // WebSocket hub — WS does its own in-band auth, so no AuthMiddleware here. hub := ws.NewHub(database, limiter, svc) + // Replay budget knobs must land before hub.Run starts (below). + hub.ConfigureReplay(cfg.EventPersistence.ReplayRingSize, cfg.EventPersistence.ReplayColdLimit) getOnlineUsers = func() int { return hub.ClientCount() } + hubAlive = func() bool { return hub.DispatchAlive() } // Auth routes: register, login, logout, me. Mounted with the hub as the // AuthBroadcaster so DELETE /api/v1/auth/account (self-service account @@ -212,19 +241,25 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri } if lkHost != "" && lkHost != "localhost" && lkHost != "127.0.0.1" && lkHost != "::1" { slog.Warn("LiveKit is externally managed but webhook endpoint is admin-IP-restricted — "+ - "ensure the LiveKit server's IP is in admin_allowed_cidrs or webhooks will be silently dropped", + "add the LiveKit server's IP to livekit_webhook_allowed_cidrs or webhooks will be silently dropped", "livekit_host", lkHost) } } - // LiveKit webhook endpoint (no auth middleware — uses LiveKit JWT verification). + // LiveKit webhook endpoint (no auth middleware — uses LiveKit JWT + // verification). The IP gate is defence-in-depth on top of that signature + // check, with its own allowlist key (livekit_webhook_allowed_cidrs) so an + // externally-hosted LiveKit can be admitted WITHOUT widening the admin + // panel's perimeter to the SFU's network. Falls back to + // admin_allowed_cidrs when unset. if lkErr == nil { - r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs, cfg.Server.TrustedProxies)). + webhookCIDRs := cfg.Server.LiveKitWebhookCIDRs() + r.With(AdminIPRestrict(webhookCIDRs, cfg.Server.TrustedProxies)). Post("/api/v1/livekit/webhook", ws.MountWebhookRoute(hub, cfg.Voice.LiveKitAPIKey, cfg.Voice.LiveKitAPISecret)) - // LiveKit health check — admin-IP-restricted. - r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs, cfg.Server.TrustedProxies)). + // LiveKit health check — same perimeter as the webhook. + r.With(AdminIPRestrict(webhookCIDRs, cfg.Server.TrustedProxies)). Get("/api/v1/livekit/health", handleLiveKitHealth(hub)) // Reverse proxy LiveKit signaling through OwnCord's HTTPS server. @@ -244,9 +279,12 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri // Mounted after hub creation so the hub can broadcast user_update events. // A storage failure leaves store unusable, so the avatar-upload route is // simply not registered; the rest of the profile surface is unaffected. - profileStore := store - if storeErr != nil { - profileStore = nil + // Built as a FileStore interface value from scratch — assigning the typed + // nil pointer would produce a non-nil interface and defeat the mount-time + // nil check. + var profileStore FileStore + if storeErr == nil { + profileStore = store } MountProfileRoutes(r, database, svc, profileStore, limiter, cfg.Server.TrustedProxies, hub) @@ -275,23 +313,33 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri handleDiagnosticsConnectivity(cfg, ver, hub)) go hub.Run() - r.Get("/api/v1/ws", ws.ServeWS(hub, database, cfg.Server.AllowedOrigins)) + r.Get("/api/v1/ws", ws.ServeWS(hub, database, cfg.Server.AllowedOrigins, cfg.Server.MaxWSConnections)) - // Metrics endpoint — admin-IP-restricted, returns runtime stats as JSON. - r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs, cfg.Server.TrustedProxies)). - Get("/api/v1/metrics", handleMetrics( - func() int { return hub.ClientCount() }, - func() int { return hub.VoiceSessionCount() }, - func() uint64 { return hub.BroadcastDropCount() }, - func(ctx context.Context) (bool, error) { return hub.LiveKitHealthCheck(ctx) }, - )) + // Metrics endpoint — IP-restricted by metrics_allowed_cidrs (falls back to + // admin_allowed_cidrs) so a central scraper can be admitted without + // widening /admin. The shape is documented in docs/deployment.md — keep + // the two in sync. + r.With(AdminIPRestrict(cfg.Server.MetricsCIDRs(), cfg.Server.TrustedProxies)). + Get("/api/v1/metrics", handleMetrics(MetricsSources{ + ConnectedUsers: hub.ClientCount, + VoiceSessions: hub.VoiceSessionCount, + BroadcastDrops: hub.BroadcastDropCount, + LiveKitHealth: hub.LiveKitHealthCheck, + ReconnectTiers: hub.ReconnectTierStats, + Backpressure: hub.BackpressureStats, + ConnRejects: hub.ConnRejectCount, + PersisterStats: hub.EventPersisterStats, + DBStats: func() sql.DBStats { return database.SQLDb().Stats() }, + PermCache: svc.Permissions.CacheStats, + DiskFree: func() (uint64, error) { return diskutil.FreeBytes(cfg.Server.DataDir) }, + })) // Phase B Step 8 — OpenTelemetry Prometheus exporter. Mounted alongside // the legacy JSON endpoint when a Prometheus exporter is wired (otel // build, exporter == "prometheus"). Returns 404 in the default no-op build // because telemetry.PrometheusHandler() returns nil. if promH := telemetry.PrometheusHandler(); promH != nil { - r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs, cfg.Server.TrustedProxies)). + r.With(AdminIPRestrict(cfg.Server.MetricsCIDRs(), cfg.Server.TrustedProxies)). Mount("/metrics", promH) } @@ -345,29 +393,105 @@ var serverStartTime = time.Now() // healthResponse is the JSON shape returned by GET /health. type healthResponse struct { - Status string `json:"status"` + Status string `json:"status"` // "ok" | "degraded" Uptime int64 `json:"uptime"` OnlineUsers int `json:"online_users"` + // Reason names the degraded subsystem ("hub", "database", "disk") and + // nothing more — this endpoint is unauthenticated, so no error details. + Reason string `json:"reason,omitempty"` } +// healthDeps are the liveness probes behind GET /health. Any nil field is +// skipped (treated as healthy) so partial wirings and tests stay simple. +type healthDeps struct { + onlineUsers func() int + dbPing func(context.Context) error + dispatchAlive func() bool + freeDiskBytes func() (uint64, error) +} + +const ( + // healthCacheTTL bounds how often the real checks run: the endpoint is + // unauthenticated AND rate-limit-exempt, so an uncached DB ping per + // request would be a free amplification lever. + healthCacheTTL = 5 * time.Second + // healthDBPingTimeout bounds the SELECT 1 so a wedged writer degrades the + // health report instead of hanging it. + healthDBPingTimeout = 1 * time.Second + // healthMinFreeDiskBytes is the free-space floor under which health + // reports degraded. SQLite WAL growth, uploads, and backups all share the + // data volume, so running dry corrupts more than one thing at once. + healthMinFreeDiskBytes = 256 << 20 // 256 MiB +) + // infoResponse is the JSON shape returned by GET /api/v1/info. type infoResponse struct { Name string `json:"name"` } -func handleHealth(getOnlineUsers func() int) http.HandlerFunc { +func handleHealth(deps healthDeps) http.HandlerFunc { + // C-2: Version removed from unauthenticated health endpoint to prevent + // server fingerprinting. Version is available on the authenticated + // diagnostics endpoint instead. + var mu syncutil.Mutex + var cachedAt time.Time + var cachedStatus, cachedReason string return func(w http.ResponseWriter, r *http.Request) { - // C-2: Version removed from unauthenticated health endpoint to prevent - // server fingerprinting. Version is available on the authenticated - // diagnostics endpoint instead. - writeJSON(w, http.StatusOK, healthResponse{ - Status: "ok", + mu.Lock() + if time.Since(cachedAt) >= healthCacheTTL { + // WithoutCancel: the result is cached and served to every caller + // for the next healthCacheTTL, so it must not inherit THIS + // request's cancellation — a probe that disconnects mid-check + // would otherwise poison the shared cache with a false + // "degraded/database" verdict. The DB ping carries its own 1s + // timeout, so the checks stay bounded regardless. + cachedStatus, cachedReason = runHealthChecks(context.WithoutCancel(r.Context()), deps) + cachedAt = time.Now() + } + status, reason := cachedStatus, cachedReason + mu.Unlock() + + online := 0 + if deps.onlineUsers != nil { + online = deps.onlineUsers() + } + code := http.StatusOK + if status != "ok" { + code = http.StatusServiceUnavailable + } + writeJSON(w, code, healthResponse{ + Status: status, Uptime: int64(time.Since(serverStartTime).Seconds()), - OnlineUsers: getOnlineUsers(), + OnlineUsers: online, + Reason: reason, }) } } +// runHealthChecks probes the hub dispatch loop, the database, and free disk, +// returning ("ok", "") or ("degraded", ). First failure wins, in +// blast-radius order. Probe errors that mean "unknown" (unsupported platform, +// missing dir in tests) count as healthy — only a positive negative degrades. +func runHealthChecks(ctx context.Context, deps healthDeps) (status, reason string) { + if deps.dispatchAlive != nil && !deps.dispatchAlive() { + return "degraded", "hub" + } + if deps.dbPing != nil { + pingCtx, cancel := context.WithTimeout(ctx, healthDBPingTimeout) + err := deps.dbPing(pingCtx) + cancel() + if err != nil { + return "degraded", "database" + } + } + if deps.freeDiskBytes != nil { + if free, err := deps.freeDiskBytes(); err == nil && free < healthMinFreeDiskBytes { + return "degraded", "disk" + } + } + return "ok", "" +} + func handleInfo(cfg *config.Config) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // C-2: Version removed from unauthenticated info endpoint. diff --git a/Server/api/totp_handler.go b/Server/api/totp_handler.go index 788881ec..c64b9619 100644 --- a/Server/api/totp_handler.go +++ b/Server/api/totp_handler.go @@ -73,6 +73,11 @@ func handleVerifyTOTP(database *db.DB, partialStore *auth.PartialAuthStore, limi // failure is recorded, defeating the per-user brute-force cap (the only // cross-IP defence). A successful verification resets the counter below, // so legitimate retries are not penalised. + // Deliberately NOT scaledAuthLimit: this cap is keyed per USER, and it + // is the only cross-IP brute-force defence on TOTP codes. The + // multiplier exists for shared-NAT per-IP limits; scaling a per-user + // threshold with it would hand a distributed attacker more guesses. + // Mirrors loginUserFailureThreshold staying unscaled in auth_handler. if !limiter.Allow(totpRateLimitKey, totpFailureRateLimit, totpFailureWindow) { writeJSON(w, http.StatusTooManyRequests, errorResponse{ Error: "RATE_LIMITED", diff --git a/Server/api/upload_handler.go b/Server/api/upload_handler.go index 0526a709..cc4dd24b 100644 --- a/Server/api/upload_handler.go +++ b/Server/api/upload_handler.go @@ -123,6 +123,28 @@ func safeStorageErrorMessage(err error) string { } } +// writeStorageSaveError maps a storage.Save failure onto the right HTTP +// class: server-side filesystem failures (storage.ErrIO — disk full, +// permissions, read-only mount) become 507 so they are distinguishable from +// bad uploads in any status dashboard; everything else stays the client's +// 400. Detail never crosses the HTTP boundary either way (path leakage — +// see safeStorageErrorMessage). +func writeStorageSaveError(w http.ResponseWriter, saveErr error, what string) { + if errors.Is(saveErr, storage.ErrIO) { + slog.Error(what+" failed: server storage error", "error", saveErr) + writeJSON(w, http.StatusInsufficientStorage, errorResponse{ + Error: "STORAGE_ERROR", + Message: "upload failed: server storage error", + }) + return + } + slog.Warn(what+" rejected", "error", saveErr) + writeJSON(w, http.StatusBadRequest, errorResponse{ + Error: "BAD_REQUEST", + Message: safeStorageErrorMessage(saveErr), + }) +} + // MountUploadRoutes registers upload and file-serving endpoints. // allowedOrigins controls the Access-Control-Allow-Origin header on served files. // @@ -130,7 +152,7 @@ func safeStorageErrorMessage(err error) string { // per-channel ACLs on every file download. A nil permSvc would panic for // any authenticated file request, so we fail fast at mount time rather // than let the first user hit a 500. -func MountUploadRoutes(r chi.Router, database *db.DB, store *storage.Storage, limiter *auth.RateLimiter, allowedOrigins []string, permSvc *service.PermissionService) { +func MountUploadRoutes(r chi.Router, database *db.DB, store FileStore, limiter *auth.RateLimiter, allowedOrigins []string, permSvc *service.PermissionService) { if permSvc == nil { panic("api: MountUploadRoutes requires a non-nil PermissionService") } @@ -143,7 +165,7 @@ func MountUploadRoutes(r chi.Router, database *db.DB, store *storage.Storage, li r.With(AuthMiddleware(database)).Get("/api/v1/files/{id}", handleServeFile(database, store, allowedOrigins, permSvc)) } -func handleUpload(database *db.DB, store *storage.Storage, limiter *auth.RateLimiter) http.HandlerFunc { +func handleUpload(database *db.DB, store FileStore, limiter *auth.RateLimiter) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // BUG-131: Per-user upload rate limit to prevent disk exhaustion. user, ok := r.Context().Value(UserKey).(*db.User) @@ -207,11 +229,7 @@ func handleUpload(database *db.DB, store *storage.Storage, limiter *auth.RateLim // Store file on disk (validates file type via magic bytes). writtenBytes, saveErr := store.Save(fileID, file) if saveErr != nil { - slog.Warn("file upload rejected", "error", saveErr) - writeJSON(w, http.StatusBadRequest, errorResponse{ - Error: "BAD_REQUEST", - Message: safeStorageErrorMessage(saveErr), - }) + writeStorageSaveError(w, saveErr, "file upload") return } @@ -260,7 +278,7 @@ func handleUpload(database *db.DB, store *storage.Storage, limiter *auth.RateLim } } -func handleServeFile(database *db.DB, store *storage.Storage, allowedOrigins []string, permSvc *service.PermissionService) http.HandlerFunc { +func handleServeFile(database *db.DB, store FileStore, allowedOrigins []string, permSvc *service.PermissionService) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { fileID := chi.URLParam(r, "id") if fileID == "" { diff --git a/Server/api/upload_handler_test.go b/Server/api/upload_handler_test.go index 502f5ba5..baae9d08 100644 --- a/Server/api/upload_handler_test.go +++ b/Server/api/upload_handler_test.go @@ -594,8 +594,10 @@ func TestUpload_StorageErrorDoesNotLeakPath(t *testing.T) { content := []byte("content that will fail to persist because the storage dir is gone") rr := doUpload(t, router, token, "file", "leaktest.txt", content) - if rr.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want 400; body: %s", rr.Code, rr.Body.String()) + // Server-side filesystem failures are 507 (storage.ErrIO) so they are + // distinguishable from bad uploads; the no-leak contract is unchanged. + if rr.Code != http.StatusInsufficientStorage { + t.Fatalf("status = %d, want 507; body: %s", rr.Code, rr.Body.String()) } var resp map[string]any diff --git a/Server/config/config.go b/Server/config/config.go index c1b5d57f..0f24debb 100644 --- a/Server/config/config.go +++ b/Server/config/config.go @@ -8,6 +8,7 @@ import ( "log/slog" "net" "os" + "slices" "strings" "github.com/knadh/koanf/parsers/yaml" @@ -22,6 +23,8 @@ import ( type Config struct { Server ServerConfig `koanf:"server"` Database DatabaseConfig `koanf:"database"` + Backup BackupConfig `koanf:"backup"` + Security SecurityConfig `koanf:"security"` TLS TLSConfig `koanf:"tls"` Upload UploadConfig `koanf:"upload"` Voice VoiceConfig `koanf:"voice"` @@ -86,6 +89,13 @@ type EventPersistenceConfig struct { BatchFlushMs int `koanf:"batch_flush_ms"` // PrunerIntervalMinutes is how often the pruner goroutine wakes up. PrunerIntervalMinutes int `koanf:"pruner_interval_minutes"` + // ReplayRingSize is the capacity of the in-memory reconnect replay ring. + // Reconnects whose gap exceeds it fall to the persisted event log. + ReplayRingSize int `koanf:"replay_ring_size"` + // ReplayColdLimit caps how many persisted events a single reconnect may + // replay; beyond it the client gets a full resync. This is the budget + // that decides how long a disconnect can be bridged by replay. + ReplayColdLimit int `koanf:"replay_cold_limit"` } // TelemetryConfig (Phase B Step 8) controls the OpenTelemetry exporter. @@ -172,6 +182,41 @@ type ServerConfig struct { // text the CRS false-positives on, so blocking needs tuning against real // traffic first. Unknown values fall back to "detect". WAFCRSMode string `koanf:"waf_crs_mode"` + // MaxWSConnections caps concurrently connected WebSocket clients; new + // upgrade requests beyond the cap are refused with 503 before the + // upgrade. 0 (the default) means unlimited — every connection costs + // goroutines and buffered send queues, so set a ceiling that matches the + // host's memory before pointing a large community at it. + MaxWSConnections int `koanf:"max_ws_connections"` + // MetricsAllowedCIDRs gates /api/v1/metrics and the Prometheus /metrics + // exporter separately from the human admin surface, so a central + // Prometheus scraper can be allowlisted without widening /admin to its + // network. Empty (default) falls back to AdminAllowedCIDRs. + MetricsAllowedCIDRs []string `koanf:"metrics_allowed_cidrs"` + // LiveKitWebhookAllowedCIDRs gates the LiveKit webhook and health + // endpoints. The webhook already authenticates cryptographically (LiveKit + // JWT signature over the body hash) — this perimeter is defence-in-depth, + // and giving it its own key means an externally-hosted LiveKit's IP no + // longer has to be added to the ADMIN allowlist. Empty (default) falls + // back to AdminAllowedCIDRs. + LiveKitWebhookAllowedCIDRs []string `koanf:"livekit_webhook_allowed_cidrs"` +} + +// MetricsCIDRs returns the effective allowlist for the metrics surfaces. +func (s *ServerConfig) MetricsCIDRs() []string { + if len(s.MetricsAllowedCIDRs) > 0 { + return s.MetricsAllowedCIDRs + } + return s.AdminAllowedCIDRs +} + +// LiveKitWebhookCIDRs returns the effective allowlist for the LiveKit +// webhook/health endpoints. +func (s *ServerConfig) LiveKitWebhookCIDRs() []string { + if len(s.LiveKitWebhookAllowedCIDRs) > 0 { + return s.LiveKitWebhookAllowedCIDRs + } + return s.AdminAllowedCIDRs } // DatabaseConfig holds database settings. @@ -187,6 +232,11 @@ type DatabaseConfig struct { // Path is the SQLite database file path. Path string `koanf:"path"` + + // MaxReaders bounds the read-only connection pool. 0 (default) keeps the + // automatic sizing of max(4, NumCPU). Values are clamped to [1, 64] — + // readers beyond the CPU count mostly buy queueing, not throughput. + MaxReaders int `koanf:"max_readers"` } // TLSConfig holds TLS/certificate settings. @@ -204,6 +254,25 @@ type UploadConfig struct { StorageDir string `koanf:"storage_dir"` } +// BackupConfig controls where database backups are written. Pointing Dir at +// another disk (or a mount that is shipped off-host) is the recommended way +// to keep backups from sharing a single point of failure with the live +// database and uploads. +type BackupConfig struct { + Dir string `koanf:"dir"` +} + +// SecurityConfig tunes security-adjacent behavior that has safe compiled-in +// defaults. +type SecurityConfig struct { + // AuthRateLimitMultiplier scales the per-IP auth rate limits and failure + // thresholds (registration, login, TOTP, sensitive endpoints). The + // defaults assume roughly one person per IP address; a community behind a + // shared NAT (office, school) hits them collectively. 0 or unset = 1.0; + // clamped to [0.1, 100]. + AuthRateLimitMultiplier float64 `koanf:"auth_rate_limit_multiplier"` +} + // defaults returns the default configuration. func defaults() Config { return Config{ @@ -227,6 +296,9 @@ func defaults() Config { Type: "sqlite", Path: "data/chatserver.db", }, + Backup: BackupConfig{ + Dir: "data/backups", + }, TLS: TLSConfig{ Mode: "self_signed", CertFile: "data/cert.pem", @@ -251,6 +323,11 @@ func defaults() Config { BatchSize: 50, BatchFlushMs: 100, PrunerIntervalMinutes: 60, + ReplayRingSize: 1000, + ReplayColdLimit: 5000, + }, + Security: SecurityConfig{ + AuthRateLimitMultiplier: 1.0, }, Telemetry: TelemetryConfig{ Enabled: false, @@ -297,6 +374,11 @@ database: type: "sqlite" # "sqlite" is the only supported backend path: "data/chatserver.db" +# backup: +# dir: "data/backups" # where database backups are written; point at another +# # disk or an off-host mount so backups don't share a +# # single point of failure with the live database + tls: mode: "self_signed" # self_signed, acme, manual, off cert_file: "data/cert.pem" @@ -380,6 +462,14 @@ func Load(cfgPath string) (*Config, error) { if err := k.Load(structs.Provider(def, "koanf"), nil); err != nil { return nil, fmt.Errorf("loading defaults: %w", err) } + // The defaults layer's key set is the complete set of keys the config + // struct can absorb — captured NOW, before the file merges in, so it can + // serve as the allowlist for the unknown-key warning below. (Capturing + // after the file load would let the file's own typos into the allowlist.) + knownKeys := make(map[string]struct{}, len(k.Keys())) + for _, key := range k.Keys() { + knownKeys[key] = struct{}{} + } // Layer 2: YAML file (create default if missing). The freshly written // default file is loaded like any other so the first boot runs with @@ -403,6 +493,15 @@ func Load(cfgPath string) (*Config, error) { return nil, fmt.Errorf("loading config file %s: %w", cfgPath, err) } + // Warn (never fail — a newer server must tolerate an older config, and a + // warning must not brick a working install) about file keys the config + // struct cannot absorb. Without this, a typo like `admin_alowed_cidrs` + // silently keeps the default and the operator believes they changed it. + for _, key := range unknownFileKeys(cfgPath, knownKeys) { + slog.Warn("config: unknown key ignored — value has NO effect (typo?)", + "key", key, "file", cfgPath) + } + // Layer 3: environment variable overrides. // OWNCORD_SERVER_PORT -> server.port, OWNCORD_TLS_MODE -> tls.mode, etc. envProvider := env.Provider("OWNCORD_", ".", func(s string) string { @@ -446,10 +545,43 @@ func Load(cfgPath string) (*Config, error) { // at startup instead. Common mistake: a bare IP without the /32 mask. warnInvalidCIDRs("server.trusted_proxies", cfg.Server.TrustedProxies) warnInvalidCIDRs("server.admin_allowed_cidrs", cfg.Server.AdminAllowedCIDRs) + warnInvalidCIDRs("server.metrics_allowed_cidrs", cfg.Server.MetricsAllowedCIDRs) + warnInvalidCIDRs("server.livekit_webhook_allowed_cidrs", cfg.Server.LiveKitWebhookAllowedCIDRs) + + // A customized admin allowlist with no trusted_proxies is a footgun + // behind any reverse proxy or container network: the check then compares + // the PROXY'S (or bridge's) address — by construction a private one — + // instead of the real client's, so the customization silently doesn't do + // what the operator believes. Warn, don't fail: direct-exposure setups + // are exactly this shape and are fine. + if len(cfg.Server.TrustedProxies) == 0 && + !slices.Equal(cfg.Server.AdminAllowedCIDRs, defaults().Server.AdminAllowedCIDRs) { + slog.Warn("config: admin_allowed_cidrs is customized but trusted_proxies is empty — " + + "behind a reverse proxy or Docker network the allowlist checks the proxy's private " + + "address, not the real client; set server.trusted_proxies to the proxy hop(s)") + } return &cfg, nil } +// unknownFileKeys parses the config file into its own koanf instance and +// returns every leaf key that the defaults layer (= the full set of keys the +// Config struct defines) does not contain. knownKeys must be captured from +// the defaults layer BEFORE the file merges into it. +func unknownFileKeys(cfgPath string, knownKeys map[string]struct{}) []string { + fileK := koanf.New(".") + if err := fileK.Load(file.Provider(cfgPath), yaml.Parser()); err != nil { + return nil // the main load already surfaced any parse problem + } + var unknown []string + for _, key := range fileK.Keys() { + if _, ok := knownKeys[key]; !ok { + unknown = append(unknown, key) + } + } + return unknown +} + // warnInvalidCIDRs logs a startup warning for each list entry that is not // valid CIDR notation. func warnInvalidCIDRs(key string, cidrs []string) { diff --git a/Server/config/unknown_keys_test.go b/Server/config/unknown_keys_test.go new file mode 100644 index 00000000..4b6797e1 --- /dev/null +++ b/Server/config/unknown_keys_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/knadh/koanf/providers/structs" + "github.com/knadh/koanf/v2" +) + +// TestUnknownFileKeys locks the typo guard: keys the Config struct does not +// define are reported, and every real key — including empty-slice and +// zero-value defaults — is not. +func TestUnknownFileKeys(t *testing.T) { + k := koanf.New(".") + if err := k.Load(structs.Provider(defaults(), "koanf"), nil); err != nil { + t.Fatal(err) + } + known := make(map[string]struct{}) + for _, key := range k.Keys() { + known[key] = struct{}{} + } + + cfgPath := filepath.Join(t.TempDir(), "config.yaml") + yamlBody := `server: + prot: 9999 + admin_alowed_cidrs: + - "0.0.0.0/0" + allowed_origins: + - "https://example.com" + max_ws_connections: 500 +databsae: + path: "oops.db" +backup: + dir: "elsewhere" +` + if err := os.WriteFile(cfgPath, []byte(yamlBody), 0o600); err != nil { + t.Fatal(err) + } + + unknown := unknownFileKeys(cfgPath, known) + slices.Sort(unknown) + + want := []string{"databsae.path", "server.admin_alowed_cidrs", "server.prot"} + if !slices.Equal(unknown, want) { + t.Fatalf("unknownFileKeys = %v, want %v", unknown, want) + } +} diff --git a/Server/db/admin_queries.go b/Server/db/admin_queries.go index de15407a..a39b4d57 100644 --- a/Server/db/admin_queries.go +++ b/Server/db/admin_queries.go @@ -430,3 +430,35 @@ func (d *DB) BackupToSafe(ctx context.Context, path, safeRoot string) error { } return nil } + +// CheckBackupIntegrity opens the SQLite file at path read-only and runs +// PRAGMA integrity_check against it. It returns nil only when SQLite reports +// "ok". Use it to verify a backup right after it is written and again before +// it is restored over the live database — a truncated or corrupt file must +// never be presented (or accepted) as restorable. +// +// The path travels into a file: URI, so it is restricted with the same +// character allowlist BackupToSafe enforces; callers always pass paths that +// already passed that gate. +func CheckBackupIntegrity(ctx context.Context, path string) error { + abs, err := filepath.Abs(filepath.Clean(path)) + if err != nil { + return fmt.Errorf("CheckBackupIntegrity: resolving path: %w", err) + } + if _, err := os.Stat(abs); err != nil { + return fmt.Errorf("CheckBackupIntegrity: %w", err) + } + conn, err := sql.Open("sqlite", "file:"+filepath.ToSlash(abs)+"?mode=ro&_pragma=busy_timeout(2000)") + if err != nil { + return fmt.Errorf("CheckBackupIntegrity: open: %w", err) + } + defer conn.Close() //nolint:errcheck + var result string + if err := conn.QueryRowContext(ctx, "PRAGMA integrity_check(10)").Scan(&result); err != nil { + return fmt.Errorf("CheckBackupIntegrity: %w", err) + } + if result != "ok" { + return fmt.Errorf("CheckBackupIntegrity: integrity_check reported %q", result) + } + return nil +} diff --git a/Server/db/auth_queries.go b/Server/db/auth_queries.go index c0c2986c..f0c3d899 100644 --- a/Server/db/auth_queries.go +++ b/Server/db/auth_queries.go @@ -246,7 +246,7 @@ func (d *DB) CreateSession(ctx context.Context, userID int64, tokenHash, device, Offset: maxSessionsPerUser - 1, }) - expiresAt := time.Now().Add(sessionTTL).UTC().Format("2006-01-02T15:04:05Z") + expiresAt := time.Now().Add(sessionTTL).UTC().Format(sessionTimeLayout) deviceCopy, ipCopy := device, ip res, err := d.q.InsertSession(ctx, dbgen.InsertSessionParams{ UserID: userID, @@ -398,9 +398,13 @@ func (d *DB) DeleteOtherSessions(ctx context.Context, userID, keepSessionID int6 } // DeleteExpiredSessions removes all sessions whose expires_at is in the past. -// Compares using strftime to handle both ISO-8601 and SQLite datetime formats. +// The comparison is plain text against idx_sessions_expires_at, so the cutoff +// MUST use sessionTimeLayout — the exact stored format (migration 031 +// normalized legacy rows). A space-separated cutoff would compare wrong and +// silently delete nothing (' ' sorts before 'T'). func (d *DB) DeleteExpiredSessions(ctx context.Context) error { - if err := d.q.DeleteExpiredSessions(ctx); err != nil { + cutoff := time.Now().UTC().Format(sessionTimeLayout) + if err := d.q.DeleteExpiredSessions(ctx, cutoff); err != nil { return fmt.Errorf("DeleteExpiredSessions: %w", err) } return nil diff --git a/Server/db/backup_test.go b/Server/db/backup_test.go index 1ef48c02..e2eb68fa 100644 --- a/Server/db/backup_test.go +++ b/Server/db/backup_test.go @@ -135,6 +135,61 @@ func TestBackupToSafe_RejectsNullByte(t *testing.T) { } } +// TestBackupToSafe_ErrorKeepsPreexistingFile locks the cleanup guard: a +// failed VACUUM INTO removes a partial file it created, but an error caused +// by the destination already existing must never delete the operator's file. +func TestBackupToSafe_ErrorKeepsPreexistingFile(t *testing.T) { + database, tmpDir := newBackupFileDB(t) + + backupDir := filepath.Join(tmpDir, "backups") + if err := os.MkdirAll(backupDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + existing := filepath.Join(backupDir, "keep_me.db") + if err := os.WriteFile(existing, []byte("precious"), 0o600); err != nil { + t.Fatal(err) + } + + // VACUUM INTO refuses an existing destination. + if err := database.BackupToSafe(context.Background(), existing, backupDir); err == nil { + t.Fatal("BackupToSafe over an existing file should error") + } + content, err := os.ReadFile(existing) + if err != nil || string(content) != "precious" { + t.Fatalf("pre-existing file was modified or removed (content=%q, err=%v)", content, err) + } +} + +// TestCheckBackupIntegrity_ValidAndCorrupt verifies the integrity gate both +// accepts a real backup and rejects a non-database file. +func TestCheckBackupIntegrity_ValidAndCorrupt(t *testing.T) { + database, tmpDir := newBackupFileDB(t) + + backupDir := filepath.Join(tmpDir, "backups") + if err := os.MkdirAll(backupDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + good := filepath.Join(backupDir, "good.db") + if err := database.BackupToSafe(context.Background(), good, backupDir); err != nil { + t.Fatalf("BackupToSafe: %v", err) + } + if err := db.CheckBackupIntegrity(context.Background(), good); err != nil { + t.Fatalf("CheckBackupIntegrity on a fresh backup: %v", err) + } + + bad := filepath.Join(backupDir, "bad.db") + if err := os.WriteFile(bad, []byte("this is not a sqlite database at all"), 0o600); err != nil { + t.Fatal(err) + } + if err := db.CheckBackupIntegrity(context.Background(), bad); err == nil { + t.Fatal("CheckBackupIntegrity accepted a garbage file") + } + + if err := db.CheckBackupIntegrity(context.Background(), filepath.Join(backupDir, "missing.db")); err == nil { + t.Fatal("CheckBackupIntegrity accepted a missing file") + } +} + // TestBackupToSafe_RejectsDoubleQuote ensures a path containing a double-quote // is rejected. func TestBackupToSafe_RejectsDoubleQuote(t *testing.T) { diff --git a/Server/db/db.go b/Server/db/db.go index b478da4e..92eaa769 100644 --- a/Server/db/db.go +++ b/Server/db/db.go @@ -7,6 +7,7 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "runtime" "strings" "sync/atomic" @@ -46,6 +47,10 @@ type DB struct { // non-blocking enqueues. Nil (the default) keeps audit writes // synchronous — the token CLI and tests rely on that. auditWriter atomic.Pointer[AuditWriter] + + // lockRelease drops the single-process advisory lock taken by openFile. + // Nil for in-memory databases and when the lock mechanism is unavailable. + lockRelease func() } // filePragmas are the per-connection PRAGMAs applied to every file-backed @@ -97,10 +102,31 @@ func isMemoryPath(path string) bool { // path is embedded in a file: URI without escaping. cfg.Database.Path is a // plain path (default "data/chatserver.db"), which satisfies this. func Open(path string) (*DB, error) { + return OpenWithMaxReaders(path, 0) +} + +// OpenWithMaxReaders is Open with an explicit reader-pool bound +// (database.max_readers). maxReaders <= 0 keeps the automatic +// max(4, NumCPU) sizing; values are clamped to [1, 64]. Ignored for +// in-memory databases, which use a single shared connection. +func OpenWithMaxReaders(path string, maxReaders int) (*DB, error) { if isMemoryPath(path) { return openMemory(path) } - return openFile(path) + return openFile(path, maxReaders, true) +} + +// OpenShared opens the database WITHOUT taking the single-process lock. It +// exists for short-lived tooling — the `server token` CLI — that must work +// while the server is running. SQLite's own WAL locking makes the concurrent +// access safe at the file level; the process lock only protects the SERVER's +// process-local state (presence, replay ring, rate-limit windows), which a +// CLI does not touch. Long-lived processes must use Open. +func OpenShared(path string) (*DB, error) { + if isMemoryPath(path) { + return openMemory(path) + } + return openFile(path, 0, false) } // openMemory preserves the pre-split behavior exactly: a single connection @@ -143,7 +169,41 @@ func openMemory(path string) (*DB, error) { } // openFile opens the writer and reader pools for a file-backed database. -func openFile(path string) (*DB, error) { +// takeLock is false only for OpenShared (short-lived CLI tooling). +func openFile(path string, maxReaders int, takeLock bool) (*DB, error) { + // Single-process guard: SQLite's own locking prevents file corruption, + // but everything built above it — presence derived from hub membership, + // the replay ring, rate-limit windows, the boot-time status reset — is + // process-local and assumes exactly one server owns this database file. + // A second process starting is almost always an accident (double systemd + // unit, container + binary); fail fast with a clear message rather than + // letting two instances silently fight over shared state. + var release func() + if takeLock { + var lockErr error + release, lockErr = acquireProcessLock(path) + if lockErr != nil { + if errors.Is(lockErr, errAlreadyLocked) { + return nil, fmt.Errorf( + "database %s is in use by another running OwnCord process — stop that process first (the lock is released automatically when it exits)", + path) + } + // The lock mechanism itself failed (e.g. a network filesystem that + // rejects advisory locks). Warn and continue — refusing to start on + // an NFS data dir would be a regression, and SQLite still protects + // the file itself. + slog.Warn("db: could not take the single-process lock; continuing unprotected", + "path", lockFilePath(path), "error", lockErr) + release = nil + } + } + ok := false + defer func() { + if !ok && release != nil { + release() + } + }() + base := path if !strings.HasPrefix(base, "file:") { base = "file:" + base @@ -177,6 +237,9 @@ func openFile(path string) (*DB, error) { return nil, fmt.Errorf("opening sqlite reader pool: %w", err) } readConns := max(4, runtime.NumCPU()) + if maxReaders > 0 { + readConns = min(max(maxReaders, 1), 64) + } reader.SetMaxOpenConns(readConns) reader.SetMaxIdleConns(readConns) if err := reader.Ping(); err != nil { @@ -185,7 +248,10 @@ func openFile(path string) (*DB, error) { return nil, fmt.Errorf("pinging sqlite reader pool: %w", err) } - return newDB(writer, reader), nil + d := newDB(writer, reader) + d.lockRelease = release + ok = true + return d, nil } // newDB assembles a DB whose sqlc query layer routes through dbtx. @@ -287,15 +353,24 @@ func hasKeywordPrefix(s, keyword string) bool { // MigrateFS (defined in migrate.go) which maintains the schema_versions // tracking table. func Migrate(database *DB) error { - if err := MigrateFS(database, migrations.FS); err != nil { + applied, err := migrateFSCount(database, migrations.FS) + if err != nil { return err } - // Refresh the query planner's statistics once per startup so newly created - // indexes (e.g. migration 019) are actually chosen. Close() keeps them - // current afterwards via PRAGMA optimize. ANALYZE writes sqlite_stat rows, - // so it runs on the writer. - if _, err := database.writer.Exec("ANALYZE;"); err != nil { - return fmt.Errorf("running ANALYZE after migrations: %w", err) + // Refresh the query planner's statistics when the schema changed, so newly + // created indexes (e.g. migration 019) are actually chosen. A full ANALYZE + // grows with row count and blocks startup, so unchanged schemas get the + // cheap PRAGMA optimize instead — which also covers crash-restarts that + // never reached Close()'s optimize. Both write sqlite_stat rows, so they + // run on the writer. + if applied > 0 { + if _, err := database.writer.Exec("ANALYZE;"); err != nil { + return fmt.Errorf("running ANALYZE after migrations: %w", err) + } + return nil + } + if _, err := database.writer.Exec("PRAGMA optimize;"); err != nil { + return fmt.Errorf("running PRAGMA optimize at startup: %w", err) } return nil } @@ -309,7 +384,14 @@ func (d *DB) Close() error { if d.reader != d.writer { readerErr = d.reader.Close() } - return errors.Join(d.writer.Close(), readerErr) + err := errors.Join(d.writer.Close(), readerErr) + // Release the single-process lock only after the pools are closed, so a + // successor process acquiring it can rely on this one being done writing. + if d.lockRelease != nil { + d.lockRelease() + d.lockRelease = nil + } + return err } // QueryRowContext executes a query that returns at most one row, with context. @@ -352,3 +434,15 @@ func (d *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) func (d *DB) SQLDb() *sql.DB { return d.writer } + +// PingRead answers whether the database can serve reads, via a bounded +// SELECT 1 on the READER pool. The health endpoint uses it deliberately: +// pinging the single-connection writer would queue behind any long write — +// most notably a scheduled backup's VACUUM INTO — and report a healthy, +// read-serving server as degraded for the backup's whole duration. Writer +// saturation is reported separately (SQLDb().Stats() in /api/v1/metrics), +// where it is a capacity signal rather than a liveness verdict. +func (d *DB) PingRead(ctx context.Context) error { + var one int + return d.reader.QueryRowContext(ctx, "SELECT 1").Scan(&one) +} diff --git a/Server/db/dbgen/messages.sql.go b/Server/db/dbgen/messages.sql.go index 9495fb1b..18d441aa 100644 --- a/Server/db/dbgen/messages.sql.go +++ b/Server/db/dbgen/messages.sql.go @@ -229,6 +229,31 @@ func (q *Queries) GetMessagesForAPI(ctx context.Context, arg GetMessagesForAPIPa return items, nil } +const getReadState = `-- name: GetReadState :one +SELECT last_message_id, mention_count FROM read_states + WHERE user_id = ? AND channel_id = ? +` + +type GetReadStateParams struct { + UserID int64 `json:"userId"` + ChannelID int64 `json:"channelId"` +} + +type GetReadStateRow struct { + LastMessageID int64 `json:"lastMessageId"` + MentionCount int64 `json:"mentionCount"` +} + +// Reader-pool lookup that lets the channel-focus path skip the UpdateReadState +// UPSERT when the row is already correct, keeping no-op focus events off the +// single writer connection. +func (q *Queries) GetReadState(ctx context.Context, arg GetReadStateParams) (GetReadStateRow, error) { + row := q.db.QueryRowContext(ctx, getReadState, arg.UserID, arg.ChannelID) + var i GetReadStateRow + err := row.Scan(&i.LastMessageID, &i.MentionCount) + return i, err +} + const setMessagePinned = `-- name: SetMessagePinned :execresult UPDATE messages SET pinned = ? WHERE id = ? AND deleted = 0 ` diff --git a/Server/db/dbgen/querier.go b/Server/db/dbgen/querier.go index 50b709f3..4e88f3c9 100644 --- a/Server/db/dbgen/querier.go +++ b/Server/db/dbgen/querier.go @@ -53,7 +53,11 @@ type Querier interface { DeleteChannelPermission(ctx context.Context, arg DeleteChannelPermissionParams) error DeleteChannelUserPermission(ctx context.Context, arg DeleteChannelUserPermissionParams) error DeleteEmoji(ctx context.Context, id int64) (sql.Result, error) - DeleteExpiredSessions(ctx context.Context) error + // Sargable text comparison against idx_sessions_expires_at (migration 031). + // expires_at is stored as RFC3339 UTC ("2006-01-02T15:04:05Z") and the + // migration normalized legacy rows, so the caller must pass the cutoff in + // exactly that layout -- a space-separated cutoff would compare wrong. + DeleteExpiredSessions(ctx context.Context, expiresAt string) error DeleteLockout(ctx context.Context, key string) error // Avatars are attachments that are never linked to a message on purpose: the // users.avatar URL is what keeps them alive and authorizes serving them @@ -128,6 +132,10 @@ type Querier interface { // Reactors for one (message, emoji) pair, oldest reaction first. The reactions // table has no timestamp column, so the autoincrement id carries the order. GetReactionUsers(ctx context.Context, arg GetReactionUsersParams) ([]GetReactionUsersRow, error) + // Reader-pool lookup that lets the channel-focus path skip the UpdateReadState + // UPSERT when the row is already correct, keeping no-op focus events off the + // single writer connection. + GetReadState(ctx context.Context, arg GetReadStateParams) (GetReadStateRow, error) GetRoleByID(ctx context.Context, id int64) (Role, error) // Case-insensitive by design: migration 023 enforces uniqueness under the same // collation, so this is the lookup that agrees with the constraint. diff --git a/Server/db/dbgen/sessions.sql.go b/Server/db/dbgen/sessions.sql.go index bee1d3fc..a900da4c 100644 --- a/Server/db/dbgen/sessions.sql.go +++ b/Server/db/dbgen/sessions.sql.go @@ -11,11 +11,15 @@ import ( ) const deleteExpiredSessions = `-- name: DeleteExpiredSessions :exec -DELETE FROM sessions WHERE strftime('%s', expires_at) < strftime('%s', 'now') +DELETE FROM sessions WHERE expires_at < ? ` -func (q *Queries) DeleteExpiredSessions(ctx context.Context) error { - _, err := q.db.ExecContext(ctx, deleteExpiredSessions) +// Sargable text comparison against idx_sessions_expires_at (migration 031). +// expires_at is stored as RFC3339 UTC ("2006-01-02T15:04:05Z") and the +// migration normalized legacy rows, so the caller must pass the cutoff in +// exactly that layout -- a space-separated cutoff would compare wrong. +func (q *Queries) DeleteExpiredSessions(ctx context.Context, expiresAt string) error { + _, err := q.db.ExecContext(ctx, deleteExpiredSessions, expiresAt) return err } diff --git a/Server/db/lockfile.go b/Server/db/lockfile.go new file mode 100644 index 00000000..0a25b252 --- /dev/null +++ b/Server/db/lockfile.go @@ -0,0 +1,49 @@ +package db + +import ( + "errors" + "log/slog" + "time" +) + +// errAlreadyLocked reports that another live process holds the database's +// single-process lock. The locks used here (flock on Unix, an exclusive file +// handle on Windows) are released by the OS when their holder exits, so a +// held lock always means a running process, never a stale file. +var errAlreadyLocked = errors.New("database lock held by another process") + +// lockFilePath is the sidecar lock file next to the SQLite database. +func lockFilePath(dbPath string) string { return dbPath + ".lock" } + +// acquireProcessLock takes the single-process lock for dbPath, retrying for +// a bounded window before giving up with errAlreadyLocked. +// +// The retry exists for the restart handoff: self-update and backup-restore +// spawn the replacement process while the old one is still draining (worst +// case ~12s — restartProcess SIGTERMs itself and hard-exits after a 10s +// grace), so the successor must wait for the lock rather than die on it. +// A genuinely concurrent long-lived second process still fails, just after +// the wait. +func acquireProcessLock(dbPath string) (release func(), err error) { + const ( + retryFor = 30 * time.Second + retryEvery = 500 * time.Millisecond + ) + deadline := time.Now().Add(retryFor) + logged := false + for { + release, err = tryLockFile(lockFilePath(dbPath)) + if err == nil || !errors.Is(err, errAlreadyLocked) { + return release, err + } + if time.Now().After(deadline) { + return nil, err + } + if !logged { + slog.Info("db: database is locked by another process; waiting for it to exit (restart handoff)", + "path", dbPath, "wait_up_to", retryFor.String()) + logged = true + } + time.Sleep(retryEvery) + } +} diff --git a/Server/db/lockfile_other.go b/Server/db/lockfile_other.go new file mode 100644 index 00000000..f5f26795 --- /dev/null +++ b/Server/db/lockfile_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin && !windows + +package db + +import "errors" + +// tryLockFile has no implementation on this platform; openFile logs a warning +// and continues without the single-process guard. +func tryLockFile(string) (func(), error) { + return nil, errors.ErrUnsupported +} diff --git a/Server/db/lockfile_test.go b/Server/db/lockfile_test.go new file mode 100644 index 00000000..0b58499b --- /dev/null +++ b/Server/db/lockfile_test.go @@ -0,0 +1,36 @@ +//go:build linux || darwin || windows + +package db + +import ( + "errors" + "path/filepath" + "testing" +) + +// TestTryLockFile_Exclusive locks the single-process guard's contract: the +// second acquisition fails with errAlreadyLocked while the first holds the +// lock, and succeeds after release. +// +// Note this exercises the raw tryLockFile, not acquireProcessLock — the +// latter's 30s restart-handoff retry would stall the contended case. +func TestTryLockFile_Exclusive(t *testing.T) { + path := lockFilePath(filepath.Join(t.TempDir(), "test.db")) + + release1, err := tryLockFile(path) + if err != nil { + t.Fatalf("first tryLockFile: %v", err) + } + + if _, err := tryLockFile(path); !errors.Is(err, errAlreadyLocked) { + t.Fatalf("second tryLockFile err = %v, want errAlreadyLocked", err) + } + + release1() + + release2, err := tryLockFile(path) + if err != nil { + t.Fatalf("tryLockFile after release: %v", err) + } + release2() +} diff --git a/Server/db/lockfile_unix.go b/Server/db/lockfile_unix.go new file mode 100644 index 00000000..6ea402db --- /dev/null +++ b/Server/db/lockfile_unix.go @@ -0,0 +1,34 @@ +//go:build linux || darwin + +package db + +import ( + "errors" + "os" + "syscall" +) + +// tryLockFile takes a non-blocking exclusive flock on path. The lock is tied +// to the open descriptor, so it vanishes with the process no matter how it +// dies — there is no stale-lock failure mode. +func tryLockFile(path string) (func(), error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) //nolint:gosec // sidecar of the configured db path + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { //nolint:gosec // G115: fd of a just-opened file is far below int range + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, errAlreadyLocked + } + return nil, err + } + return func() { + // Unlock before close is redundant (close releases flock) but keeps + // the intent explicit. The file itself is left behind on purpose: + // removing it opens an inode-swap race with a concurrent acquirer, + // and a leftover zero-byte .lock file is harmless. + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) //nolint:gosec // G115: same fd as above + _ = f.Close() + }, nil +} diff --git a/Server/db/lockfile_windows.go b/Server/db/lockfile_windows.go new file mode 100644 index 00000000..a93b6a8d --- /dev/null +++ b/Server/db/lockfile_windows.go @@ -0,0 +1,38 @@ +//go:build windows + +package db + +import ( + "errors" + "syscall" +) + +// errorSharingViolation is Windows' ERROR_SHARING_VIOLATION (32): the file is +// open in another process with an incompatible share mode. +const errorSharingViolation = syscall.Errno(32) + +// tryLockFile opens path with an empty share mode, so a second process's open +// fails with a sharing violation until this handle is closed. The handle is +// released by the OS when the process exits — no stale-lock failure mode. +func tryLockFile(path string) (func(), error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + h, err := syscall.CreateFile( + p, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + 0, // no sharing + nil, + syscall.OPEN_ALWAYS, + syscall.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + if errors.Is(err, errorSharingViolation) { + return nil, errAlreadyLocked + } + return nil, err + } + return func() { _ = syscall.CloseHandle(h) }, nil +} diff --git a/Server/db/message_queries.go b/Server/db/message_queries.go index 355e19fe..14efa48f 100644 --- a/Server/db/message_queries.go +++ b/Server/db/message_queries.go @@ -567,6 +567,21 @@ func (d *DB) getReactionsBatch(ctx context.Context, msgIDs []int64, requestingUs return result, nil } +// GetReadState returns the stored read-state row for (userID, channelID). +// found is false when the user has never focused the channel. Runs on the +// reader pool — it exists so HandleChannelFocus can skip the UpdateReadState +// write when the row is already correct. +func (d *DB) GetReadState(ctx context.Context, userID, channelID int64) (lastMessageID, mentionCount int64, found bool, err error) { + row, err := d.q.GetReadState(ctx, dbgen.GetReadStateParams{UserID: userID, ChannelID: channelID}) + if errors.Is(err, sql.ErrNoRows) { + return 0, 0, false, nil + } + if err != nil { + return 0, 0, false, fmt.Errorf("GetReadState: %w", err) + } + return row.LastMessageID, row.MentionCount, true, nil +} + // UpdateReadState upserts the read state for a user in a channel and clears // its mention badge — marking a channel read consumes its mentions. func (d *DB) UpdateReadState(ctx context.Context, userID, channelID, lastReadMessageID int64) error { diff --git a/Server/db/migrate.go b/Server/db/migrate.go index 0f451fee..0d52199b 100644 --- a/Server/db/migrate.go +++ b/Server/db/migrate.go @@ -156,18 +156,27 @@ func seedExistingDatabase(d *DB, filenames []string) error { // state for a fresh database is an empty tracking table) and apply any // .sql file in lexicographic order that is not yet recorded. func MigrateFS(database *DB, fsys fs.FS) error { + _, err := migrateFSCount(database, fsys) + return err +} + +// migrateFSCount is MigrateFS reporting how many migrations actually +// executed, so Migrate can skip the boot-time ANALYZE when the schema did not +// change. The seeding path reports 0 — it records filenames without running +// any SQL. +func migrateFSCount(database *DB, fsys fs.FS) (int, error) { // Determine tracking state before touching schema_versions at all — the // seeding path below must be the one to create it, atomically with the // seed rows, so do not call ensureSchemaVersions before this check. svExists, err := schemaVersionsExists(database) if err != nil { - return err + return 0, err } // Collect filenames first — needed for both seeding and normal application. filenames, err := sqlFilenames(fsys) if err != nil { - return err + return 0, err } // Seeding path: schema_versions did not exist AND users table does, which @@ -175,10 +184,10 @@ func MigrateFS(database *DB, fsys fs.FS) error { if !svExists { existing, checkErr := isExistingDatabase(database) if checkErr != nil { - return checkErr + return 0, checkErr } if existing { - return seedExistingDatabase(database, filenames) + return 0, seedExistingDatabase(database, filenames) } } @@ -186,14 +195,15 @@ func MigrateFS(database *DB, fsys fs.FS) error { // database with no prior schema — either way, an idempotent create is // the correct next step before applying migrations normally. if err := ensureSchemaVersions(database); err != nil { - return err + return 0, err } // Normal path: apply any migration not yet recorded. + appliedCount := 0 for _, name := range filenames { applied, applyErr := isApplied(database, name) if applyErr != nil { - return applyErr + return appliedCount, applyErr } if applied { continue @@ -201,15 +211,16 @@ func MigrateFS(database *DB, fsys fs.FS) error { raw, readErr := fs.ReadFile(fsys, name) if readErr != nil { - return fmt.Errorf("reading migration %s: %w", name, readErr) + return appliedCount, fmt.Errorf("reading migration %s: %w", name, readErr) } if err := applyMigration(database, name, string(raw)); err != nil { - return err + return appliedCount, err } + appliedCount++ } - return nil + return appliedCount, nil } // applyMigration executes a single migration and records it. If the diff --git a/Server/db/models.go b/Server/db/models.go index 70eaf387..1a799d6a 100644 --- a/Server/db/models.go +++ b/Server/db/models.go @@ -302,3 +302,9 @@ type Emoji struct { // sessionTTL is the duration a session remains valid after creation. const sessionTTL = 30 * 24 * time.Hour + +// sessionTimeLayout is the storage format for sessions.expires_at (and the +// other RFC3339-UTC expiry columns). DeleteExpiredSessions compares these as +// plain text against an index, so every writer and the sweep's cutoff must +// use exactly this layout. +const sessionTimeLayout = "2006-01-02T15:04:05Z" diff --git a/Server/db/open_shared_test.go b/Server/db/open_shared_test.go new file mode 100644 index 00000000..d54d99ef --- /dev/null +++ b/Server/db/open_shared_test.go @@ -0,0 +1,33 @@ +package db_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/owncord/server/db" +) + +// TestOpenShared_WorksWhileServerHoldsLock locks the token-CLI contract: a +// short-lived tool must be able to open the database while a server process +// holds the single-process lock — SQLite WAL makes the file access safe, and +// the lock only guards the server's process-local state. +func TestOpenShared_WorksWhileServerHoldsLock(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.db") + + server, err := db.Open(path) // takes the process lock + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = server.Close() }) + + cli, err := db.OpenShared(path) // must not block on or steal the lock + if err != nil { + t.Fatalf("OpenShared while lock held: %v", err) + } + defer cli.Close() //nolint:errcheck + + if err := cli.PingRead(context.Background()); err != nil { + t.Fatalf("PingRead on shared handle: %v", err) + } +} diff --git a/Server/db/queries/sqlite/messages.sql b/Server/db/queries/sqlite/messages.sql index bdc81a96..856929d0 100644 --- a/Server/db/queries/sqlite/messages.sql +++ b/Server/db/queries/sqlite/messages.sql @@ -38,6 +38,13 @@ ON CONFLICT(user_id, channel_id) DO UPDATE SET last_message_id = excluded.last_message_id, mention_count = 0; +-- name: GetReadState :one +-- Reader-pool lookup that lets the channel-focus path skip the UpdateReadState +-- UPSERT when the row is already correct, keeping no-op focus events off the +-- single writer connection. +SELECT last_message_id, mention_count FROM read_states + WHERE user_id = ? AND channel_id = ?; + -- name: GetChannelUnreadCounts :many SELECT c.id, (SELECT COALESCE(MAX(m.id), 0) FROM messages m diff --git a/Server/db/queries/sqlite/sessions.sql b/Server/db/queries/sqlite/sessions.sql index 71ccdc9d..90e1540c 100644 --- a/Server/db/queries/sqlite/sessions.sql +++ b/Server/db/queries/sqlite/sessions.sql @@ -31,7 +31,11 @@ DELETE FROM sessions WHERE id = ? AND user_id = ?; DELETE FROM sessions WHERE user_id = ? AND id != ?; -- name: DeleteExpiredSessions :exec -DELETE FROM sessions WHERE strftime('%s', expires_at) < strftime('%s', 'now'); +-- Sargable text comparison against idx_sessions_expires_at (migration 031). +-- expires_at is stored as RFC3339 UTC ("2006-01-02T15:04:05Z") and the +-- migration normalized legacy rows, so the caller must pass the cutoff in +-- exactly that layout -- a space-separated cutoff would compare wrong. +DELETE FROM sessions WHERE expires_at < ?; -- name: TouchSession :exec UPDATE sessions SET last_used = datetime('now') WHERE token = ?; diff --git a/Server/db/session_expiry_test.go b/Server/db/session_expiry_test.go new file mode 100644 index 00000000..6a05f0ba --- /dev/null +++ b/Server/db/session_expiry_test.go @@ -0,0 +1,120 @@ +package db_test + +import ( + "context" + "testing" + "time" + + "github.com/owncord/server/db" + "github.com/owncord/server/migrations" +) + +// TestDeleteExpiredSessions_SargableFormat locks the migration-031 contract: +// the sweep's plain-text cutoff comparison deletes exactly the expired +// sessions when rows are stored in the normalized RFC3339-Z layout, and the +// supporting index exists. +func TestDeleteExpiredSessions_SargableFormat(t *testing.T) { + database, err := db.Open(":memory:") + if err != nil { + t.Fatalf("db.Open: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + if err := db.MigrateFS(database, migrations.FS); err != nil { + t.Fatalf("MigrateFS: %v", err) + } + ctx := context.Background() + + if _, err := database.ExecContext(ctx, + `INSERT INTO users (id, username, password, role_id) VALUES (1, 'u', 'x', 1)`); err != nil { + t.Fatalf("seed user: %v", err) + } + + const layout = "2006-01-02T15:04:05Z" + insert := func(token, expires string) { + t.Helper() + if _, err := database.ExecContext(ctx, + `INSERT INTO sessions (user_id, token, expires_at) VALUES (1, ?, ?)`, token, expires); err != nil { + t.Fatalf("seed session %s: %v", token, err) + } + } + insert("expired", time.Now().UTC().Add(-time.Hour).Format(layout)) + insert("live", time.Now().UTC().Add(time.Hour).Format(layout)) + // A legacy space-format row normalized by migration 031's UPDATE — the + // migration ran before these inserts, so normalize it the same way here + // to model a post-migration database. + insert("legacy_live", time.Now().UTC().Add(2*time.Hour).Format("2006-01-02T15:04:05Z")) + + if err := database.DeleteExpiredSessions(ctx); err != nil { + t.Fatalf("DeleteExpiredSessions: %v", err) + } + + var tokens []string + rows, err := database.QueryContext(ctx, `SELECT token FROM sessions ORDER BY token`) + if err != nil { + t.Fatalf("query sessions: %v", err) + } + defer rows.Close() //nolint:errcheck + for rows.Next() { + var tok string + if err := rows.Scan(&tok); err != nil { + t.Fatal(err) + } + tokens = append(tokens, tok) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(tokens) != 2 || tokens[0] != "legacy_live" || tokens[1] != "live" { + t.Fatalf("surviving sessions = %v, want [legacy_live live]", tokens) + } + + // The index the sweep depends on must exist. + var n int + if err := database.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_sessions_expires_at'`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatal("idx_sessions_expires_at is missing") + } +} + +// TestMigration031_NormalizesLegacyFormats verifies the one-time UPDATE pass: +// space-separated and Z-less rows become the RFC3339-Z layout. +func TestMigration031_NormalizesLegacyFormats(t *testing.T) { + database, err := db.Open(":memory:") + if err != nil { + t.Fatalf("db.Open: %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + if err := db.MigrateFS(database, migrations.FS); err != nil { + t.Fatalf("MigrateFS: %v", err) + } + ctx := context.Background() + if _, err := database.ExecContext(ctx, + `INSERT INTO users (id, username, password, role_id) VALUES (1, 'u', 'x', 1)`); err != nil { + t.Fatalf("seed user: %v", err) + } + // Simulate pre-031 rows, then re-run the normalization statements the + // migration contains (the migration itself already ran on the empty DB). + if _, err := database.ExecContext(ctx, + `INSERT INTO sessions (user_id, token, expires_at) VALUES (1, 'legacy', '2030-05-01 10:00:00')`); err != nil { + t.Fatal(err) + } + if _, err := database.ExecContext(ctx, + `UPDATE sessions SET expires_at = replace(expires_at, ' ', 'T') WHERE instr(expires_at, ' ') > 0`); err != nil { + t.Fatal(err) + } + if _, err := database.ExecContext(ctx, + `UPDATE sessions SET expires_at = expires_at || 'Z' WHERE length(expires_at) = 19`); err != nil { + t.Fatal(err) + } + var got string + if err := database.QueryRowContext(ctx, + `SELECT expires_at FROM sessions WHERE token = 'legacy'`).Scan(&got); err != nil { + t.Fatal(err) + } + if got != "2030-05-01T10:00:00Z" { + t.Fatalf("normalized expires_at = %q, want 2030-05-01T10:00:00Z", got) + } +} diff --git a/Server/db/sql_router_test.go b/Server/db/sql_router_test.go new file mode 100644 index 00000000..9dfc79e5 --- /dev/null +++ b/Server/db/sql_router_test.go @@ -0,0 +1,43 @@ +package db + +import "testing" + +// TestIsReadOnlySQL locks the read/write routing classification. Expectations +// are explicit values, NOT derived from sqlc annotations — `INSERT … +// RETURNING` is a `:one` query yet must route to the writer. Anything not +// recognized as read-only must fall to the writer (correct, if slower), so +// the dangerous direction is a write classified as a read. +func TestIsReadOnlySQL(t *testing.T) { + cases := []struct { + sql string + want bool + }{ + {"SELECT * FROM users", true}, + {"select id from sessions where token = ?", true}, + {" \n\tSELECT 1", true}, + {"-- leading comment\nSELECT 1", true}, + {"-- c1\n-- c2\nPRAGMA user_version", true}, + {"PRAGMA integrity_check", true}, + + {"INSERT INTO users (username) VALUES (?)", false}, + {"INSERT INTO messages (...) VALUES (...) RETURNING id", false}, + {"UPDATE users SET status = ?", false}, + {"DELETE FROM sessions WHERE expires_at < ?", false}, + {"VACUUM INTO 'x'", false}, + {"ANALYZE", false}, + {"BEGIN IMMEDIATE", false}, + // A CTE-led read is misrouted to the writer today — safe but + // serializing. If isReadOnlySQL ever learns WITH, flip this to true + // after auditing for writable CTEs (INSERT ... WITH). + {"WITH cte AS (SELECT 1) SELECT * FROM cte", false}, + // Prefix must be a whole keyword, not a prefix match. + {"SELECTIVE_TABLE_OP something", false}, + {"", false}, + {"-- only a comment", false}, + } + for _, tc := range cases { + if got := isReadOnlySQL(tc.sql); got != tc.want { + t.Errorf("isReadOnlySQL(%q) = %v, want %v", tc.sql, got, tc.want) + } + } +} diff --git a/Server/diskutil/diskutil.go b/Server/diskutil/diskutil.go new file mode 100644 index 00000000..b654840c --- /dev/null +++ b/Server/diskutil/diskutil.go @@ -0,0 +1,9 @@ +// Package diskutil reports free disk space for a path, with per-OS +// implementations behind build tags. Consumers treat errors (including +// ErrUnsupported on exotic platforms) as "unknown", never as "full". +package diskutil + +import "errors" + +// ErrUnsupported is returned on platforms without an implementation. +var ErrUnsupported = errors.New("diskutil: free-space query not supported on this platform") diff --git a/Server/diskutil/free_other.go b/Server/diskutil/free_other.go new file mode 100644 index 00000000..28081b17 --- /dev/null +++ b/Server/diskutil/free_other.go @@ -0,0 +1,8 @@ +//go:build !linux && !darwin && !windows + +package diskutil + +// FreeBytes is unsupported on this platform. +func FreeBytes(string) (uint64, error) { + return 0, ErrUnsupported +} diff --git a/Server/diskutil/free_unix.go b/Server/diskutil/free_unix.go new file mode 100644 index 00000000..147412da --- /dev/null +++ b/Server/diskutil/free_unix.go @@ -0,0 +1,16 @@ +//go:build linux || darwin + +package diskutil + +import "syscall" + +// FreeBytes returns the bytes available to unprivileged processes on the +// filesystem containing path. +func FreeBytes(path string) (uint64, error) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, err + } + // Bavail is what non-root can use; Bsize is the fundamental block size. + return st.Bavail * uint64(st.Bsize), nil //nolint:gosec // Bsize is a positive block size +} diff --git a/Server/diskutil/free_windows.go b/Server/diskutil/free_windows.go new file mode 100644 index 00000000..21b64039 --- /dev/null +++ b/Server/diskutil/free_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package diskutil + +import ( + "syscall" + "unsafe" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + procGetDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW") +) + +// FreeBytes returns the bytes available to the calling user on the volume +// containing path. +func FreeBytes(path string) (uint64, error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var freeToCaller, total, totalFree uint64 + // The unsafe.Pointer conversions below are the standard Win32 + // out-parameter calling pattern for a LazyProc: the pointees are local + // variables that outlive the syscall, nothing is aliased or reinterpreted. + r1, _, callErr := procGetDiskFreeSpaceEx.Call( + uintptr(unsafe.Pointer(p)), //nolint:gosec // G103: audited — syscall arg, pointee outlives the call + uintptr(unsafe.Pointer(&freeToCaller)), //nolint:gosec // G103: audited — out-parameter + uintptr(unsafe.Pointer(&total)), //nolint:gosec // G103: audited — out-parameter + uintptr(unsafe.Pointer(&totalFree)), //nolint:gosec // G103: audited — out-parameter + ) + if r1 == 0 { + return 0, callErr + } + return freeToCaller, nil +} diff --git a/Server/docker-compose.yml b/Server/docker-compose.yml index ac45fa1c..acb47cb2 100644 --- a/Server/docker-compose.yml +++ b/Server/docker-compose.yml @@ -10,6 +10,14 @@ # Browser clients reach LiveKit directly via the ports published below, # so your firewall must allow TCP 7880-7881 and UDP 50000-60000. +# Log rotation shared by both services: without a cap the default json-file +# driver grows unbounded on the host disk. +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + services: owncord: @@ -27,6 +35,21 @@ services: - ./config.yaml:/app/config.yaml:ro # Persistent data (SQLite DB + uploaded files) - owncord-data:/app/data + # The binary is its own probe (`healthcheck` subcommand) — the distroless + # image has no shell or curl. Plain docker compose only SURFACES unhealthy + # (docker ps, events); pair with an external watchdog (or an autoheal + # container) if you want automatic restarts on hang. + healthcheck: + test: ["CMD", "/chatserver", "healthcheck"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + logging: *default-logging + # Uncomment and size to your host to keep a runaway process from starving + # the machine (values are a starting point for a small community): + # mem_limit: 1g + # cpus: "2.0" depends_on: - livekit networks: @@ -43,6 +66,7 @@ services: - "50000-60000:50000-60000/udp" # WebRTC UDP media streams volumes: - ./livekit.yaml:/etc/livekit/livekit.yaml:ro + logging: *default-logging networks: - owncord-net diff --git a/Server/go.mod b/Server/go.mod index 77146b76..8676279f 100644 --- a/Server/go.mod +++ b/Server/go.mod @@ -37,6 +37,7 @@ require ( golang.org/x/mod v0.38.0 golang.org/x/sync v0.22.0 google.golang.org/protobuf v1.36.11 + gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.56.0 ) @@ -140,7 +141,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/grpc v1.83.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/Server/main.go b/Server/main.go index 90061f5c..17b898cf 100644 --- a/Server/main.go +++ b/Server/main.go @@ -3,7 +3,10 @@ package main import ( + "bytes" "context" + "crypto/tls" + "encoding/pem" "errors" "fmt" "io" @@ -13,16 +16,21 @@ import ( "net/http" "os" "os/signal" + "path/filepath" "runtime" + "strconv" "strings" "syscall" "time" + "gopkg.in/yaml.v3" + "github.com/owncord/server/admin" "github.com/owncord/server/api" "github.com/owncord/server/auth" "github.com/owncord/server/config" "github.com/owncord/server/db" + "github.com/owncord/server/diskutil" "github.com/owncord/server/logctx" "github.com/owncord/server/plugin" "github.com/owncord/server/storage" @@ -34,6 +42,12 @@ import ( var version = "dev" func main() { + // `server healthcheck` probes the running instance's /health and exits + // 0/1. It exists for container healthchecks: the distroless image has no + // shell or curl, so the binary is its own probe. + if len(os.Args) > 1 && os.Args[1] == "healthcheck" { + os.Exit(runHealthcheckCLI()) + } // `server token ...` is a direct-to-DB CLI (mint/list/revoke API tokens) — // handled before any server/logging setup so it stays quiet and standalone. if len(os.Args) > 1 && os.Args[1] == "token" { @@ -68,8 +82,13 @@ func main() { func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) error { // bgCtx is a cancellable context shared by all background goroutines // (event persister, event pruner, plugin loader, maintenance loop). - // It is cancelled early in the shutdown sequence so in-flight DB - // operations do not block after the database is being torn down. + // + // This first deferred bgCancel is only the LIFO backstop — because it is + // registered before `defer database.Close()`, it would otherwise run + // AFTER the database is closed, leaving background goroutines running + // through teardown. The persistence and maintenance blocks below register + // their own later (= earlier-running) defers that cancel bgCtx and JOIN + // their goroutines before the database closes. bgCtx, bgCancel := context.WithCancel(context.Background()) defer bgCancel() @@ -108,6 +127,15 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er return fmt.Errorf("creating data dir %s: %w", cfg.Server.DataDir, mkdirErr) } + // Disk-space awareness: the database (WAL growth included), uploads, + // certs, and by default backups all live on this volume, and running it + // dry breaks several of them at once. Probe errors are ignored — unknown + // is not "full". /health repeats this check continuously at 256 MiB. + warnLowDisk(log, "data dir", cfg.Server.DataDir) + if cfg.Backup.Dir != "" && cfg.Backup.Dir != filepath.Join(cfg.Server.DataDir, "backups") { + warnLowDisk(log, "backup dir", cfg.Backup.Dir) + } + // ── 3. TLS ──────────────────────────────────────────────────────────── tlsResult, err := auth.LoadOrGenerate(cfg.TLS) if err != nil { @@ -126,7 +154,7 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er return fmt.Errorf("database.type=%q is not supported; set \"sqlite\" or omit it", t) } - database, err := db.Open(cfg.Database.Path) + database, err := db.OpenWithMaxReaders(cfg.Database.Path, cfg.Database.MaxReaders) if err != nil { return fmt.Errorf("opening database: %w", err) } @@ -136,6 +164,9 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er // without this, it falls back to a hardcoded "data/chatserver.db" and // silently no-ops on any server with a configured database.path. admin.SetDatabasePath(cfg.Database.Path) + // Backup handlers and the scheduled-backup maintenance write to the + // configured backup directory (defaults to data/backups). + admin.SetBackupDir(cfg.Backup.Dir) if err := db.Migrate(database); err != nil { return fmt.Errorf("running migrations: %w", err) @@ -226,11 +257,21 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er retention := time.Duration(cfg.EventPersistence.RetentionHours) * time.Hour prunerInterval := time.Duration(cfg.EventPersistence.PrunerIntervalMinutes) * time.Minute - ws.StartEventPruner(bgCtx, database, retention, prunerInterval) + prunerDone := ws.StartEventPruner(bgCtx, database, retention, prunerInterval) defer func() { stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) defer stopCancel() persister.Stop(stopCtx) + // Cancel the shared background context and JOIN the pruner before + // the (LIFO-later) database.Close defer runs, so no prune is still + // mid-query against a closing pool. Bounded: a stuck prune delays + // shutdown by at most the timeout, then Close proceeds anyway. + bgCancel() + select { + case <-prunerDone: + case <-stopCtx.Done(): + log.Warn("event pruner did not exit before shutdown timeout") + } }() } @@ -289,8 +330,21 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er } stopMaintenance := make(chan struct{}) - defer close(stopMaintenance) // backstop for early returns below; see hub.GracefulStop defer above + maintenanceDone := make(chan struct{}) + defer func() { + // Backstop for early returns below (see hub.GracefulStop defer above), + // and a bounded join so an in-flight tick (which can hold the writer — + // scheduled backups run VACUUM INTO) isn't still using the database + // while the LIFO-later Close defer tears it down. + close(stopMaintenance) + select { + case <-maintenanceDone: + case <-time.After(5 * time.Second): + log.Warn("maintenance loop did not exit before shutdown timeout") + } + }() go func() { + defer close(maintenanceDone) ticker := time.NewTicker(15 * time.Minute) defer ticker.Stop() consecutiveFailures := 0 @@ -312,6 +366,13 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er tickFailed = true } + // Scheduled backups + retention pruning, driven by the + // backup_schedule / backup_retention admin settings. + if err := admin.MaintainBackups(bgCtx, database); err != nil { + log.Warn("backup maintenance failed", "error", err) + tickFailed = true + } + // Clean up orphaned attachments (uploaded but never linked to a message). // // Skipped entirely with no file storage configured: the delete is @@ -399,18 +460,158 @@ func run(log *slog.Logger, logBuf *admin.RingBuffer, levelVar *slog.LevelVar) er } } - // Stop the WebSocket hub: close all PeerConnections, voice rooms, and - // notify connected clients before draining HTTP connections. - hub.GracefulStop() + // Drain in-flight HTTP handlers FIRST: their broadcasts must still reach + // a live hub (and the event persister) or the frames vanish from the + // replay/event store across the restart. Shutdown does not wait on + // hijacked WebSocket connections, so the hub's own stop below is not + // delayed by connected clients — they get the restart notice right after + // the drain instead of right before it. + shutdownErr := srv.Shutdown(shutdownCtx) - if err := srv.Shutdown(shutdownCtx); err != nil { - return fmt.Errorf("graceful shutdown: %w", err) + // Stop the WebSocket hub: notify clients, stop LiveKit, close all client + // connections. Threaded with the same 30s budget the operator was told + // about — the notice sleep and LiveKit stop count against it rather than + // extending it. + hub.GracefulStopContext(shutdownCtx) + + if shutdownErr != nil { + return fmt.Errorf("graceful shutdown: %w", shutdownErr) } log.Info("server stopped cleanly") return nil } +// runHealthcheckCLI probes the local server's /health endpoint and returns a +// process exit code: 0 healthy, 1 degraded or unreachable. /health answers +// 503 with a subsystem reason when the hub, database, or disk is unhealthy, +// so a container orchestrator's healthcheck surfaces those too. +func runHealthcheckCLI() int { + // Deliberately NOT config.Load: that writes a default config.yaml when + // none exists, and a probe must have no side effects. Peek at the file + // (and the env overrides) for just the values that shape the URL and the + // certificate pin. + port := 8443 + scheme := "https" + certFile := "data/cert.pem" + tlsMode := "" + acmeDomain := "" + if raw, err := os.ReadFile(config.DefaultPath); err == nil { + var partial struct { + Server struct { + Port int `yaml:"port"` + } `yaml:"server"` + TLS struct { + Mode string `yaml:"mode"` + CertFile string `yaml:"cert_file"` + Domain string `yaml:"domain"` + } `yaml:"tls"` + } + if yaml.Unmarshal(raw, &partial) == nil { + if partial.Server.Port > 0 { + port = partial.Server.Port + } + tlsMode = partial.TLS.Mode + if partial.TLS.Mode == "off" { + scheme = "http" + } + if partial.TLS.CertFile != "" { + certFile = partial.TLS.CertFile + } + acmeDomain = partial.TLS.Domain + } + } + if env := os.Getenv("OWNCORD_SERVER_PORT"); env != "" { + if p, err := strconv.Atoi(env); err == nil && p > 0 { + port = p + } + } + if env := os.Getenv("OWNCORD_TLS_MODE"); env != "" { + tlsMode = env + if env == "off" { + scheme = "http" + } + } + if env := os.Getenv("OWNCORD_TLS_DOMAIN"); env != "" { + acmeDomain = env + } + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: healthcheckTLSConfig(tlsMode, certFile, acmeDomain), + }, + } + if port < 1 || port > 65535 { + port = 8443 + } + resp, err := client.Get(fmt.Sprintf("%s://127.0.0.1:%d/health", scheme, port)) //nolint:gosec // G704: host is hardcoded loopback; only the port comes from the operator's own config + if err != nil { + fmt.Fprintln(os.Stderr, "healthcheck: unreachable:", err) + return 1 + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + fmt.Fprintf(os.Stderr, "healthcheck: status %d: %s\n", resp.StatusCode, body) + return 1 + } + return 0 +} + +// healthcheckTLSConfig builds the probe's TLS config, per TLS mode: +// +// - acme: the served cert is CA-issued for the configured domain, so +// standard WebPKI verification works — but the probe dials 127.0.0.1, so +// ServerName must be overridden to the domain or hostname verification +// fails unconditionally and the probe reports a healthy server as down. +// A stale pre-ACME data/cert.pem must NOT be pinned in this mode either; +// the pin would mismatch the served ACME leaf forever. +// - self_signed / manual: the cert can never pass WebPKI (the generated one +// has no SANs and IsCA=false), so hostname/chain checks are replaced (not +// skipped) by pinning: the presented leaf must be byte-identical to the +// local cert file. +// - anything else with no readable local cert: plain WebPKI. +func healthcheckTLSConfig(tlsMode, certFile, acmeDomain string) *tls.Config { + if tlsMode == "acme" && acmeDomain != "" { + return &tls.Config{MinVersion: tls.VersionTLS12, ServerName: acmeDomain} + } + pinned := loadPinnedCert(certFile) + if pinned == nil { + return &tls.Config{MinVersion: tls.VersionTLS12} + } + return &tls.Config{ + MinVersion: tls.VersionTLS12, + // Chain/hostname verification is replaced by the exact-match pin + // below, which is strictly stronger for a cert we hold on disk. + // VerifyConnection (not VerifyPeerCertificate) so the pin also runs + // on resumed sessions (gosec G123). + InsecureSkipVerify: true, //nolint:gosec // G402: VerifyConnection below pins the exact local certificate + VerifyConnection: func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return errors.New("healthcheck: server presented no certificate") + } + if !bytes.Equal(cs.PeerCertificates[0].Raw, pinned) { + return errors.New("healthcheck: server certificate does not match " + certFile) + } + return nil + }, + } +} + +// loadPinnedCert reads the first PEM certificate block from path, returning +// its DER bytes, or nil when unavailable. +func loadPinnedCert(path string) []byte { + raw, err := os.ReadFile(path) //nolint:gosec // G304: path is the operator's own configured cert file + if err != nil { + return nil + } + block, _ := pem.Decode(raw) + if block == nil || block.Type != "CERTIFICATE" { + return nil + } + return block.Bytes +} + // seedHubReplayState restores the hub's monotonic seq counter from the // persisted MAX(events.seq) so wrapped-payload seqs stay monotonic across // restarts. Without this, the events table accumulates rows whose payload @@ -503,6 +704,29 @@ func wsURL(httpScheme, ip string, port int) string { return fmt.Sprintf("%s://%s:%d", ws, ip, port) } +// Free-space thresholds for the boot-time disk warning. /health uses its own +// (lower) continuous threshold; these only shape startup log noise. +const ( + diskWarnBytes = 1 << 30 // 1 GiB — warn + diskCriticalBytes = 256 << 20 // 256 MiB — error +) + +// warnLowDisk logs when the volume holding path is low on space. Probe +// failures (unsupported platform, missing dir) are silent — unknown ≠ full. +func warnLowDisk(log *slog.Logger, label, path string) { + free, err := diskutil.FreeBytes(path) + if err != nil { + return + } + switch { + case free < diskCriticalBytes: + log.Error("disk space critically low — writes will start failing soon", + "volume", label, "path", path, "free_mb", free>>20) + case free < diskWarnBytes: + log.Warn("disk space low", "volume", label, "path", path, "free_mb", free>>20) + } +} + // getOutboundIP returns the preferred outbound IP of this machine by dialing // a known external address (no actual connection is made with UDP). func getOutboundIP() string { diff --git a/Server/main_test.go b/Server/main_test.go index 80254db8..c7260860 100644 --- a/Server/main_test.go +++ b/Server/main_test.go @@ -121,7 +121,7 @@ func TestSeedHubReplayState_ForcesFullResyncForOfflineClient(t *testing.T) { seedHubReplayState(ctx, hub, database, log) hub.SetEventStore(database) - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() diff --git a/Server/migrations/031_sessions_expiry_index.sql b/Server/migrations/031_sessions_expiry_index.sql new file mode 100644 index 00000000..2781b581 --- /dev/null +++ b/Server/migrations/031_sessions_expiry_index.sql @@ -0,0 +1,13 @@ +-- Make the periodic session-expiry sweep sargable. The sweep used to run +-- strftime over every row on the single writer connection every 15 minutes. +-- The server writes expires_at as RFC3339 UTC ("2006-01-02T15:04:05Z") -- +-- normalize any legacy space-separated rows to that layout so plain text +-- comparison is correct, then index the column. + +UPDATE sessions SET expires_at = replace(expires_at, ' ', 'T') + WHERE instr(expires_at, ' ') > 0; + +UPDATE sessions SET expires_at = expires_at || 'Z' + WHERE length(expires_at) = 19; + +CREATE INDEX IF NOT EXISTS idx_sessions_expires_at ON sessions(expires_at); diff --git a/Server/scripts/k6/ws-load.js b/Server/scripts/k6/ws-load.js index 9123e25f..e9049a69 100644 --- a/Server/scripts/k6/ws-load.js +++ b/Server/scripts/k6/ws-load.js @@ -1,12 +1,25 @@ // k6 WebSocket load test for OwnCord server // Run: k6 run --vus 50 --duration 60s scripts/k6/ws-load.js // +// The wire protocol is the envelope format from docs/protocol.md: every +// client->server frame is {type, id?, payload:{...}} and the first frame MUST +// be an `auth` envelope. If you change docs/protocol-schema.json, grep this +// script — it is not generated and CI does not run it, so it rots silently +// (it once drifted to pre-envelope framing and reported green while every +// auth failed). +// +// Prerequisites: the target server must already have the loadtest users +// (K6_USERNAME, all sharing K6_PASSWORD) registered, and the +// target channel readable by them. +// // Environment variables: -// K6_WS_URL - WebSocket URL (default: ws://localhost:8443/ws) -// K6_HTTP_URL - HTTP base URL (default: http://localhost:8443) +// K6_WS_URL - WebSocket URL (default: wss://localhost:8443/api/v1/ws) +// K6_HTTP_URL - HTTP base URL (default: https://localhost:8443) // K6_USERNAME - Test user prefix (default: loadtest) // K6_PASSWORD - Test user password (default: LoadTest123!) // K6_CHANNEL_ID - Channel ID to send messages in (default: 1) +// +// Self-signed TLS (the default server cert): run k6 with --insecure-skip-tls-verify. import ws from "k6/ws"; import http from "k6/http"; @@ -15,15 +28,19 @@ import { Counter, Rate, Trend } from "k6/metrics"; // Custom metrics const wsConnections = new Counter("ws_connections"); +const wsAuthed = new Counter("ws_authed"); +const wsReady = new Counter("ws_ready"); const wsMessages = new Counter("ws_messages_sent"); +const wsAcks = new Counter("ws_send_acks"); const wsErrors = new Counter("ws_errors"); const wsConnectTime = new Trend("ws_connect_time", true); const wsMessageRate = new Rate("ws_message_success"); const authTime = new Trend("auth_time", true); +const broadcastLatency = new Trend("ws_broadcast_latency_ms", true); // Configuration -const WS_URL = __ENV.K6_WS_URL || "ws://localhost:8443/ws"; -const HTTP_URL = __ENV.K6_HTTP_URL || "http://localhost:8443"; +const WS_URL = __ENV.K6_WS_URL || "wss://localhost:8443/api/v1/ws"; +const HTTP_URL = __ENV.K6_HTTP_URL || "https://localhost:8443"; const USERNAME_PREFIX = __ENV.K6_USERNAME || "loadtest"; const PASSWORD = __ENV.K6_PASSWORD || "LoadTest123!"; const CHANNEL_ID = parseInt(__ENV.K6_CHANNEL_ID || "1"); @@ -35,23 +52,33 @@ export const options = { executor: "ramping-vus", startVUs: 0, stages: [ - { duration: "10s", target: 10 }, // warm up - { duration: "30s", target: 50 }, // ramp to 50 - { duration: "60s", target: 50 }, // sustain - { duration: "10s", target: 100 }, // spike - { duration: "30s", target: 100 }, // sustain spike - { duration: "10s", target: 0 }, // ramp down + { duration: "10s", target: 10 }, // warm up + { duration: "30s", target: 50 }, // ramp to 50 + { duration: "60s", target: 50 }, // sustain + { duration: "10s", target: 100 }, // spike + { duration: "30s", target: 100 }, // sustain spike + { duration: "10s", target: 0 }, // ramp down ], }, }, thresholds: { - ws_connect_time: ["p(95)<2000"], // 95% connect under 2s - ws_message_success: ["rate>0.95"], // 95% message success - ws_errors: ["count<50"], // fewer than 50 errors - auth_time: ["p(95)<1000"], // 95% auth under 1s + ws_connect_time: ["p(95)<2000"], // 95% connect under 2s + ws_message_success: ["rate>0.95"], // 95% of sends acked + ws_errors: ["count<50"], // fewer than 50 errors + auth_time: ["p(95)<1000"], // 95% auth under 1s + // A run where nobody authenticated or went ready is a broken run, no + // matter how green everything else looks — this is the assertion that + // was missing when the script drifted off the wire protocol. + ws_authed: ["count>0"], + ws_ready: ["count>0"], }, }; +// envelope wraps a client->server frame in the protocol's outer shape. +function envelope(type, payload) { + return JSON.stringify({ type: type, payload: payload }); +} + // Login and get session token function authenticate(username) { const start = Date.now(); @@ -88,27 +115,48 @@ export default function () { wsConnectTime.add(Date.now() - connectStart); wsConnections.add(1); - // Send auth on connect - socket.send( - JSON.stringify({ - type: "auth", - token: token, - }), - ); + let authed = false; + let ready = false; + let msgCount = 0; + const maxMessages = 10; + const pendingSends = {}; // send-id -> Date.now() at send + + // First frame must be the auth envelope (serve_auth.go). + socket.send(envelope("auth", { token: token })); - // Handle incoming messages socket.on("message", function (msg) { try { const data = JSON.parse(msg); - - // After auth_ok, focus a channel and start sending - if (data.type === "ready") { - socket.send( - JSON.stringify({ - type: "channel_focus", - channel_id: CHANNEL_ID, - }), - ); + switch (data.type) { + case "auth_ok": + authed = true; + wsAuthed.add(1); + break; + case "auth_error": + wsErrors.add(1); + socket.close(); + break; + case "ready": + ready = true; + wsReady.add(1); + socket.send(envelope("channel_focus", { channel_id: CHANNEL_ID })); + break; + case "chat_send_ok": + wsAcks.add(1); + wsMessageRate.add(true); + if (data.id && pendingSends[data.id]) { + broadcastLatency.add(Date.now() - pendingSends[data.id]); + delete pendingSends[data.id]; + } + break; + case "error": + wsErrors.add(1); + wsMessageRate.add(false); + break; + default: + // Broadcast traffic (chat_message, presence, typing, seq'd + // frames) — receiving it is the point of the load, no assertion. + break; } } catch (_e) { wsErrors.add(1); @@ -119,49 +167,46 @@ export default function () { wsErrors.add(1); }); - // Send messages periodically (respecting rate limits) - let msgCount = 0; - const maxMessages = 10; - + // Send messages periodically (respecting rate limits). Gated on ready: + // sends before the session is established only measure error handling. socket.setInterval(function () { + if (!ready) { + return; + } if (msgCount >= maxMessages) { socket.close(); return; } - - const msg = JSON.stringify({ - type: "chat_send", - channel_id: CHANNEL_ID, - content: `Load test message ${vuId}-${msgCount} at ${Date.now()}`, - }); - - socket.send(msg); + const id = `${vuId}-${msgCount}-${Date.now()}`; + pendingSends[id] = Date.now(); + socket.send( + JSON.stringify({ + type: "chat_send", + id: id, + payload: { + channel_id: CHANNEL_ID, + content: `Load test message ${vuId}-${msgCount}`, + }, + }), + ); wsMessages.add(1); - wsMessageRate.add(true); msgCount++; }, 2000); // 1 message every 2 seconds (well under rate limit) - // Send typing indicators + // Typing indicators (client->server type is typing_start, not "typing"). socket.setInterval(function () { - if (msgCount < maxMessages) { - socket.send( - JSON.stringify({ - type: "typing", - channel_id: CHANNEL_ID, - }), - ); + if (ready && msgCount < maxMessages) { + socket.send(envelope("typing_start", { channel_id: CHANNEL_ID })); } - }, 4000); // 1 typing every 4 seconds (under 1/3s limit) + }, 4000); - // Send presence updates + // Presence updates (client->server type is presence_update; bare + // "presence" is the server->client broadcast). socket.setInterval(function () { - socket.send( - JSON.stringify({ - type: "presence", - status: "online", - }), - ); - }, 15000); // 1 presence every 15 seconds (under 1/10s limit) + if (authed) { + socket.send(envelope("presence_update", { status: "online" })); + } + }, 15000); // Keep connection alive for the test duration socket.setTimeout(function () { diff --git a/Server/service/channel.go b/Server/service/channel.go index 89fdff2b..ebd2ec50 100644 --- a/Server/service/channel.go +++ b/Server/service/channel.go @@ -266,8 +266,22 @@ func (s *ChannelService) HandleChannelFocus(ctx context.Context, userID, channel // Mark channel as read. latestID == 0 (no undeleted messages) still // writes: the upsert is what zeroes mention_count, and a last_read of 0 is // correct then — any future message id is larger, so unread counts hold. + // + // Skip the UPSERT when the stored row already says exactly this (same + // last_message_id, no mentions to clear): channel_focus/mark_read fire on + // every refocus at up to 10/s/user, and even a no-op write occupies the + // single writer connection and opens a transaction. The extra read runs on + // the reader pool, which doesn't serialize. Same problem-shape as the + // session-touch throttle (api/middleware.go). A read failure falls through + // to the write — the write is the load-bearing half. latestID, err := s.st.GetLatestMessageID(ctx, channelID) if err == nil { + lastRead, mentions, found, rsErr := s.st.GetReadState(ctx, userID, channelID) + if rsErr == nil && found && lastRead == latestID && mentions == 0 { + slog.Debug("channel_focus: read state already current, skipping write", + "user_id", userID, "channel_id", channelID) + return ch, nil + } _ = s.st.UpdateReadState(ctx, userID, channelID, latestID) } diff --git a/Server/service/channel_focus_writeskip_test.go b/Server/service/channel_focus_writeskip_test.go new file mode 100644 index 00000000..14982fa3 --- /dev/null +++ b/Server/service/channel_focus_writeskip_test.go @@ -0,0 +1,83 @@ +package service + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/owncord/server/db" + "github.com/owncord/server/permissions" +) + +// countingReadStateStore wraps a real *db.DB and counts UpdateReadState calls +// so the test can observe whether channel focus hit the writer. +type countingReadStateStore struct { + *db.DB + writes atomic.Int64 +} + +func (s *countingReadStateStore) UpdateReadState(ctx context.Context, userID, channelID, lastReadMessageID int64) error { + s.writes.Add(1) + return s.DB.UpdateReadState(ctx, userID, channelID, lastReadMessageID) +} + +// TestHandleChannelFocus_SkipsNoOpReadStateWrite locks the write-skip: +// refocusing a channel whose read state is already current must not occupy +// the single writer connection, while a new message (or outstanding mention) +// must write again. +func TestHandleChannelFocus_SkipsNoOpReadStateWrite(t *testing.T) { + database := newTestDB(t) + seedRole(t, database, &db.Role{ + ID: permissions.MemberRoleID, + Name: "member", + Permissions: permissions.SendMessages | permissions.ReadMessages, + Position: 1, + }) + seedUserRole(t, database, 1, permissions.MemberRoleID) + seedChannel(t, database, &db.Channel{ID: 10, Name: "general", Type: "text"}) + + st := &countingReadStateStore{DB: database} + svc := NewChannelService(st, NewPermissionService(database, permissions.NewChecker(database))) + ctx := context.Background() + + // First focus writes (no read-state row exists yet). + if _, err := svc.HandleChannelFocus(ctx, 1, 10); err != nil { + t.Fatalf("focus #1: %v", err) + } + if got := st.writes.Load(); got != 1 { + t.Fatalf("writes after first focus = %d, want 1", got) + } + + // Refocus with nothing new: the row already says latest+0 mentions. + if _, err := svc.HandleChannelFocus(ctx, 1, 10); err != nil { + t.Fatalf("focus #2: %v", err) + } + if got := st.writes.Load(); got != 1 { + t.Fatalf("writes after no-op refocus = %d, want 1 (skipped)", got) + } + + // A new message moves the latest id — focus must write again. + if _, err := database.ExecContext(ctx, + `INSERT INTO messages (channel_id, user_id, content) VALUES (10, 1, 'hi')`); err != nil { + t.Fatalf("seed message: %v", err) + } + if _, err := svc.HandleChannelFocus(ctx, 1, 10); err != nil { + t.Fatalf("focus #3: %v", err) + } + if got := st.writes.Load(); got != 2 { + t.Fatalf("writes after new message = %d, want 2", got) + } + + // An outstanding mention must also force the write (it zeroes the badge) + // even when last_message_id is unchanged. + if _, err := database.ExecContext(ctx, + `UPDATE read_states SET mention_count = 3 WHERE user_id = 1 AND channel_id = 10`); err != nil { + t.Fatal(err) + } + if _, err := svc.HandleChannelFocus(ctx, 1, 10); err != nil { + t.Fatalf("focus #4: %v", err) + } + if got := st.writes.Load(); got != 3 { + t.Fatalf("writes after mention = %d, want 3 (mention must clear)", got) + } +} diff --git a/Server/service/datastore.go b/Server/service/datastore.go index fcc4fd0c..572a4fe4 100644 --- a/Server/service/datastore.go +++ b/Server/service/datastore.go @@ -37,6 +37,7 @@ type Store interface { GetReactions(ctx context.Context, messageID int64) ([]db.ReactionCount, error) GetReactionUsers(ctx context.Context, messageID int64, emoji string, limit int) ([]db.ReactionUser, error) UpdateReadState(ctx context.Context, userID, channelID, lastReadMessageID int64) error + GetReadState(ctx context.Context, userID, channelID int64) (lastMessageID, mentionCount int64, found bool, err error) GetChannelUnreadCounts(ctx context.Context, userID int64) (map[int64]db.ChannelUnread, error) // ── Mentions ── diff --git a/Server/service/permission.go b/Server/service/permission.go index f4f26373..ad3f71ec 100644 --- a/Server/service/permission.go +++ b/Server/service/permission.go @@ -4,6 +4,7 @@ import ( "context" "log/slog" "sync" + "sync/atomic" "time" "github.com/owncord/server/db" @@ -36,6 +37,11 @@ type PermissionService struct { // its DB read and refuses to cache if it changed, so an invalidation that // races a populate can't be lost (F6). gen uint64 + + // hits/misses are atomics, not mu-guarded ints: the hit path holds mu only + // as an RLock, so a plain increment there would race. + hits atomic.Uint64 + misses atomic.Uint64 } // NewPermissionService creates a PermissionService backed by the given DB. @@ -130,6 +136,15 @@ func (s *PermissionService) Checker() *permissions.Checker { return s.checker } +// CacheStats returns the lifetime hit/miss counters of the permission cache. +// A miss is any lookup that had to repopulate from the store — including +// TTL-expired entries and post-invalidation lookups — so a burst of misses +// right after a role or override change is the cache-wide invalidation cost +// showing up, not a bug. Safe to call from any goroutine. +func (s *PermissionService) CacheStats() (hits, misses uint64) { + return s.hits.Load(), s.misses.Load() +} + // getOrPopulate returns cached perms for the user, populating the cache // on miss or staleness. Returns nil if the user's role can't be loaded. func (s *PermissionService) getOrPopulate(ctx context.Context, userID int64) *cachedPerms { @@ -137,10 +152,12 @@ func (s *PermissionService) getOrPopulate(ctx context.Context, userID int64) *ca cp, ok := s.cache[userID] if ok && time.Since(cp.populatedAt) < permCacheTTL { s.mu.RUnlock() + s.hits.Add(1) return cp } startGen := s.gen s.mu.RUnlock() + s.misses.Add(1) // Populate. role, err := s.st.GetRoleForUser(ctx, userID) diff --git a/Server/service/permission_stats_test.go b/Server/service/permission_stats_test.go new file mode 100644 index 00000000..c05ba28f --- /dev/null +++ b/Server/service/permission_stats_test.go @@ -0,0 +1,36 @@ +package service + +import ( + "context" + "testing" + + "github.com/owncord/server/db" + "github.com/owncord/server/permissions" +) + +// TestCacheStats_HitsAndMisses locks the semantics the metrics endpoint +// documents: a miss is any lookup that repopulated (first touch, TTL expiry, +// post-invalidation), a hit is a fresh cached entry. +func TestCacheStats_HitsAndMisses(t *testing.T) { + svc, database := newTestPermService(t) + seedChannel(t, database, &db.Channel{ID: 10, Name: "general", Type: "text"}) + + if h, m := svc.CacheStats(); h != 0 || m != 0 { + t.Fatalf("fresh service CacheStats = (%d, %d), want (0, 0)", h, m) + } + + ctx := context.Background() + svc.HasChannelPerm(ctx, 1, 10, permissions.SendMessages) // populate → miss + svc.HasChannelPerm(ctx, 1, 10, permissions.SendMessages) // cached → hit + + if h, m := svc.CacheStats(); h != 1 || m != 1 { + t.Fatalf("CacheStats after populate+hit = (%d, %d), want (1, 1)", h, m) + } + + svc.InvalidateAll() + svc.HasChannelPerm(ctx, 1, 10, permissions.SendMessages) // repopulate → miss + + if h, m := svc.CacheStats(); h != 1 || m != 2 { + t.Fatalf("CacheStats after invalidation = (%d, %d), want (1, 2)", h, m) + } +} diff --git a/Server/storage/errio_test.go b/Server/storage/errio_test.go new file mode 100644 index 00000000..f65af16a --- /dev/null +++ b/Server/storage/errio_test.go @@ -0,0 +1,45 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestSave_FilesystemFailureIsErrIO locks the error classification handlers +// depend on for status codes: a server-side filesystem failure carries ErrIO, +// while content rejections (blocked type, size) do not. +func TestSave_FilesystemFailureIsErrIO(t *testing.T) { + dir := filepath.Join(t.TempDir(), "store") + s, err := New(dir, 1) + if err != nil { + t.Fatal(err) + } + + // Remove the storage dir so os.Create fails — the same class of failure + // as a read-only mount or a full disk. + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + _, saveErr := s.Save("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", strings.NewReader("hello")) + if saveErr == nil { + t.Fatal("Save into a missing dir should fail") + } + if !errors.Is(saveErr, ErrIO) { + t.Fatalf("filesystem failure not marked ErrIO: %v", saveErr) + } + + // Content rejections stay non-ErrIO (the client's fault → 400). + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + _, blockedErr := s.Save("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeef", strings.NewReader("MZ executable bytes")) + if blockedErr == nil { + t.Fatal("blocked file type should fail") + } + if errors.Is(blockedErr, ErrIO) { + t.Fatalf("content rejection wrongly marked ErrIO: %v", blockedErr) + } +} diff --git a/Server/storage/storage.go b/Server/storage/storage.go index a4bce227..d6ce0c71 100644 --- a/Server/storage/storage.go +++ b/Server/storage/storage.go @@ -3,14 +3,23 @@ package storage import ( "bytes" + "errors" "fmt" "io" + "io/fs" "log/slog" "os" "path/filepath" "strings" ) +// ErrIO marks a Save failure caused by the server's own filesystem — disk +// full, permissions, a read-only mount — as opposed to the uploaded content +// being invalid. Handlers use it to return a 5xx instead of blaming the +// client with a 400, so infrastructure failures are distinguishable in any +// status-class dashboard. +var ErrIO = errors.New("storage io failure") + // blockedMagic maps format names to their magic byte signatures. Files whose // leading bytes match any entry are rejected by ValidateFileType. var blockedMagic = []struct { @@ -126,7 +135,7 @@ func (s *Storage) Save(uuid string, r io.Reader) (int64, error) { f, err := os.Create(dst) if err != nil { - return 0, fmt.Errorf("creating file %s: %w", dst, err) + return 0, fmt.Errorf("creating file %s: %w: %w", dst, ErrIO, err) } // Any failure after this point must remove the partial file: the orphan // sweep is DB-row-driven, so a file without a DB row is never reclaimed. @@ -147,6 +156,13 @@ func (s *Storage) Save(uuid string, r io.Reader) (int64, error) { limited := io.LimitReader(full, maxBytes) written, err := io.Copy(f, limited) if err != nil { + // Write-side failures surface as *fs.PathError (File.Write wraps + // them); read-side failures (client aborted mid-upload) do not, and + // those stay the client's fault. + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + return 0, fmt.Errorf("writing file: %w: %w", ErrIO, err) + } return 0, fmt.Errorf("writing file: %w", err) } // Probe for one more byte to detect if the file exceeds the limit. @@ -157,7 +173,7 @@ func (s *Storage) Save(uuid string, r io.Reader) (int64, error) { } } if syncErr := f.Sync(); syncErr != nil { - return 0, fmt.Errorf("syncing file %s: %w", dst, syncErr) + return 0, fmt.Errorf("syncing file %s: %w: %w", dst, ErrIO, syncErr) } success = true return written, nil @@ -175,8 +191,21 @@ func (s *Storage) Delete(uuid string) error { return os.Remove(dst) } +// File is what serving a stored blob requires of an opened file. Seeking is +// load-bearing, not incidental: both serve paths hand the file to +// http.ServeContent, which needs io.ReadSeeker for range requests — the +// exact capability that makes a remote backend (e.g. S3) the hard part of +// any future storage swap. Stat provides size and modtime the same way. +// *os.File satisfies it. +type File interface { + io.Reader + io.Seeker + io.Closer + Stat() (os.FileInfo, error) +} + // Open opens the file named uuid for reading. -func (s *Storage) Open(uuid string) (*os.File, error) { +func (s *Storage) Open(uuid string) (File, error) { if err := sanitizeFilename(uuid); err != nil { return nil, err } diff --git a/Server/telemetry/metrics.go b/Server/telemetry/metrics.go index fdff7b74..14fa15e1 100644 --- a/Server/telemetry/metrics.go +++ b/Server/telemetry/metrics.go @@ -16,7 +16,6 @@ import ( const ( scopeWS = "github.com/owncord/server/ws" scopeService = "github.com/owncord/server/service" - scopeDB = "github.com/owncord/server/db" scopeVoice = "github.com/owncord/server/voice" ) @@ -31,7 +30,6 @@ type AppMetrics struct { WSEventsPersisted Counter WSEventsDropped Counter WSEventsPersistErrors Counter - DBQueryDurationSec Histogram VoiceActiveSessions Gauge VoiceParticipants Gauge ServiceCallDurationSec Histogram @@ -63,7 +61,6 @@ func NewAppMetrics() *AppMetrics { } ws := GlobalMeter(scopeWS) svc := GlobalMeter(scopeService) - db := GlobalMeter(scopeDB) voice := GlobalMeter(scopeVoice) m := &AppMetrics{ WSMessagesTotal: ws.Counter("ws_messages_total", "WebSocket messages broadcast"), @@ -73,7 +70,6 @@ func NewAppMetrics() *AppMetrics { WSEventsPersisted: ws.Counter("ws_events_persisted_total", "Events written to the cold-tier event log"), WSEventsDropped: ws.Counter("ws_events_dropped_total", "Events dropped because the persister queue was full"), WSEventsPersistErrors: ws.Counter("ws_events_persist_errors_total", "PersistEvent calls that returned an error from the underlying store"), - DBQueryDurationSec: db.Histogram("db_query_duration_seconds", "Per-query wall time", "s"), VoiceActiveSessions: voice.Gauge("voice_active_sessions", "Active LiveKit rooms"), VoiceParticipants: voice.Gauge("voice_participants", "Connected LiveKit participants across all rooms"), ServiceCallDurationSec: svc.Histogram("service_call_duration_seconds", "Service-layer method execution time", "s"), diff --git a/Server/token_cli.go b/Server/token_cli.go index f3f5cb5d..cf77a14f 100644 --- a/Server/token_cli.go +++ b/Server/token_cli.go @@ -29,7 +29,11 @@ func runTokenCLI(args []string) int { fmt.Fprintf(os.Stderr, "error: load config: %v\n", err) return 1 } - database, err := db.Open(cfg.Database.Path) + // OpenShared: the CLI must work while the server is running (the docs' + // cron-backup recipe mints tokens against a live server). SQLite WAL makes + // the concurrent access safe; the single-process lock protects only the + // server's process-local state, which this CLI never touches. + database, err := db.OpenShared(cfg.Database.Path) if err != nil { fmt.Fprintf(os.Stderr, "error: open database: %v\n", err) return 1 diff --git a/Server/ws/client.go b/Server/ws/client.go index 01b5d42f..9bb0f3a0 100644 --- a/Server/ws/client.go +++ b/Server/ws/client.go @@ -205,6 +205,9 @@ func (c *Client) sendMsg(msg []byte) { c.msgsSent++ default: c.msgsDropped++ + if c.hub != nil { // hub-less clients exist only in unit tests + c.hub.bpQueueDisconnects.Add(1) + } slog.Warn("ws: client send buffer full, closing connection to force reconnect", "user_id", c.userID) c.closeAllSendLocked() @@ -226,11 +229,17 @@ func (c *Client) sendHighMsg(msg []byte) { c.msgsSent++ default: // Fall back to normal priority channel. + if c.hub != nil { + c.hub.bpHighFallbacks.Add(1) + } select { case c.send <- msg: c.msgsSent++ default: c.msgsDropped++ + if c.hub != nil { + c.hub.bpQueueDisconnects.Add(1) + } slog.Warn("ws: client high+normal buffers full, closing connection", "user_id", c.userID) c.closeAllSendLocked() @@ -252,7 +261,12 @@ func (c *Client) sendLowMsg(msg []byte) { c.msgsSent++ default: c.msgsDropped++ - // Do NOT disconnect — low-priority messages are safely droppable. + // Do NOT disconnect — low-priority messages are safely droppable. No + // per-drop log either (typing/presence bursts would flood it); the + // aggregate counter is the only place these drops are visible. + if c.hub != nil { + c.hub.bpLowDrops.Add(1) + } } } @@ -271,6 +285,9 @@ func (c *Client) trySendMsg(msg []byte) bool { return true default: c.msgsDropped++ + if c.hub != nil { + c.hub.bpQueueDisconnects.Add(1) + } slog.Warn("ws: client send buffer full (trySend), closing connection to force reconnect", "user_id", c.userID) c.closeAllSendLocked() diff --git a/Server/ws/conn_cap_test.go b/Server/ws/conn_cap_test.go new file mode 100644 index 00000000..af6455bb --- /dev/null +++ b/Server/ws/conn_cap_test.go @@ -0,0 +1,47 @@ +package ws + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// TestServeWS_ConnectionCap locks the capacity guardrail: at or above the +// configured cap, upgrade requests are refused with 503 before the WebSocket +// handshake, and the rejection is counted. +func TestServeWS_ConnectionCap(t *testing.T) { + h := &Hub{clients: map[int64]*Client{1: {}, 2: {}}} + handler := ServeWS(h, nil, nil, 2) + + rec := httptest.NewRecorder() + handler(rec, httptest.NewRequest(http.MethodGet, "/api/v1/ws", nil)) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status at cap = %d, want 503", rec.Code) + } + if rec.Header().Get("Retry-After") == "" { + t.Error("503 response missing Retry-After header") + } + if got := h.ConnRejectCount(); got != 1 { + t.Errorf("ConnRejectCount = %d, want 1", got) + } + + // Below the cap the request proceeds to the upgrade (which fails without + // WebSocket headers — but NOT with the capacity 503). + h2 := &Hub{clients: map[int64]*Client{1: {}}} + rec2 := httptest.NewRecorder() + ServeWS(h2, nil, nil, 2)(rec2, httptest.NewRequest(http.MethodGet, "/api/v1/ws", nil)) + if rec2.Code == http.StatusServiceUnavailable { + t.Fatalf("below-cap request was refused with 503") + } + if got := h2.ConnRejectCount(); got != 0 { + t.Errorf("below-cap ConnRejectCount = %d, want 0", got) + } + + // Cap 0 = unlimited: never the capacity 503. + rec3 := httptest.NewRecorder() + ServeWS(h, nil, nil, 0)(rec3, httptest.NewRequest(http.MethodGet, "/api/v1/ws", nil)) + if rec3.Code == http.StatusServiceUnavailable { + t.Fatalf("cap=0 request was refused with 503") + } +} diff --git a/Server/ws/emit.go b/Server/ws/emit.go index f77cae3a..b04dcf9b 100644 --- a/Server/ws/emit.go +++ b/Server/ws/emit.go @@ -29,6 +29,13 @@ func (h *Hub) EmitEvents(ctx context.Context, events []Event) { case VoiceChannelEvent: h.sendToVoiceChannelExcept(e.VoiceChannelID(), e.ExcludeUserID(), e.Payload()) case ExcludeSenderEvent: + // An invisible user's public presence half rides this branch; + // like the visible case below, it must invalidate any queued + // coalescer entry so a stale connect-time presence can't flush + // after (and overwrite) this fresher user-chosen status. + if po, isPresence := ev.(PresenceOthersEvent); isPresence { + h.dropQueuedPresence(po.excludeUserID) + } // Low priority: typing indicators are ephemeral. h.broadcastExcludeLow(e.ChannelID(), e.ExcludeUserID(), e.Payload()) case UserTargetedEvent: @@ -65,7 +72,14 @@ func (h *Hub) EmitEvents(ctx context.Context, events []Event) { // still sitting in the low queue — leaving the observer's final // view of that user's status stale. Routing everything through // BroadcastToAll keeps every source of one user's presence in a - // single ordered, seq-stamped, replayable stream. + // single ordered, seq-stamped, replayable stream (OC-0214). + if pe, isPresence := ev.(PresenceEvent); isPresence { + // A user-chosen presence also bypasses the connect/disconnect + // coalescer; drop any entry still queued for this user or the + // pending flush (up to 300ms later) would overwrite this + // fresher status with the stale connect-time one. + h.dropQueuedPresence(pe.userID) + } h.BroadcastToAll(e.Payload()) default: slog.Warn("EmitEvents: unknown event type", "type", fmt.Sprintf("%T", ev)) diff --git a/Server/ws/event.go b/Server/ws/event.go index bb9a57fe..604d62ce 100644 --- a/Server/ws/event.go +++ b/Server/ws/event.go @@ -235,7 +235,10 @@ func (e TypingDMEvent) TargetUserID() int64 { return e.targetUserID } func (e TypingDMEvent) Payload() []byte { return e.payload } // PresenceEvent is a presence update broadcast to all connected clients. +// userID identifies whose presence this is, so EmitEvents can invalidate any +// stale entry the connect/disconnect coalescer still holds for that user. type PresenceEvent struct { + userID int64 payload []byte } @@ -279,7 +282,7 @@ func (e PresenceSelfEvent) Payload() []byte { return e.payload } func presenceEvents(userID int64, status string, customStatus *string) []Event { public := db.BroadcastStatus(status) if public == status { - return []Event{PresenceEvent{payload: buildPresenceMsg(userID, status, customStatus)}} + return []Event{PresenceEvent{userID: userID, payload: buildPresenceMsg(userID, status, customStatus)}} } return []Event{ PresenceOthersEvent{ diff --git a/Server/ws/event_persister.go b/Server/ws/event_persister.go index 3e313fa6..66df03d4 100644 --- a/Server/ws/event_persister.go +++ b/Server/ws/event_persister.go @@ -115,6 +115,9 @@ func (p *EventPersister) Enqueue(seq int64, eventType string, channelID int64, p case p.queue <- pendingEvent{seq: seq, eventType: eventType, channelID: channelID, payload: payload}: default: p.dropped.Add(1) + // The WSEventsDropped OTel counter is synced from this atomic by + // run()'s ticker (which has a real context) — an instrumentation call + // here would sit under the caller's seqMu and trip contextcheck. } } @@ -163,6 +166,16 @@ func (p *EventPersister) run(ctx context.Context) { // Cache the AppMetrics bundle once instead of looking it up per event. metrics := telemetry.NewAppMetrics() + // Enqueue only bumps the p.dropped atomic (it runs under the hub's seqMu + // with no context); this loop owns syncing the OTel counter from it. + var droppedReported uint64 + syncDropped := func() { + if d := p.dropped.Load(); d > droppedReported { + metrics.WSEventsDropped.Add(ctx, int64(d-droppedReported)) //nolint:gosec // monotonic counter delta + droppedReported = d + } + } + batch := make([]pendingEvent, 0, p.batchSize) // Scratch slice reused across flushes for the store's batch shape. rows := make([]db.PersistedEvent, 0, p.batchSize) @@ -232,6 +245,7 @@ func (p *EventPersister) run(ctx context.Context) { } case <-tick.C: flush() + syncDropped() } } } diff --git a/Server/ws/event_pruner.go b/Server/ws/event_pruner.go index 1b419076..bca57d36 100644 --- a/Server/ws/event_pruner.go +++ b/Server/ws/event_pruner.go @@ -20,10 +20,15 @@ import ( const maxStartupDelay = time.Minute // StartEventPruner launches a goroutine that wakes every interval and deletes -// events older than retention. The goroutine exits when ctx is cancelled. -func StartEventPruner(ctx context.Context, s EventStore, retention, interval time.Duration) { +// events older than retention. The goroutine exits when ctx is cancelled; the +// returned channel closes when it has fully exited (i.e. no prune can still +// be touching the store) — the same join contract EventPersister.Stop gives, +// so shutdown can order "background work done" before "database closed". +func StartEventPruner(ctx context.Context, s EventStore, retention, interval time.Duration) <-chan struct{} { + done := make(chan struct{}) if s == nil { - return + close(done) + return done } if retention <= 0 { retention = 24 * time.Hour @@ -35,6 +40,7 @@ func StartEventPruner(ctx context.Context, s EventStore, retention, interval tim // (e.g. 100ms in event_pruner_test.go) don't wait a full minute. startupDelayDuration := min(interval, maxStartupDelay) go func() { + defer close(done) // Run once shortly after startup so a tiny dataset stays small. startupDelay := time.NewTimer(startupDelayDuration) defer startupDelay.Stop() @@ -56,6 +62,7 @@ func StartEventPruner(ctx context.Context, s EventStore, retention, interval tim } } }() + return done } func runPrune(ctx context.Context, s EventStore, retention time.Duration) { diff --git a/Server/ws/graceful_stop_ctx_test.go b/Server/ws/graceful_stop_ctx_test.go new file mode 100644 index 00000000..6579e8f0 --- /dev/null +++ b/Server/ws/graceful_stop_ctx_test.go @@ -0,0 +1,43 @@ +package ws + +import ( + "context" + "testing" + "time" +) + +// TestGracefulStopContext_IdleSkipsNoticeWait locks the idle fast path: with +// nobody connected there is no one to hear the restart notice, so shutdown +// must not sleep the 5s countdown. +func TestGracefulStopContext_IdleSkipsNoticeWait(t *testing.T) { + h := &Hub{stop: make(chan struct{})} + start := time.Now() + h.GracefulStopContext(context.Background()) + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("idle GracefulStop took %v, want fast (no notice sleep)", elapsed) + } +} + +// TestGracefulStopContext_BudgetBoundsNoticeWait locks the shutdown-budget +// contract: with clients connected, the notice wait ends when the caller's +// context expires instead of always burning the full 5 seconds. +func TestGracefulStopContext_BudgetBoundsNoticeWait(t *testing.T) { + send := make(chan []byte, 8) + h := &Hub{stop: make(chan struct{})} + c := &Client{hub: h, send: send, sendHigh: send, sendLow: send} + h.clients = map[int64]*Client{1: c} + h.pubsub = NewPubSub() + h.pubsub.Subscribe(c, TopicGlobal) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + h.GracefulStopContext(ctx) + elapsed := time.Since(start) + if elapsed >= 5*time.Second { + t.Fatalf("GracefulStopContext ignored the ctx budget (took %v)", elapsed) + } + if !c.isSendClosed() { + t.Fatal("client connection was not closed by GracefulStopContext") + } +} diff --git a/Server/ws/harvest_s4_internal_test.go b/Server/ws/harvest_s4_internal_test.go index c72e5d53..2e0fee4d 100644 --- a/Server/ws/harvest_s4_internal_test.go +++ b/Server/ws/harvest_s4_internal_test.go @@ -263,7 +263,10 @@ func TestFailedHandshake_MarksUserOfflineWhenNoReplacementRemains(t *testing.T) t.Error("user stuck online after a failed handshake with no surviving connection") } - // The other clients must hear about it too. + // The other clients must hear about it too. Presence goes through the + // QueuePresence coalescer now — force the flush instead of waiting out + // the window. + h.flushPresenceQueue() select { case bm := <-h.broadcast: if !bytes.Contains(bm.msg, []byte("presence")) { diff --git a/Server/ws/hub.go b/Server/ws/hub.go index c35dbb37..d0b5e059 100644 --- a/Server/ws/hub.go +++ b/Server/ws/hub.go @@ -4,6 +4,7 @@ package ws import ( "context" "log/slog" + "os" "sync" "sync/atomic" "time" @@ -72,6 +73,33 @@ type Hub struct { // call fails loudly instead of racing the dispatch loop. running atomic.Bool + // dispatchExited flips when Run returns for good — normal Stop or the + // panic breaker. /health reads it (via DispatchAlive) because clients keep + // registering and appearing online through registerNow even with the + // dispatch loop dead, so nothing else makes the outage observable. + dispatchExited atomic.Bool + + // fatalFn runs when the panic breaker trips (3 panics/60s). A hub that + // panicked three times in a minute has unknown state, so production exits + // the process and lets the supervisor restart it rather than serving + // connections that can never receive a broadcast. Tests replace it. + fatalFn func() + + // Aggregate per-client backpressure counters. The per-client msgsDropped + // field is read once at disconnect and lost; these survive as process + // totals for the metrics endpoint. + bpQueueDisconnects atomic.Uint64 // clients disconnected because send (or high+send) overflowed + bpHighFallbacks atomic.Uint64 // high-priority sends that fell back to the normal buffer + bpLowDrops atomic.Uint64 // low-priority messages silently dropped on overflow + + // connRejects counts upgrade requests refused by the max_ws_connections + // capacity guardrail (ServeWS). + connRejects atomic.Uint64 + + // coldReplayLimit caps persisted-event replay per reconnect. 0 = the + // compiled-in default (maxColdReplay). Set via ConfigureReplay before Run. + coldReplayLimit int + // In-flight guards for the DB-heavy sweeps Run kicks off in their own // goroutines (startSweep): a tick that arrives while the previous sweep // is still running is skipped rather than stacked. @@ -101,6 +129,12 @@ type Hub struct { // Protected by keyHolderMu. keyHolderMu sync.RWMutex voiceKeyHolders map[int64]int64 + + // Presence coalescer (QueuePresence): latest queued presence per user and + // whether a flush timer is armed. Guarded by presenceMu. + presenceMu syncutil.Mutex + presenceQueue map[int64]pendingPresence + presenceFlushArmed bool } // NewHub creates a Hub ready to be started with Run. @@ -124,6 +158,7 @@ func NewHub(database *db.DB, limiter *auth.RateLimiter, svc *service.Services) * settingsName: "OwnCord Server", settingsMotd: "Welcome!", voiceKeyHolders: make(map[int64]int64), + fatalFn: func() { os.Exit(1) }, } // V2 handler registrations (need Hub fields for deps). @@ -219,6 +254,7 @@ func (h *Hub) refreshSettingsLocked(ctx context.Context) { // avoid a tight crash loop. func (h *Hub) Run() { h.running.Store(true) + defer h.dispatchExited.Store(true) var panicCount int var lastPanicReset time.Time @@ -246,8 +282,19 @@ func (h *Hub) Run() { "stack", stackutil.Capture()) if panicCount >= 3 { - slog.Error("hub: too many panics in 60s, stopping") + // The hub's state after three panics in a minute is + // unknown, and a stopped dispatch loop is invisible from + // the outside: registerNow keeps admitting clients that + // can never receive a broadcast. Exit and let the + // process supervisor restart us (fatalFn is os.Exit(1) + // in production; tests substitute a no-op and rely on + // the Stop below). + slog.Error("hub: too many panics in 60s, stopping and exiting for supervisor restart") h.Stop() + h.dispatchExited.Store(true) + if h.fatalFn != nil { + h.fatalFn() + } return } } @@ -300,19 +347,40 @@ func (h *Hub) Stop() { } // GracefulStop stops the LiveKit process (if managed) and then stops the hub. -// Safe to call multiple times concurrently. +// Safe to call multiple times concurrently. Prefer GracefulStopContext where a +// shutdown budget exists — this variant waits the full client-notice window. func (h *Hub) GracefulStop() { + h.GracefulStopContext(context.Background()) +} + +// GracefulStopContext is GracefulStop bounded by ctx: the client-notice wait +// ends early when ctx expires, so the hub's drain counts against the caller's +// shutdown budget instead of extending it. Safe to call multiple times +// concurrently (only the first call's ctx is used). +func (h *Hub) GracefulStopContext(ctx context.Context) { h.gracefulOnce.Do(func() { - // Broadcast restart notice to all connected clients. - h.BroadcastServerRestart("shutdown", 5) + // The notice window matters only when someone is connected to hear + // it — an idle server (and every early-return startup path) skips + // straight to teardown. + hasClients := h.ClientCount() > 0 + if hasClients { + // Broadcast restart notice to all connected clients. + h.BroadcastServerRestart("shutdown", 5) + } // Stop LiveKit process. if h.lkProcess != nil { h.lkProcess.Stop() } - // Give clients 5 seconds to disconnect gracefully. - time.Sleep(5 * time.Second) + // Give clients the promised notice window to disconnect gracefully — + // the 5s matches the countdown BroadcastServerRestart told them. + if hasClients { + select { + case <-time.After(5 * time.Second): + case <-ctx.Done(): + } + } // Close all remaining client connections. h.mu.Lock() @@ -576,6 +644,63 @@ func (h *Hub) BroadcastDropCount() uint64 { return h.broadcastDrops.Load() } +// DispatchAlive reports whether the hub's dispatch loop is still running. +// It is true before Run starts (so a health probe racing startup does not +// flap) and false once Run has returned — normal shutdown or the panic +// breaker. Safe to call from any goroutine. +func (h *Hub) DispatchAlive() bool { + return !h.dispatchExited.Load() +} + +// BackpressureStats returns the process-lifetime per-client backpressure +// counters: connections closed due to send-buffer overflow, high-priority +// sends that fell back to the normal buffer, and low-priority messages +// silently dropped. Safe to call from any goroutine. +func (h *Hub) BackpressureStats() (queueDisconnects, highFallbacks, lowDrops uint64) { + return h.bpQueueDisconnects.Load(), h.bpHighFallbacks.Load(), h.bpLowDrops.Load() +} + +// ConnRejectCount returns how many WebSocket upgrade requests were refused by +// the max_ws_connections capacity guardrail. Safe to call from any goroutine. +func (h *Hub) ConnRejectCount() uint64 { + return h.connRejects.Load() +} + +// ConfigureReplay resizes the reconnect replay budget: the in-memory ring and +// the persisted-event cap per reconnect (event_persistence.replay_ring_size / +// replay_cold_limit). Zero or negative values keep the compiled-in defaults. +// Must be called before Run — the dispatch loop reads replayBuf unlocked. +func (h *Hub) ConfigureReplay(ringSize, coldLimit int) { + if h.rejectIfRunning("ConfigureReplay") { + return + } + if ringSize > 0 { + h.replayBuf = NewEventRingBuffer(ringSize) + } + if coldLimit > 0 { + h.coldReplayLimit = coldLimit + } +} + +// maxColdReplayLimit returns the effective persisted-replay cap. +func (h *Hub) maxColdReplayLimit() int { + if h.coldReplayLimit > 0 { + return h.coldReplayLimit + } + return maxColdReplay +} + +// EventPersisterStats returns the attached persister's lifetime counters. +// ok is false when event persistence is disabled (no persister attached). +func (h *Hub) EventPersisterStats() (persisted, dropped, flushes, errs uint64, ok bool) { + p := h.eventPersister.Load() + if p == nil { + return 0, 0, 0, 0, false + } + persisted, dropped, flushes, errs = p.Stats() + return persisted, dropped, flushes, errs, true +} + // VoiceSessionCount returns the number of clients currently in a voice channel. func (h *Hub) VoiceSessionCount() int { h.mu.RLock() diff --git a/Server/ws/hub_broadcast.go b/Server/ws/hub_broadcast.go index 83ff91b7..d168defc 100644 --- a/Server/ws/hub_broadcast.go +++ b/Server/ws/hub_broadcast.go @@ -3,9 +3,11 @@ package ws import ( "context" "log/slog" + "time" "github.com/owncord/server/db" "github.com/owncord/server/permissions" + "github.com/owncord/server/telemetry" ) // broadcastMsg is an internal message queued for delivery. @@ -18,6 +20,9 @@ type broadcastMsg struct { // recipient's role may not READ, and the audience is resolved off the hub // goroutine so deliverBroadcast stays free of permission queries. recipients []int64 + // enqueuedAt stamps the enqueue site so deliverBroadcast can record + // enqueue→fanout latency. Zero on test-constructed messages; skipped then. + enqueuedAt time.Time } // BroadcastToChannel enqueues msg for delivery to all clients subscribed to @@ -25,7 +30,7 @@ type broadcastMsg struct { // Non-blocking: if the broadcast channel is full the message is dropped with a warning. func (h *Hub) BroadcastToChannel(channelID int64, msg []byte) { select { - case h.broadcast <- broadcastMsg{channelID: channelID, msg: msg}: + case h.broadcast <- broadcastMsg{channelID: channelID, msg: msg, enqueuedAt: time.Now()}: default: h.broadcastDrops.Add(1) slog.Warn("hub: broadcast channel full, dropping message", @@ -37,7 +42,7 @@ func (h *Hub) BroadcastToChannel(channelID int64, msg []byte) { // Non-blocking: if the broadcast channel is full the message is dropped with a warning. func (h *Hub) BroadcastToAll(msg []byte) { select { - case h.broadcast <- broadcastMsg{channelID: 0, msg: msg}: + case h.broadcast <- broadcastMsg{channelID: 0, msg: msg, enqueuedAt: time.Now()}: default: h.broadcastDrops.Add(1) slog.Warn("hub: broadcast channel full, dropping global message", @@ -127,6 +132,7 @@ func (h *Hub) broadcastChannelScopedTo(channelID int64, msg []byte, recipients [ channelID: channelID, msg: msg, recipients: recipients, + enqueuedAt: time.Now(), } select { case h.broadcast <- bm: @@ -557,10 +563,73 @@ func (h *Hub) BroadcastUserUpdate(u UserUpdate) { h.BroadcastToAll(buildUserUpdate(u)) } +// presenceCoalesceWindow is how long QueuePresence buffers connect/disconnect +// presence before flushing. Long enough to collapse a socket flap +// (disconnect+reconnect through a proxy blip) into one frame, short enough +// that a genuine arrival still looks immediate to humans. +const presenceCoalesceWindow = 300 * time.Millisecond + +// pendingPresence is the coalescer's latest-wins entry for one user. +type pendingPresence struct { + status string + customStatus *string +} + +// QueuePresence coalesces connect/disconnect presence broadcasts: the latest +// state per user is buffered for presenceCoalesceWindow and then flushed via +// BroadcastPresence. Each un-coalesced presence change is a sequenced global +// broadcast — an O(connected clients) fan-out under seqMu — so a reconnect +// storm (proxy blip, deploy, network hiccup) used to fire O(users) of them +// from the connect critical path all at once. Latest-wins is exactly +// presence's semantics: a flap inside the window collapses to its final +// state, and the flushed frames are ordinary sequenced presence messages, so +// the wire format and replay behaviour are unchanged. User-chosen status +// changes (presence_update handler) do not pass through here. +func (h *Hub) QueuePresence(userID int64, status string, customStatus *string) { + h.presenceMu.Lock() + if h.presenceQueue == nil { + h.presenceQueue = make(map[int64]pendingPresence) + } + h.presenceQueue[userID] = pendingPresence{status: status, customStatus: customStatus} + armed := h.presenceFlushArmed + h.presenceFlushArmed = true + h.presenceMu.Unlock() + if !armed { + time.AfterFunc(presenceCoalesceWindow, h.flushPresenceQueue) + } +} + +// dropQueuedPresence removes a user's pending coalesced presence, if any. +// Called when a fresher presence for that user is broadcast directly (the +// presence_update handler path), so the coalescer's later flush cannot +// resurrect the stale connect-time state over it. Ordering holds because a +// user's connect (which queues) and their presence_update (which drops) run +// serially on the same connection's readPump. +func (h *Hub) dropQueuedPresence(userID int64) { + h.presenceMu.Lock() + delete(h.presenceQueue, userID) + h.presenceMu.Unlock() +} + +// flushPresenceQueue drains the coalescer and broadcasts each user's latest +// presence. Runs on the AfterFunc timer goroutine, never under presenceMu +// during the fan-out. +func (h *Hub) flushPresenceQueue() { + h.presenceMu.Lock() + queued := h.presenceQueue + h.presenceQueue = nil + h.presenceFlushArmed = false + h.presenceMu.Unlock() + for uid, p := range queued { + h.BroadcastPresence(uid, p.status, p.customStatus) + } +} + // BroadcastPresence fans a presence change out with the invisible mapping // applied: everyone else sees db.BroadcastStatus(status), the user themselves // sees the truth. It is the non-handler counterpart of presenceEvents, used by -// the connect and disconnect paths which write to the hub directly. +// the connect and disconnect paths (via the QueuePresence coalescer, which +// delivers through here). func (h *Hub) BroadcastPresence(userID int64, status string, customStatus *string) { public := db.BroadcastStatus(status) if public == status { @@ -801,6 +870,17 @@ func (h *Hub) deliverBroadcast(bm broadcastMsg) { return seq, delivered, channelSend }() + // Instrumentation runs after seqMu is released: a metrics provider must + // never extend the critical section that serializes every broadcast. + // seq == 0 means the topic limiter shed the frame before delivery. + if seq != 0 { + m := telemetry.NewAppMetrics() + m.WSMessagesTotal.Add(context.Background(), 1) + if !bm.enqueuedAt.IsZero() { + m.WSBroadcastLatency.Record(context.Background(), time.Since(bm.enqueuedAt).Seconds()) + } + } + if channelSend { slog.Debug("hub: channel broadcast", "channel_id", bm.channelID, "delivered", delivered, "seq", seq) diff --git a/Server/ws/hub_sweep.go b/Server/ws/hub_sweep.go index c64852bc..c066b0f1 100644 --- a/Server/ws/hub_sweep.go +++ b/Server/ws/hub_sweep.go @@ -9,6 +9,7 @@ import ( "github.com/owncord/server/auth" "github.com/owncord/server/db" "github.com/owncord/server/permissions" + "github.com/owncord/server/telemetry" ) // staleClientTimeout is the maximum duration a client can go without sending @@ -22,6 +23,32 @@ func (h *Hub) onStaleTick() { // Per-channel token buckets are created on first broadcast; prune idle // ones here or the bucket map grows for the process lifetime. h.topicLimiter.Cleanup(10 * time.Minute) + h.refreshTelemetryGauges() +} + +// refreshTelemetryGauges recomputes the connection and voice gauges from live +// client state. Periodic (30s tick, driven by the ctx-less Run loop) rather +// than event-driven: join/leave and register/unregister paths are spread +// across handlers, sweeps, and webhooks — many of them request-scoped, where +// a context.Background() instrumentation call would trip contextcheck — and a +// gauge only needs to be right at scrape time. +func (h *Hub) refreshTelemetryGauges() { + participants := 0 + rooms := make(map[int64]struct{}) + h.mu.RLock() + connected := len(h.clients) + for _, c := range h.clients { + if chID := c.getVoiceChID(); chID != 0 { + participants++ + rooms[chID] = struct{}{} + } + } + h.mu.RUnlock() + m := telemetry.NewAppMetrics() + ctx := context.Background() + m.WSActiveConnections.Set(ctx, float64(connected)) + m.VoiceParticipants.Set(ctx, float64(participants)) + m.VoiceActiveSessions.Set(ctx, float64(len(rooms))) } // kickClient forcibly removes a client from the hub and closes its send channel, diff --git a/Server/ws/observability_test.go b/Server/ws/observability_test.go new file mode 100644 index 00000000..b3d72849 --- /dev/null +++ b/Server/ws/observability_test.go @@ -0,0 +1,111 @@ +package ws + +import ( + "testing" + "time" +) + +// TestBackpressureStats_CountsPerPolicy locks the aggregate counters onto the +// three distinct overflow policies: normal overflow disconnects, high-priority +// overflow falls back then disconnects, low-priority overflow silently drops. +func TestBackpressureStats_CountsPerPolicy(t *testing.T) { + h := &Hub{} + c := &Client{ + hub: h, + send: make(chan []byte, 1), + sendHigh: make(chan []byte, 1), + sendLow: make(chan []byte, 1), + } + + // Low priority: first fills the buffer, second silently drops. + c.sendLowMsg([]byte("a")) + c.sendLowMsg([]byte("b")) + + // High priority: first fills sendHigh; second falls back into send (room); + // third finds both full → fallback counted, then disconnect counted. + c.sendHighMsg([]byte("c")) + c.sendHighMsg([]byte("d")) + c.sendHighMsg([]byte("e")) + + qd, hf, ld := h.BackpressureStats() + if ld != 1 { + t.Errorf("lowDrops = %d, want 1", ld) + } + if hf != 2 { + t.Errorf("highFallbacks = %d, want 2", hf) + } + if qd != 1 { + t.Errorf("queueDisconnects = %d, want 1", qd) + } + if !c.isSendClosed() { + t.Error("client should be disconnected after high+normal overflow") + } + + // A hub-less client must not panic on any overflow path. + loner := &Client{send: make(chan []byte), sendHigh: make(chan []byte), sendLow: make(chan []byte)} + loner.sendLowMsg([]byte("x")) + loner.sendMsg([]byte("y")) +} + +// TestDispatchAlive_FlipsOnStop locks the /health liveness contract: alive +// before Run, alive while running, dead once Run has returned. +func TestDispatchAlive_FlipsOnStop(t *testing.T) { + h := &Hub{ + stop: make(chan struct{}), + clientEvents: make(chan clientEvent, 1), + broadcast: make(chan broadcastMsg, 1), + } + if !h.DispatchAlive() { + t.Fatal("hub must report alive before Run starts") + } + done := make(chan struct{}) + go func() { h.Run(); close(done) }() + h.Stop() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not exit after Stop") + } + if h.DispatchAlive() { + t.Fatal("hub must report dead after Run returns") + } +} + +// TestPanicBreaker_CallsFatalFn locks the supervisor-restart contract: three +// dispatch-loop panics inside the 60s window stop the hub AND invoke fatalFn +// (os.Exit(1) in production), so a supervisor can restart the process instead +// of the outage staying invisible. +func TestPanicBreaker_CallsFatalFn(t *testing.T) { + h := &Hub{ + stop: make(chan struct{}), + clientEvents: make(chan clientEvent, 1), + broadcast: make(chan broadcastMsg, 3), + } + fatal := make(chan struct{}) + h.fatalFn = func() { close(fatal) } + + // replayBuf is nil, so deliverBroadcast panics on Push — inside the + // closure whose deferred seqMu unlock keeps the lock state clean across + // the recover, unlike a hand-rolled unlock would. + bad := broadcastMsg{channelID: 0, msg: []byte(`{"type":"x"}`)} + h.broadcast <- bad + h.broadcast <- bad + h.broadcast <- bad + + done := make(chan struct{}) + go func() { h.Run(); close(done) }() + + select { + case <-fatal: + case <-time.After(5 * time.Second): + t.Fatal("fatalFn was not called after 3 panics") + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not exit after the breaker tripped") + } + if h.DispatchAlive() { + t.Fatal("hub must report dead after the breaker tripped") + } +} diff --git a/Server/ws/presence_coalesce_drop_test.go b/Server/ws/presence_coalesce_drop_test.go new file mode 100644 index 00000000..5802b71d --- /dev/null +++ b/Server/ws/presence_coalesce_drop_test.go @@ -0,0 +1,56 @@ +package ws + +import ( + "context" + "testing" +) + +// TestEmitEvents_DirectPresenceDropsQueuedEntry locks the coalescer-inversion +// fix: a user-chosen presence_update broadcast must invalidate any queued +// connect-time presence for the same user, or the coalescer's later flush +// (up to 300ms) would overwrite the fresh status with the stale one. +func TestEmitEvents_DirectPresenceDropsQueuedEntry(t *testing.T) { + h := &Hub{ + broadcast: make(chan broadcastMsg, 8), + pubsub: NewPubSub(), + } + + // Simulate connect: queue the connect-time presence. + h.QueuePresence(7, "online", nil) + h.presenceMu.Lock() + _, queued := h.presenceQueue[7] + h.presenceMu.Unlock() + if !queued { + t.Fatal("QueuePresence did not queue the entry") + } + + // The user immediately sets a status via presence_update (visible path). + h.EmitEvents(context.Background(), presenceEvents(7, "dnd", nil)) + + h.presenceMu.Lock() + _, stillQueued := h.presenceQueue[7] + h.presenceMu.Unlock() + if stillQueued { + t.Fatal("direct presence broadcast left the stale queued entry; flush would overwrite the fresh status") + } + + // The invisible path must drop it too (public half rides PresenceOthersEvent). + h.QueuePresence(7, "online", nil) + h.EmitEvents(context.Background(), presenceEvents(7, "invisible", nil)) + h.presenceMu.Lock() + _, stillQueued = h.presenceQueue[7] + h.presenceMu.Unlock() + if stillQueued { + t.Fatal("invisible presence broadcast left the stale queued entry") + } + + // Other users' queued entries are untouched. + h.QueuePresence(8, "online", nil) + h.EmitEvents(context.Background(), presenceEvents(7, "idle", nil)) + h.presenceMu.Lock() + _, otherKept := h.presenceQueue[8] + h.presenceMu.Unlock() + if !otherKept { + t.Fatal("dropQueuedPresence removed an unrelated user's entry") + } +} diff --git a/Server/ws/presence_coalesce_test.go b/Server/ws/presence_coalesce_test.go new file mode 100644 index 00000000..ec337747 --- /dev/null +++ b/Server/ws/presence_coalesce_test.go @@ -0,0 +1,51 @@ +package ws + +import ( + "bytes" + "testing" +) + +// TestQueuePresence_CoalescesLatestWins locks the coalescer's contract: a +// flap (multiple queued states for one user inside the window) flushes as ONE +// broadcast carrying the latest state, and distinct users each get their own. +func TestQueuePresence_CoalescesLatestWins(t *testing.T) { + h := &Hub{broadcast: make(chan broadcastMsg, 16)} + + h.QueuePresence(1, "offline", nil) + h.QueuePresence(1, "online", nil) // same user: latest wins + h.QueuePresence(2, "offline", nil) + + // Nothing may reach the broadcast queue before the flush. + if got := len(h.broadcast); got != 0 { + t.Fatalf("broadcasts before flush = %d, want 0 (coalesced)", got) + } + + h.flushPresenceQueue() + + var frames [][]byte + for len(h.broadcast) > 0 { + frames = append(frames, (<-h.broadcast).msg) + } + if len(frames) != 2 { + t.Fatalf("flushed %d broadcasts, want 2 (one per user)", len(frames)) + } + sawUser1Online := false + for _, f := range frames { + if bytes.Contains(f, []byte(`"user_id":1`)) { + if bytes.Contains(f, []byte("offline")) { + t.Fatalf("user 1's flap flushed the stale state: %s", f) + } + sawUser1Online = bytes.Contains(f, []byte("online")) + } + } + if !sawUser1Online { + t.Fatal("user 1's latest (online) presence was not flushed") + } + + // The flush disarms the timer state — a later queue+flush works again. + h.QueuePresence(1, "idle", nil) + h.flushPresenceQueue() + if got := len(h.broadcast); got != 1 { + t.Fatalf("second cycle flushed %d broadcasts, want 1", got) + } +} diff --git a/Server/ws/reconnect_active_channel_test.go b/Server/ws/reconnect_active_channel_test.go index 3fa1889a..ae8954d6 100644 --- a/Server/ws/reconnect_active_channel_test.go +++ b/Server/ws/reconnect_active_channel_test.go @@ -82,7 +82,7 @@ func TestReconnect_AuthFrameActiveChannelRestoresSubscription(t *testing.T) { rb.Push(99, chID, []byte(`{"seq":99,"type":"chat_message","payload":{}}`)) rb.Push(100, chID, []byte(`{"seq":100,"type":"chat_message","payload":{}}`)) - srv := httptest.NewServer(ServeWS(hub, database, []string{"*"})) + srv := httptest.NewServer(ServeWS(hub, database, []string{"*"}, 0)) defer srv.Close() dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -198,7 +198,7 @@ func TestReconnect_AuthFrameActiveChannelIsReadGated(t *testing.T) { hub.ReplayBuffer().Push(99, 0, []byte(`{"seq":99,"type":"presence","payload":{}}`)) hub.ReplayBuffer().Push(100, 0, []byte(`{"seq":100,"type":"presence","payload":{}}`)) - srv := httptest.NewServer(ServeWS(hub, database, []string{"*"})) + srv := httptest.NewServer(ServeWS(hub, database, []string{"*"}, 0)) defer srv.Close() dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) diff --git a/Server/ws/reconnect_db_test.go b/Server/ws/reconnect_db_test.go index 2254535c..b3ed6f8a 100644 --- a/Server/ws/reconnect_db_test.go +++ b/Server/ws/reconnect_db_test.go @@ -106,7 +106,7 @@ func TestReconnect_BufferMiss_FallsBackToDBTier(t *testing.T) { } // Spin up a real HTTP+WS server. - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -224,7 +224,7 @@ func TestReconnect_ColdTierAtRowLimit_ForcesFullReady(t *testing.T) { t.Fatalf("pre-condition: expected oldestSeq=501, got %d", oldest) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -314,7 +314,7 @@ func TestReconnect_ColdTierMergesRingBufferTail(t *testing.T) { rb.Push(seq, 0, fmt.Appendf(nil, `{"seq":%d,"type":"broadcast"}`, seq)) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() diff --git a/Server/ws/reconnect_interior_gap_test.go b/Server/ws/reconnect_interior_gap_test.go index ea411b63..637adbfd 100644 --- a/Server/ws/reconnect_interior_gap_test.go +++ b/Server/ws/reconnect_interior_gap_test.go @@ -93,7 +93,7 @@ func TestReconnect_InteriorGap_ForcesFullReady(t *testing.T) { t.Fatalf("pre-condition: expected oldestSeq=501, got %d", oldest) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() diff --git a/Server/ws/reconnect_pruned_prefix_test.go b/Server/ws/reconnect_pruned_prefix_test.go index 41b80c5e..ed889cba 100644 --- a/Server/ws/reconnect_pruned_prefix_test.go +++ b/Server/ws/reconnect_pruned_prefix_test.go @@ -81,7 +81,7 @@ func TestReconnect_PrunedPrefix_ForcesFullReady(t *testing.T) { t.Fatalf("pre-condition: expected oldestSeq=1000, got %d", oldest) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() diff --git a/Server/ws/reconnect_voice_supplement_test.go b/Server/ws/reconnect_voice_supplement_test.go index e15ee892..9447ccbf 100644 --- a/Server/ws/reconnect_voice_supplement_test.go +++ b/Server/ws/reconnect_voice_supplement_test.go @@ -107,7 +107,7 @@ func TestReconnect_ReplaysOwnVoiceRoomOutsideReadableChannels(t *testing.T) { push(102, MsgTypeVoiceState) push(103, MsgTypeVoiceLeaveBC) - srv := httptest.NewServer(ServeWS(hub, database, []string{"*"})) + srv := httptest.NewServer(ServeWS(hub, database, []string{"*"}, 0)) defer srv.Close() dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second) diff --git a/Server/ws/serve.go b/Server/ws/serve.go index 7ffdea3f..9f139f2a 100644 --- a/Server/ws/serve.go +++ b/Server/ws/serve.go @@ -39,9 +39,21 @@ const ( // allowedOrigins controls which HTTP origins may open a WebSocket connection. // Pass nil or []string{"*"} to allow all origins (insecure, for development). // Pass explicit origins such as []string{"https://example.com"} to restrict access. -func ServeWS(hub *Hub, database *db.DB, allowedOrigins []string) http.HandlerFunc { +// +// maxConns, when > 0, refuses new connections with 503 once that many clients +// are registered — a static capacity guardrail (server.max_ws_connections). +// The check runs before the upgrade so a refused connection costs one HTTP +// request, not a socket plus goroutines. Registered count trails pre-auth +// connections by design; the 10s auth deadline bounds that gap. +func ServeWS(hub *Hub, database *db.DB, allowedOrigins []string, maxConns int) http.HandlerFunc { acceptOpts := OriginAcceptOptions(allowedOrigins) return func(w http.ResponseWriter, r *http.Request) { + if maxConns > 0 && hub.ClientCount() >= maxConns { + hub.connRejects.Add(1) + w.Header().Set("Retry-After", "30") + http.Error(w, "server at connection capacity", http.StatusServiceUnavailable) + return + } conn, err := websocket.Accept(w, r, acceptOpts) if err != nil { slog.Warn("ws upgrade failed", "err", err) @@ -195,12 +207,13 @@ func (h *Hub) handleReconnect( for cid := range allowedChannelIDs { channelIDs = append(channelIDs, cid) } - persisted, dbErr := es.GetEventsSinceForChannels(ctx, int64(lastSeq), channelIDs, maxColdReplay) //nolint:gosec // lastSeq is a sequence counter bounded well below MaxInt64 + coldCap := h.maxColdReplayLimit() + persisted, dbErr := es.GetEventsSinceForChannels(ctx, int64(lastSeq), channelIDs, coldCap) //nolint:gosec // lastSeq is a sequence counter bounded well below MaxInt64 switch { case dbErr != nil: slog.Warn("ws handleReconnect: cold-tier replay query failed", "user_id", c.userID, "err", dbErr) - case len(persisted) >= maxColdReplay: + case len(persisted) >= coldCap: // The query is "ORDER BY seq ASC LIMIT maxColdReplay", so a full // result means the gap exceeds the cap and the NEWEST events were // dropped. Replaying it would look like a complete resume to the @@ -208,7 +221,7 @@ func (h *Hub) handleReconnect( // silently losing state events that REST history never repairs. // Leave events nil so the fall-through forces a full ready. slog.Warn("ws handleReconnect: cold-tier replay hit the row cap, forcing full ready", - "user_id", c.userID, "last_seq", lastSeq, "cap", maxColdReplay) + "user_id", c.userID, "last_seq", lastSeq, "cap", coldCap) case len(persisted) > 0: // Retention pruning (PruneEventsOlderThan) deletes purely by // created_at with no seq-floor coordination, so this @@ -462,7 +475,7 @@ func (h *Hub) liveVoiceEventsSince(ctx context.Context, afterSeq uint64, chID in raw = buf } else if esp := h.eventStore.Load(); esp != nil { es := *esp - persisted, err := es.GetEventsSinceForChannels(ctx, int64(afterSeq), []int64{chID}, maxColdReplay) //nolint:gosec // afterSeq is a sequence counter bounded well below MaxInt64 + persisted, err := es.GetEventsSinceForChannels(ctx, int64(afterSeq), []int64{chID}, h.maxColdReplayLimit()) //nolint:gosec // afterSeq is a sequence counter bounded well below MaxInt64 if err != nil { return nil } @@ -522,7 +535,7 @@ func (h *Hub) unregisterFailedHandshake(ctx context.Context, c *Client) { // note in serve_pumps.go's readPump defer — that field is an // auth-time snapshot, never updated, so broadcasting it here can // resurrect a status the user already changed or cleared. - h.BroadcastToAll(buildPresenceMsg(c.userID, db.StatusOffline, nil)) + h.QueuePresence(c.userID, db.StatusOffline, nil) } } @@ -550,7 +563,7 @@ func applyConnectStatus(ctx context.Context, database *db.DB, c *Client) { // announceConnectPresence fans out the status applyConnectStatus settled on, // with the invisible mapping applied. func (h *Hub) announceConnectPresence(c *Client) { - h.BroadcastPresence(c.userID, c.user.Status, c.user.CustomStatus) + h.QueuePresence(c.userID, c.user.Status, c.user.CustomStatus) } // computeAllowedChannels returns the set of channel IDs a user may access, diff --git a/Server/ws/serve_failed_handshake_teardown_test.go b/Server/ws/serve_failed_handshake_teardown_test.go index 15bc7f22..f277786b 100644 --- a/Server/ws/serve_failed_handshake_teardown_test.go +++ b/Server/ws/serve_failed_handshake_teardown_test.go @@ -105,6 +105,7 @@ func TestFailedHandshake_TearsDownTransferredVoiceSession(t *testing.T) { // The voice_leave broadcast must go out too, or every other client keeps // rendering a tile for a participant that is gone. sawVoiceLeave := false + h.flushPresenceQueue() // presence is coalesced; flush before inspecting for len(h.broadcast) > 0 { bm := <-h.broadcast if bytes.Contains(bm.msg, []byte(`"type":"`+MsgTypeVoiceLeaveBC+`"`)) { @@ -139,6 +140,7 @@ func TestFailedHandshake_OfflineBroadcastDropsStaleCustomStatus(t *testing.T) { h.unregisterFailedHandshake(ctx, c) var presence []byte + h.flushPresenceQueue() // presence is coalesced; flush before inspecting for len(h.broadcast) > 0 { bm := <-h.broadcast if bytes.Contains(bm.msg, []byte(`"type":"`+MsgTypePresence+`"`)) { diff --git a/Server/ws/serve_pumps.go b/Server/ws/serve_pumps.go index d4a7b29f..008e7163 100644 --- a/Server/ws/serve_pumps.go +++ b/Server/ws/serve_pumps.go @@ -206,7 +206,7 @@ func readPump(ctx context.Context, conn *websocket.Conn, hub *Hub, c *Client) { // or cleared mid-session. presentableMembers applies the same // rule for a fresh ready payload (serve_ready.go) — a member with // no live connection shows no custom status. - hub.BroadcastToAll(buildPresenceMsg(c.userID, db.StatusOffline, nil)) + hub.QueuePresence(c.userID, db.StatusOffline, nil) } } }() diff --git a/Server/ws/serve_reconnect_double_teardown_test.go b/Server/ws/serve_reconnect_double_teardown_test.go index 4840475c..cfd36616 100644 --- a/Server/ws/serve_reconnect_double_teardown_test.go +++ b/Server/ws/serve_reconnect_double_teardown_test.go @@ -118,6 +118,7 @@ func TestHandleReconnect_HandshakeWriteFailure_TearsDownOnlyOnce(t *testing.T) { } var offlineBroadcasts int + h.flushPresenceQueue() // presence is coalesced; flush before inspecting for len(h.broadcast) > 0 { bm := <-h.broadcast if bytes.Contains(bm.msg, []byte(`"status":"offline"`)) { diff --git a/Server/ws/ws_integration_test.go b/Server/ws/ws_integration_test.go index a6029757..b7c4b13e 100644 --- a/Server/ws/ws_integration_test.go +++ b/Server/ws/ws_integration_test.go @@ -32,7 +32,7 @@ func TestServeWS_InvalidUpgrade_ReturnsError(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -60,7 +60,7 @@ func TestAuthenticateConn_NoAuthMessage_ServerClosesConn(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -99,7 +99,7 @@ func TestAuthenticateConn_InvalidJSON_ReceivesAuthError(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -145,7 +145,7 @@ func TestAuthenticateConn_WrongMessageType_ReceivesAuthError(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -194,7 +194,7 @@ func TestAuthenticateConn_MissingToken_ReceivesAuthError(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -242,7 +242,7 @@ func TestAuthenticateConn_InvalidToken_ReceivesAuthError(t *testing.T) { go hub.Run() defer hub.Stop() - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -304,7 +304,7 @@ func TestServeWS_ValidAuth_FullHandshake(t *testing.T) { t.Fatalf("CreateSession: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -384,7 +384,7 @@ func TestServeWS_ImmediateDisconnect_DoesNotLeaveGhostClient(t *testing.T) { t.Fatalf("CreateSession: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -460,7 +460,7 @@ func TestServeWS_DuplicateLogin_KeepsUserOnline(t *testing.T) { t.Fatalf("CreateSession: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -548,7 +548,7 @@ func TestServeWS_Reconnect_PreservesVoiceState(t *testing.T) { t.Fatalf("CreateChannel: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -707,7 +707,7 @@ func TestServeWS_ReplayFallback_PreservesVoiceState(t *testing.T) { t.Fatalf("CreateChannel: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -839,7 +839,7 @@ func TestServeWS_Reconnect_AuthorizedVoiceClientKeepsChannelStream(t *testing.T) t.Fatalf("CreateChannel: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -994,7 +994,7 @@ func TestServeWS_FreshReconnect_CleansStaleVoiceState(t *testing.T) { t.Fatalf("CreateChannel: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -1203,7 +1203,7 @@ func TestServeWS_writePump_MessageDelivered(t *testing.T) { t.Fatalf("CreateSession: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() @@ -1316,7 +1316,7 @@ func TestIntegration_MessageRoundTrip(t *testing.T) { t.Fatalf("CreateChannel: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") @@ -1445,7 +1445,7 @@ func TestIntegration_SequenceNumbers(t *testing.T) { t.Fatalf("CreateSession: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") @@ -1543,7 +1543,7 @@ func TestServeWS_BannedUser_ReceivesError(t *testing.T) { t.Fatalf("BanUser: %v", err) } - handler := ws.ServeWS(hub, database, []string{"*"}) + handler := ws.ServeWS(hub, database, []string{"*"}, 0) srv := httptest.NewServer(handler) defer srv.Close() diff --git a/deploy/owncord.service b/deploy/owncord.service new file mode 100644 index 00000000..f4ad6e66 --- /dev/null +++ b/deploy/owncord.service @@ -0,0 +1,47 @@ +# OwnCord server — systemd unit template. +# +# Install: +# 1. Create a service user and install directory: +# sudo useradd --system --home /opt/owncord --shell /usr/sbin/nologin owncord +# sudo mkdir -p /opt/owncord && sudo chown owncord:owncord /opt/owncord +# 2. Place the chatserver binary (from GitHub Releases) in /opt/owncord/. +# 3. sudo cp deploy/owncord.service /etc/systemd/system/owncord.service +# 4. sudo systemctl daemon-reload && sudo systemctl enable --now owncord +# +# Logs: journalctl -u owncord -f + +[Unit] +Description=OwnCord chat server +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=owncord +Group=owncord +WorkingDirectory=/opt/owncord +ExecStart=/opt/owncord/chatserver +Restart=on-failure +RestartSec=3 + +# The server drains gracefully on SIGTERM with a 30s budget; give it a little +# headroom before systemd escalates to SIGKILL. +TimeoutStopSec=35 + +# ── Hardening ─────────────────────────────────────────────────────────────── +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +# The install directory must stay WRITABLE: the admin panel's self-update +# renames the new binary into place (chatserver -> chatserver.old swap), and +# the data dir (SQLite, uploads, certs, backups) lives beneath it by default. +# If you disable self-update and move data_dir elsewhere, narrow this. +ReadWritePaths=/opt/owncord + +# Only needed for tls.mode: acme (binds :80 for HTTP-01 challenges) as a +# non-root user. Harmless otherwise; remove if you prefer. +AmbientCapabilities=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/docs/api.md b/docs/api.md index 97fde7c1..30e22ad0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1480,14 +1480,48 @@ Runtime server metrics. Restricted to admin-allowed CIDRs. "connected_users": 8, "voice_sessions": 2, "broadcast_drops": 0, - "livekit_healthy": true + "livekit_healthy": true, + "reconnect_tier_buffer": 120, + "reconnect_tier_db": 4, + "reconnect_tier_full": 1, + "backpressure_queue_disconnects": 0, + "backpressure_high_fallbacks": 0, + "backpressure_low_drops": 17, + "ws_conn_rejects": 0, + "disk_free_mb": 51200.5, + "db_writer_wait_count": 3, + "db_writer_wait_seconds": 0.021, + "perm_cache_hits": 5120, + "perm_cache_misses": 84, + "event_persister": { + "persisted": 4021, + "dropped": 0, + "flushes": 311, + "errors": 0 + } } ``` -`voice_sessions` is the number of active voice connections; `broadcast_drops` -is the cumulative count of WebSocket events dropped because a client send -queue was full. `livekit_healthy` is omitted when no LiveKit health check is -wired. +`voice_sessions` is the number of active voice connections. `broadcast_drops` +is the cumulative count of events dropped because the **hub-wide broadcast +queue** was full — sequenced events lost before delivery, worth alerting on +if it ever grows. Per-client send-queue pressure is reported separately: +`backpressure_queue_disconnects` (clients disconnected to force a +replay-recovering reconnect), `backpressure_high_fallbacks` (high-priority +sends that fell back to the normal queue), and `backpressure_low_drops` +(typing/presence messages silently dropped — safe to lose, but a growth trend +means clients are draining too slowly). `reconnect_tier_*` counts resume +attempts served from the in-memory ring buffer, the persisted event log, and +full-resync fallback; a rising `full` share means the replay budget is too +small for observed disconnect gaps. `db_writer_wait_count`/`_seconds` +accumulate time requests spent queueing for SQLite's single write connection — +the most direct saturation signal for the write path. `perm_cache_*` report +permission-cache effectiveness (a miss is any lookup that repopulated from the +database). `ws_conn_rejects` counts upgrades refused by the +`server.max_ws_connections` cap, and `disk_free_mb` is free space on the data +volume (omitted when the platform can't report it). `livekit_healthy` is +omitted when no LiveKit health check is wired; `event_persister` is omitted +when event persistence is disabled. ### GET /metrics (Prometheus) @@ -1807,6 +1841,16 @@ A flat map of key → string value. Allowed keys: `server_name`, `server_icon`, `registration_open`, `backup_schedule`, `backup_retention`. Boolean settings accept `1/0/true/false` and are normalized to `1`/`0`. +`backup_schedule` (`off`/`daily`/`weekly`) and `backup_retention` (days) are +enforced by the server's maintenance loop — see the Backup Strategy section +of `docs/deployment.md` for the exact semantics. + +Three keys are accepted and stored but have **no runtime effect**: +`server_icon` (reserved for a future release), `max_upload_bytes` (the real +limit is `upload.max_size_mb` in config.yaml, applied at startup), and +`voice_quality` (the real setting is `voice.quality` in config.yaml). The +admin panel shows them read-only for this reason. + Enabling `require_2fa` is refused unless registration is closed **and** every user has TOTP enabled. diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index b82f2e76..a49b150f 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -79,6 +79,7 @@ flowchart TB R2["in-memory pub/sub + replay ring buffer"] R3["process-local TOTP replay store"] R4["SQLite single-writer (MaxOpenConns=1)"] + R5["process-local presence/voice state
(wiped and rebuilt per process at boot)"] end BIN --- constraints ``` @@ -86,13 +87,23 @@ flowchart TB **What this shows.** The deployment unit is one process per community — TLS (self-signed, custom, or ACME), the DB, uploads, the admin panel, and optionally LiveKit are all owned by that process. The design is explicitly -single-instance: rate-limit windows, pub/sub, the replay ring buffer, and the -TOTP replay store are process-local, and SQLite runs with a single writer. -Horizontal scaling is out of scope today; the constraint boxes name exactly -what would have to move to shared infrastructure if that ever changes. A -15-minute maintenance goroutine (expired sessions, orphaned attachments, with a -circuit breaker) and graceful drain on SIGINT/SIGTERM round out the process -lifecycle. +single-instance: rate-limit windows, pub/sub, the replay ring buffer, the +TOTP replay store, and presence/voice state (derived from live hub membership +and cold-reset at every boot) are process-local, and SQLite runs with a +single writer — enforced by an OS-level lock beside the database file, so a +second process fails fast instead of silently fighting the first over that +state. Horizontal scaling is out of scope today; the constraint boxes name +exactly what would have to move to shared infrastructure if that ever +changes. A 15-minute maintenance goroutine (expired sessions, orphaned +attachments, scheduled backups, with a circuit breaker) and graceful drain on +SIGINT/SIGTERM round out the process lifecycle. + +A note on client platforms while the deployment story is in view: the desktop +client ships for Windows and Linux (x86_64 + ARM64) only. macOS is a +deliberate scope decision, not an oversight — a trustworthy macOS build +requires Apple notarization (paid developer enrollment plus CI signing +secrets), and an unsigned bundle would train users to bypass Gatekeeper. +Revisit when that commitment is on the table. **Source of truth:** `Server/main.go`, `Server/config/config.go`, `Server/docker-compose.yml`, `docs/deployment.md`, `docs/server-configuration.md`, diff --git a/docs/deployment.md b/docs/deployment.md index a725c604..fc213988 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -140,6 +140,30 @@ their own LiveKit can turn the toggle off or set `voice.livekit_binary`. The server listens on `https://0.0.0.0:8443` by default. See [Server Configuration](server-configuration.md) for all options. +## Running as a Linux Service (systemd) + +A crash — a panic under load, the OOM killer, a failed self-update — leaves a +bare-metal server down until someone notices, so run the binary under a +supervisor. A ready-made unit template ships in the repo at +[`deploy/owncord.service`](../deploy/owncord.service); installation steps are +in its header comments. The important choices it encodes: + +- `Restart=on-failure` — the server deliberately exits (rather than limping + along) when its WebSocket dispatch loop dies; the supervisor is what turns + that into a recovery. +- `TimeoutStopSec=35` — the server drains gracefully on SIGTERM with a 30s + budget; systemd waits it out before escalating. +- `ReadWritePaths=/opt/owncord` under `ProtectSystem=strict` — the install + directory must stay writable or the admin panel's self-update (which + renames the new binary into place) breaks. +- `AmbientCapabilities=CAP_NET_BIND_SERVICE` — only needed for + `tls.mode: acme`, which binds :80 for HTTP-01 challenges as a non-root + user. + +Pair it with the scheduled backups in the admin panel — or an external cron +line (see Backup Strategy below) if you prefer driving backups outside the +server. + ## Running as a Windows Service ### Option 1: NSSM (Non-Sucking Service Manager) @@ -211,6 +235,48 @@ tls: mode: "off" ``` +## Reverse Proxy Topology + +OwnCord terminates its own TLS by default and does not require a reverse +proxy. If you front it with one anyway (shared host, existing nginx, central +cert management), three things matter: + +1. **What the proxy can front.** Everything on port 8443 — the REST API, the + WebSocket at `/api/v1/ws`, the admin panel, uploads, **and LiveKit + signaling**, which the server already proxies at `/livekit/*`. You do NOT + need to expose LiveKit's port 7880 through your proxy. +2. **What the proxy cannot front.** WebRTC media: UDP 50000–60000 (and the + TCP 7881 fallback) must remain directly reachable on the host running + LiveKit. An HTTP reverse proxy never carries this traffic. +3. **Tell OwnCord about the proxy.** Set `server.trusted_proxies` to the + proxy's own address(es) (e.g. `["10.0.0.2/32"]`) so client IPs come from + `X-Forwarded-For` for rate limiting and the admin IP allowlist. List only + the proxy hops, never client networks. + +Working nginx snippet: + +```nginx +server { + listen 443 ssl; + server_name chat.example.com; + # ssl_certificate / ssl_certificate_key ... + + location / { + proxy_pass https://127.0.0.1:8443; # or http:// with tls.mode: off + proxy_http_version 1.1; # required for WebSocket upgrade + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Idle chat WebSockets outlive nginx's 60s default read timeout; + # the client pings every 30s, so 300s has comfortable margin. + proxy_read_timeout 300s; + proxy_send_timeout 300s; + client_max_body_size 100m; # match upload.max_size_mb + } +} +``` + ## Backup Strategy ### SQLite WAL Considerations @@ -226,11 +292,45 @@ The database uses SQLite WAL mode. Do NOT copy the `.db` file directly while the | `/admin/api/backups/{name}` | DELETE | Delete a backup (owner-only) | | `/admin/api/backups/{name}/restore` | POST | Restore from backup (owner-only; creates pre-restore safety backup first) | -Backups are stored in `data/backups/` with timestamps. +Backups are stored in the configured backup directory (default +`data/backups/`) with timestamps. Point it somewhere safer than the data +volume — another disk, or a mount that is shipped off-host (rsync, rclone, +a synced folder) — so backups don't share a single point of failure with the +live database and uploads: + +```yaml +backup: + dir: "/mnt/backup-disk/owncord" +``` + +Every backup is verified with SQLite's `integrity_check` right after it is +written (a failed backup is removed, never listed), and again before a +restore is allowed to overwrite the live database. + +Note that a backup runs `VACUUM INTO` on the database's single write +connection: writes queue for the duration (reads keep serving). On a large +database, prefer scheduling backups at a low-traffic time of day. ### Scheduled Backups -Use Windows Task Scheduler with PowerShell: +The **Backup Schedule** (off / daily / weekly) and **Retention (days)** +settings in the admin panel are enforced by the server's maintenance loop +(checked every 15 minutes): + +- A scheduled backup is taken when the newest backup on disk is older than + the schedule interval — a manual backup resets the clock too. +- Retention deletes backups older than the configured number of days, but + always keeps the newest one, so a stale schedule can never delete your + last copy. + +External scheduling still works if you prefer it — e.g. Linux cron: + +```bash +# Nightly at 03:00 via an admin API token +0 3 * * * curl -sk -X POST -H "Authorization: Bearer $OWNCORD_TOKEN" https://localhost:8443/admin/api/backup +``` + +or Windows Task Scheduler with PowerShell: ```powershell $headers = @{ "Cookie" = "session=" } @@ -255,6 +355,16 @@ Restoring replaces the live database file. A pre-restore safety backup is create } ``` +`status` is a real verdict, not a constant: the server probes its own +WebSocket dispatch loop, runs a bounded `SELECT 1` against the database, and +checks free disk space on the data volume. When any of those fail, the +endpoint returns HTTP 503 with `"status": "degraded"` and a `reason` field +naming the subsystem (`hub`, `database`, or `disk` — no further detail, since +the endpoint is unauthenticated). Checks are cached for a few seconds, so +polling it aggressively does not multiply database load. Point your uptime +monitor or container healthcheck at this endpoint and treat any 503 as +actionable. + The server version is deliberately not exposed on this unauthenticated endpoint (anti-fingerprinting hardening). @@ -273,10 +383,35 @@ endpoint (anti-fingerprinting hardening). "connected_users": 12, "voice_sessions": 3, "broadcast_drops": 0, - "livekit_healthy": true + "livekit_healthy": true, + "reconnect_tier_buffer": 120, + "reconnect_tier_db": 4, + "reconnect_tier_full": 1, + "backpressure_queue_disconnects": 0, + "backpressure_high_fallbacks": 0, + "backpressure_low_drops": 17, + "ws_conn_rejects": 0, + "disk_free_mb": 51200.5, + "db_writer_wait_count": 3, + "db_writer_wait_seconds": 0.021, + "perm_cache_hits": 5120, + "perm_cache_misses": 84, + "event_persister": { "persisted": 4021, "dropped": 0, "flushes": 311, "errors": 0 } } ``` +Signals worth watching as a community grows (see `docs/api.md` for full field +descriptions): + +- `broadcast_drops` growing at all → the hub-wide broadcast queue overflowed + and sequenced events were lost; alert on any growth. +- `db_writer_wait_seconds` climbing faster than uptime → requests are queueing + on SQLite's single write connection; the write path is saturating. +- `reconnect_tier_full` becoming a noticeable share of reconnects → the replay + budget is too small for real disconnect gaps. +- `backpressure_queue_disconnects` growing → clients are being force-cycled + because they drain too slowly (slow links or an overloaded server). + ### LiveKit Health `GET /api/v1/livekit/health` -- checks LiveKit companion process reachability. diff --git a/docs/plans/infrastructure-roadmap.md b/docs/plans/infrastructure-roadmap.md new file mode 100644 index 00000000..c03d83f4 --- /dev/null +++ b/docs/plans/infrastructure-roadmap.md @@ -0,0 +1,182 @@ +# Infrastructure roadmap — design + +Date: 2026-08-15 +Status: implemented 2026-08-15 (same PR), with two deliberate leftovers: +the TOTP/partial-auth persister seam (Track 2 §3 — lowest impact, cut to +bound the change) and published capacity numbers (Track 3 §7 — the +`load-baseline` workflow now exists to produce them; publish only measured +values). + +## Problem + +OwnCord is deliberately single-instance (D8, `docs/architecture/system-overview.md`), +and that decision holds. A multi-model review pass (inventory sweep, eight review +lenses, adversarial verification) concluded the runtime is further along than the +operations story: the biggest growth risks are operational gaps, not throughput +ceilings. This roadmap records the verified, non-sensitive recommendations in +three tracks. Findings with security-sensitive detail were reported to the +maintainer separately per `docs/security.md` and are intentionally not itemized +here. + +Overall verdict: no re-architecting needed. The codebase already fixes its own +bottlenecks where it finds them (session-touch throttle, batched audit/event +writers, WAL reader/writer pool split, the auth lockout persister seam). The +work below lets a single instance absorb roughly a 10x user increase without +revisiting D8. + +## Track 1 — Raise the single-instance ceiling + +Ordered by leverage. All stay inside D8; no new subsystems. + +1. **Hub dispatch-loop liveness.** The hub's panic breaker (3 panics/60s, + `Server/ws/hub.go`) stops broadcast delivery permanently with nothing + observing it — clients still connect and appear online. On trip: `os.Exit(1)` + so a supervisor restarts the process, and expose dispatch-loop liveness on + `/health`. Do not attempt in-process self-recovery. +2. **Connection capacity guardrail.** Add a configurable global ceiling on + concurrent WebSocket connections, checked before the upgrade and returning + 503 when reached. Static value; no adaptive logic. +3. **Presence broadcast coalescing.** Connect/disconnect presence broadcasts go + through the sequenced path and fan out to all clients; a reconnect storm + (proxy blip, deploy) multiplies that. Coalesce into one frame per + ~250–500 ms window. Do **not** restructure `seqMu`'s fan-out itself — per-client + FIFO ordering depends on it (see "What not to do"). +4. **Narrow permission-cache invalidation.** Role-scoped channel-override edits + call `InvalidateAll()` and then `RefreshChannelVisibility`, repopulating + ~2×N entries synchronously inside the admin request. Narrow to the affected + role's users — the per-user endpoints already use `InvalidateUser` with + exactly this rationale (`Server/admin/handlers_channel_perms.go`). +5. **Read-state write short-circuit.** `channel_focus`/`mark_read` UPSERT the + read-state row on the single writer even when it is already correct + (`Server/service/channel.go`). Skip the write when `latestID` matches and + `mention_count` is 0; optionally debounce bursts. Same shape as the + session-touch throttle already in `Server/api/middleware.go`. +6. **Single-process file lock.** Take an exclusive flock on a `.lock` beside the + SQLite file and fail fast with a legible message (warn-and-continue if the + lock syscall errors — network filesystems). Process-local presence/replay + state assumes one process owns the DB; make that assumption enforced. +7. **Small DB items.** Make the replay ring (1000) and cold-replay cap (5000) + configurable; add `database.max_readers` with a sane bound; rewrite + `DeleteExpiredSessions` to be index-friendly (note: `expires_at` is stored in + RFC3339 `T` format — format the cutoff to match or migrate the data); gate + boot-time `ANALYZE` on schema change plus a cheap `PRAGMA optimize`; add a + table-driven test for the read/write SQL router (`isReadOnlySQL`). + +## Track 2 — Cheap seams for a multi-instance future + +Interfaces and documentation only. Nothing here builds distributed systems. + +1. **Storage backend interface.** `*storage.Storage` is threaded concretely + through ~9 handler signatures. Carve a consumer-side interface in `api/` + (repo precedent: `service.Store`). Note `Open` must return + `io.ReadSeekCloser` + size/modtime because both serve paths use + `http.ServeContent` — that constraint is exactly what makes an S3 backend + nontrivial, and discovering it now is the point. Do not implement S3. +2. **Split the admin CIDR list by purpose.** `/admin`, `/api/v1/metrics`, the + Prometheus exporter, and the LiveKit webhook all share `admin_allowed_cidrs`. + The webhook is already cryptographically authenticated; metrics scraping and + human admin access are different trust domains. Separate config keys so + moving one off-box never widens another. Cheapest seam for voice as a + separate scaling unit. +3. **Persist TOTP/partial-auth stores via the existing persister seam.** + `RateLimiter` already has the optional-persister shape; give + `UsedTOTPCodeStore`/`PartialAuthStore` the same (store hashes, not raw + codes). Do **not** persist the rate-limiter sliding windows — hottest path, + benefit only exists post-multi-instance. +4. **Document the fifth D8 blocker.** Boot-time presence/voice reset + (`Server/main.go`) is a single-instance assumption missing from D8's blocker + list. One doc bullet: "process-local presence/voice state, wiped and rebuilt + per process." + +## Track 3 — Ops hygiene + +The highest-impact track. Ordered. + +1. **Implement the backup scheduler and retention — or visibly disable the + controls.** `backup_schedule`/`backup_retention` exist in the settings table, + admin UI, and API docs, but nothing reads them; a fresh install shows + "Daily" selected and never backs up. Implement inside the existing 15-min + maintenance loop (retention is in days, per the UI), or grey the controls + out today. Do not build a general job scheduler. +2. **Enrich the default-build metrics surface.** `/api/v1/metrics` omits signals + already computed in memory: reconnect tier stats, event-persister stats, + writer-pool `WaitCount`/`WaitDuration` (the single most direct signal for the + single-writer bottleneck), aggregate per-client backpressure counters, and + permission-cache hit/miss. ~60 lines across ~4 files; do this before + touching the otel path. Also wire or delete the seven declared-but-never- + recorded OTel instruments in `Server/telemetry/metrics.go`. +3. **Make `/health` honest.** It returns a static "ok" — never checks the DB, + disk, or hub dispatch loop. Add a bounded `SELECT 1`, disk-free check, and + hub liveness; return 503 with a reason. Cache the result — the endpoint is + unauthenticated and rate-limit exempt. +4. **Boot-smoke release artifacts.** The release pipeline signs and publishes + server binaries and a Docker image it never executes. Boot each artifact + against a scratch dir, poll `/health`, kill it; gate signing/publishing on + that. This is the failure whose blast radius scales with adoption via + self-update. +5. **Bare-metal Linux posture.** Ship a systemd unit template (note: + `ProtectSystem=strict` breaks the self-updater unless the install dir is + writable; ACME needs `AmbientCapabilities=CAP_NET_BIND_SERVICE`; + `TimeoutStopSec=35` matches the 30s drain), a "Linux (systemd)" deployment + section, and a cron backup one-liner. Add a "Reverse Proxy Topology" section + with a working nginx snippet — and state correctly that LiveKit *signaling* + is already proxied at `/livekit/*`; only WebRTC media (UDP range / TCP + fallback) must be directly reachable. +6. **Backup robustness.** Make the backup directory configurable (mirror the + `SetDatabasePath` plumb), document an optional post-backup hook command for + off-host shipping (rsync/rclone left to the operator), remove the output + file on `VACUUM INTO` error, and run `PRAGMA integrity_check` before listing + a backup as restorable. Document that backups stall writes for their + duration and schedule them off-peak. No S3, no manifests. +7. **Fix the k6 load script, then publish one capacity number.** + `Server/scripts/k6/ws-load.js` predates the envelope protocol: auth fails on + the first frame, three of four message types are wrong, and the only + assertion checks HTTP 101 — a fully broken run reports green. Fix it, gate + VU-connected on `auth_ok`/`ready`, add a `workflow_dispatch`-only job, and + publish one reference sizing (connections vs p99 broadcast latency vs + CPU/RAM) naming the two real bottlenecks. Do not gate main CI on it. +8. **Config and admin-settings honesty.** Warn (never fail) on unknown config + keys — capture the koanf key set after the defaults layer as the allow-list + and diff a second instance loaded from the file. Remove or disable the five + admin-settings fields nothing reads (`server_icon`, `backup_schedule`, + `backup_retention`, `max_upload_bytes`, `voice_quality`) and document which + require config.yaml + restart. Add a boot-time disk-free warning and a + metric (needs a build-tagged Windows path). Return 507/503, not 400, when + upload storage fails at the OS level. +9. **CI/release polish.** Add a `concurrency` group to `release.yml` + (`cancel-in-progress: false`); move `client-check`/`client-tests` off + windows-latest or write down why they are there; record a graduation + criterion for the non-blocking admin-e2e job; record the macOS scope + decision near D8 rather than adding an unsigned build. Add the Tailscale + CGNAT range note to `docs/tailscale.md` (admin routes 403 by default from + `100.x.y.z` addresses). + +## What not to do + +- Do not decompose the hub's `seqMu` fan-out or make its queue sizes tunable — + per-client FIFO ordering depends on the current structure + (`Server/ws/hub_broadcast.go`, `Server/ws/CLAUDE.md`). Architecture fix or + nothing; never a dial. +- Do not derive broadcast audience from pubsub subscribers — + `hub_broadcast.go` documents why that was rejected. +- Do not build S3, a job scheduler, cloud backup shipping, disk-based admission + control, adaptive connection limits, or settings hot-reload. +- Do not persist rate-limiter sliding windows or carve a pub/sub-slash-replay + interface — speculative abstraction over the most delicate code in the repo. +- Do not publish capacity numbers before the k6 script is fixed. +- Do not attempt hub self-recovery after the panic breaker trips. + +## Suggested sequencing + +1. Track 3 #1 (backup scheduler) — the current UI state misleads operators. +2. Track 3 #2 + #3 (metrics + health) — everything in Track 1 is guesswork + without these signals. +3. Track 3 #4 (release boot-smoke) — blast radius scales with adoption. +4. Track 3 #5 + #6 (systemd + proxy docs + backup dir) — one coherent + bare-metal pass; also the prerequisite for the hub breaker `os.Exit(1)`. +5. Track 1 #1 + #2 (hub liveness + connection ceiling) — small, self-contained, + no locking changes. + +Follow-up review passes suggested where this one was thin: release-path supply +chain, TLS/ACME renewal failure handling and secrets at rest, and voice/LiveKit +failure and scaling behavior. diff --git a/docs/server-configuration.md b/docs/server-configuration.md index 0ef90705..ce2567ae 100644 --- a/docs/server-configuration.md +++ b/docs/server-configuration.md @@ -33,6 +33,9 @@ the server automatically when a startup-only value changed. Note that | `server.port` | int | `8443` | HTTP(S) listen port | | `server.name` | string | `"OwnCord Server"` | Server display name (shown in `/api/v1/info` and admin panel) | | `server.data_dir` | string | `"data"` | Directory for database, certs, uploads, backups | +| `server.max_ws_connections` | int | `0` | Cap on concurrently connected WebSocket clients; further upgrades get 503 until connections free up. `0` = unlimited. Every connection costs goroutines and buffered send queues — set a ceiling that matches the host's memory before opening the server to a large community. | +| `server.metrics_allowed_cidrs` | []string | `[]` | Separate allowlist for `/api/v1/metrics` and the Prometheus `/metrics` exporter, so a central scraper can be admitted without widening `/admin` to its network. Empty = falls back to `admin_allowed_cidrs`. | +| `server.livekit_webhook_allowed_cidrs` | []string | `[]` | Separate allowlist for the LiveKit webhook/health endpoints (which also authenticate cryptographically) — an externally-hosted LiveKit's IP goes here, not in the admin allowlist. Empty = falls back to `admin_allowed_cidrs`. | | `server.allowed_origins` | string[] | `[]` | WebSocket CORS allowed origins for **web/browser** clients; empty list DENIES all cross-origin (set to `["*"]` to allow any origin). The OwnCord desktop client needs no entry here — its webview origins (`http(s)://tauri.localhost`, `tauri://localhost`) are always accepted. | | `server.trusted_proxies` | string[] | `[]` | CIDRs of trusted reverse proxies (for X-Forwarded-For) | | `server.admin_allowed_cidrs` | string[] | private networks | CIDRs allowed to access `/admin` routes. Default: `127.0.0.0/8`, `::1/128`, `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `fc00::/7` | @@ -56,6 +59,19 @@ the server automatically when a startup-only value changed. Note that |-----|------|---------|-------------| | `database.type` | string | `"sqlite"` | Database backend. `sqlite` (or empty) is the only supported value — any other value makes the server refuse to start. | | `database.path` | string | `"data/chatserver.db"` | Path to SQLite database file | +| `database.max_readers` | int | `0` | Bound on the read-only connection pool. `0` = automatic (`max(4, CPU count)`); clamped to 1–64. Readers beyond the CPU count mostly buy queueing, not throughput. | + +### Backups (`backup`) + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `backup.dir` | string | `"data/backups"` | Directory where database backups are written and pruned. Point it at another disk or an off-host mount so backups don't share a single point of failure with the live database. The admin panel's Backup Schedule and Retention settings operate on this directory. | + +### Security (`security`) + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `security.auth_rate_limit_multiplier` | float | `1.0` | Scales the per-IP auth rate limits and failure thresholds (registration, login, TOTP, sensitive endpoints). The defaults assume roughly one person per IP; raise this for communities behind a shared NAT (office, school). Clamped to 0.1–100. | ### Uploads (`upload`) @@ -96,7 +112,7 @@ For LiveKit options OwnCord does not model, you can take ownership of the auto-s ### Event Persistence (`event_persistence`) -Controls the tiered event log used for WebSocket reconnection replay. When enabled, missed events are stored in the database so clients that reconnect after the in-memory ring buffer window (1 000 events) can still replay missed events from the DB tier. +Controls the tiered event log used for WebSocket reconnection replay. When enabled, missed events are stored in the database so clients that reconnect after the in-memory ring buffer window (`replay_ring_size` events) can still replay missed events from the DB tier. | Key | Type | Default | Description | |-----|------|---------|-------------| @@ -105,6 +121,8 @@ Controls the tiered event log used for WebSocket reconnection replay. When enabl | `event_persistence.batch_size` | int | `50` | Maximum events per database flush | | `event_persistence.batch_flush_ms` | int | `100` | Maximum delay between flushes (milliseconds) | | `event_persistence.pruner_interval_minutes` | int | `60` | How often the pruner goroutine wakes up to delete expired events | +| `event_persistence.replay_ring_size` | int | `1000` | Capacity of the in-memory reconnect replay ring. Larger rings bridge longer disconnects without touching the database, at ~1 message payload of memory per slot. | +| `event_persistence.replay_cold_limit` | int | `5000` | Maximum persisted events a single reconnect may replay; a larger gap falls back to a full resync. Watch the `reconnect_tier_full` metric before raising it. | ### Telemetry / OpenTelemetry (`telemetry`) diff --git a/docs/tailscale.md b/docs/tailscale.md index 76fc09af..cedffd00 100644 --- a/docs/tailscale.md +++ b/docs/tailscale.md @@ -15,6 +15,20 @@ It works behind CGNAT and strict home routers, so setup is usually faster than m 4. Keep OwnCord on port `8443`. 5. Connect clients to `https://:8443`. +> **Admin panel over Tailscale:** chat works out of the box, but `/admin`, +> `/api/v1/metrics`, and the LiveKit health/webhook routes are gated by +> `server.admin_allowed_cidrs`, whose default covers only loopback and +> RFC1918 private ranges — Tailscale's `100.x.y.z` addresses (CGNAT range +> `100.64.0.0/10`) are **not** included and will get a 403. To administer +> over the tailnet, add it to your `config.yaml`: +> +> ```yaml +> server: +> admin_allowed_cidrs: +> - "127.0.0.0/8" +> - "100.64.0.0/10" # Tailscale tailnet +> ``` + ## TLS Recommendation - Recommended: keep `tls.mode: self_signed` (default).