Files
OwnCord/Server/restart_test.go
T
J3vbandClaude Fable 5 6a26f2a839 fix(server): drain fully before the self-update/restore restart handoff (#1380)
* feat(server): supervisor detection and server.restart_mode config key

RunningUnderSupervisor detects systemd (INVOCATION_ID) and, best-effort,
NSSM (NSSM_SERVICE_NAME — 2.24 does not set it, so NSSM deployments set
the mode explicitly). server.restart_mode (auto|spawn|supervised, default
auto, env OWNCORD_SERVER_RESTART_MODE) selects how a self-restart hands
off after the server drains: exit for the supervisor to relaunch, or
spawn the replacement directly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp

* fix(server): make the self-restart handoff drain fully before starting the successor

The update/restore/wizard restart previously spawned the replacement
while the old server was still serving, then SIGTERMed itself and
hard-exited after 10s. That design failed in every documented deployment
mode: under the shipped systemd unit the spawned child (same cgroup) was
killed when the old main process exited and Restart=on-failure never
relaunched a clean exit; on Windows the self-SIGTERM is unsupported and
silently dropped, so graceful shutdown never ran — hub.GracefulStop (the
only caller of LiveKitProcess.Stop) was skipped, orphaning livekit-server
on TCP 7880/UDP 50000-60000 and dropping queued event/audit batches; and
NSSM's relaunch raced the self-spawned replacement for the database lock.

Admin handlers now perform only the on-disk swap and request a restart
through an injected hook (admin.SetRestartHandoff). The main package's
restart coordinator cancels the parent of run()'s signal.NotifyContext —
the exact drain a SIGTERM triggers, on every platform — and after run()
has fully torn down (listeners closed, hub and LiveKit stopped, queues
flushed, DB closed and its lock released) main() performs the handoff:
spawn the replacement in spawn mode, or exit 0 for the supervisor in
supervised mode. A 90s backstop force-exits a wedged teardown; the
DB-lock and bind retries demote to safety nets.

A three-state guard (idle/busy/restart-pending) serializes update apply,
backup restore, and setup-wizard restarts against each other: concurrent
applies no longer race the same staged .new file or broadcast a spurious
update_aborted, and conflicting requests get 409 UPDATE_IN_PROGRESS /
RESTART_PENDING. The swap being free of process side effects also makes
the apply success path unit-testable for the first time.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp

* fix(server): errno-based bind-conflict detection, ACME bind retry, LiveKit Pdeathsig

isAddrInUse now unwraps to the platform errno (EADDRINUSE; WSAEADDRINUSE
10048 on Windows) with the English strings kept only as fallback — the
string-only match never fired on localized Windows, silently disabling
the bind retry. The retry loop is extracted into serveWithBindRetry and
now also covers the ACME :80 challenge server, which previously gave up
on first conflict and stayed dead (breaking HTTP-01 renewals) until the
next restart. The .old-binary boot cleanup retries briefly for the
window where a spawn-mode predecessor has not fully exited. The
companion livekit-server gets Pdeathsig SIGKILL on Linux so a parent
killed without teardown (kill -9, OOM, backstop exit) cannot orphan it
with the voice ports held.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp

* docs(deploy): Restart=always unit and per-supervisor restart-mode guidance

Restart=always is what lets the deliberate clean exit after a
self-update/restore relaunch under systemd (systemctl stop is never
auto-restarted; failure exits behave as before). Deployment docs gain
the required NSSM AppEnvironmentExtra line, the Task Scheduler and
Docker restart-policy notes, and the new drain-then-handoff update flow.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp

---------

Co-authored-by: Claude <noreply@anthropic.com>
2026-08-16 08:25:40 +02:00

261 lines
7.8 KiB
Go

package main
import (
"fmt"
"io"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"testing"
"time"
"go.uber.org/goleak"
"github.com/owncord/server/admin"
)
func TestRestartCoordinator_RequestIdempotent(t *testing.T) {
rc := newRestartCoordinator(time.Hour, nil)
defer rc.disarm()
if reason, ok := rc.Requested(); ok || reason != "" {
t.Fatalf("fresh coordinator Requested() = %q, %v; want none", reason, ok)
}
rc.Request("first")
rc.Request("second")
reason, ok := rc.Requested()
if !ok || reason != "first" {
t.Errorf("Requested() = %q, %v; want first reason to win", reason, ok)
}
select {
case <-rc.Context().Done():
default:
t.Error("coordinator context not cancelled after Request")
}
}
// Cancelling the coordinator's context must drive a signal.NotifyContext
// built on top of it — that is the whole mechanism by which a restart
// request drains run() exactly like a SIGTERM, on every platform.
func TestRestartCoordinator_CancelDrivesNotifyContext(t *testing.T) {
rc := newRestartCoordinator(time.Hour, nil)
defer rc.disarm()
ctx, stop := signal.NotifyContext(rc.Context(), os.Interrupt)
defer stop()
rc.Request("test")
select {
case <-ctx.Done():
case <-time.After(2 * time.Second):
t.Fatal("NotifyContext not done after coordinator Request — restart requests would never drain the server")
}
}
func TestRestartCoordinator_Backstop(t *testing.T) {
t.Run("fires after the delay", func(t *testing.T) {
fired := make(chan struct{})
rc := newRestartCoordinator(10*time.Millisecond, func() { close(fired) })
rc.Request("test")
select {
case <-fired:
case <-time.After(2 * time.Second):
t.Fatal("backstop did not fire")
}
})
t.Run("disarm stops it", func(t *testing.T) {
fired := make(chan struct{})
rc := newRestartCoordinator(50*time.Millisecond, func() { close(fired) })
rc.Request("test")
rc.disarm()
select {
case <-fired:
t.Fatal("backstop fired after disarm")
case <-time.After(200 * time.Millisecond):
}
})
t.Run("not armed without a request", func(t *testing.T) {
fired := make(chan struct{})
_ = newRestartCoordinator(10*time.Millisecond, func() { close(fired) })
select {
case <-fired:
t.Fatal("backstop fired without a restart request")
case <-time.After(100 * time.Millisecond):
}
})
}
func TestResolveRestartMode(t *testing.T) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
// Pin every detector input: CI runners themselves can execute under
// systemd and carry a real INVOCATION_ID into the test process.
// present-but-empty counts as unset for all three.
pinBare := func(t *testing.T) {
t.Setenv("INVOCATION_ID", "")
t.Setenv("NSSM_SERVICE_NAME", "")
t.Setenv("OWNCORD_CONTAINER", "0")
}
t.Run("explicit values win over detection", func(t *testing.T) {
t.Setenv("INVOCATION_ID", "detected-supervisor")
if got := resolveRestartMode("spawn", log); got != restartModeSpawn {
t.Errorf("resolveRestartMode(spawn) = %q under systemd, want spawn", got)
}
pinBare(t)
if got := resolveRestartMode("supervised", log); got != restartModeSupervised {
t.Errorf("resolveRestartMode(supervised) = %q on bare metal, want supervised", got)
}
})
t.Run("auto on bare metal spawns", func(t *testing.T) {
pinBare(t)
for _, v := range []string{"auto", "", "bogus-mode"} {
if got := resolveRestartMode(v, log); got != restartModeSpawn {
t.Errorf("resolveRestartMode(%q) = %q, want spawn", v, got)
}
}
})
t.Run("auto under systemd is supervised", func(t *testing.T) {
pinBare(t)
t.Setenv("INVOCATION_ID", "4a1f3b0e9c8d4e2f")
if got := resolveRestartMode("auto", log); got != restartModeSupervised {
t.Errorf("resolveRestartMode(auto) = %q, want supervised", got)
}
})
t.Run("auto in a container is supervised", func(t *testing.T) {
pinBare(t)
t.Setenv("OWNCORD_CONTAINER", "1")
if got := resolveRestartMode("auto", log); got != restartModeSupervised {
t.Errorf("resolveRestartMode(auto) = %q, want supervised (restore/wizard restarts in Docker rely on the restart policy)", got)
}
})
}
func TestPerformRestartHandoff(t *testing.T) {
log := slog.New(slog.NewTextHandler(io.Discard, nil))
type call struct {
exe string
args []string
}
var calls []call
prev := spawnReplacement
spawnReplacement = func(exePath string, args []string) error {
calls = append(calls, call{exePath, args})
return nil
}
defer func() { spawnReplacement = prev }()
performRestartHandoff("update", restartModeSupervised, log)
if len(calls) != 0 {
t.Fatalf("supervised handoff spawned a process: %+v — the supervisor owns the relaunch", calls)
}
performRestartHandoff("update", restartModeSpawn, log)
if len(calls) != 1 {
t.Fatalf("spawn handoff made %d spawn calls, want 1", len(calls))
}
if calls[0].exe == "" {
t.Error("spawn handoff passed an empty executable path")
}
// A failing spawn must be survivable (logged, no panic) — there is no
// hub left to notify at this point.
spawnReplacement = func(string, []string) error { return fmt.Errorf("injected spawn failure") }
performRestartHandoff("update", restartModeSpawn, log)
}
// TestRun_RestartRequest_DrainsCleanly drives run() end to end: boot on a
// real port, request a restart through the coordinator (exactly what an
// admin update/restore does via admin.SetRestartHandoff), and assert run()
// drains and returns nil with no leaked goroutines — the property main()'s
// post-run handoff depends on (DB closed and lock released, port free,
// LiveKit stopped) before it starts the successor.
func TestRun_RestartRequest_DrainsCleanly(t *testing.T) {
t.Chdir(t.TempDir())
// Grab a free port, release it, and hand it to run(). The tiny window
// where something else could take it is absorbed by run()'s bind retry.
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("probe listen: %v", err)
}
port := l.Addr().(*net.TCPAddr).Port
_ = l.Close()
t.Setenv("OWNCORD_SERVER_PORT", fmt.Sprint(port))
t.Setenv("OWNCORD_TLS_MODE", "off")
t.Setenv("OWNCORD_VOICE_AUTO_DOWNLOAD_LIVEKIT", "false") // keep the test offline
logBuf := admin.NewRingBuffer(64)
levelVar := new(slog.LevelVar)
log := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: levelVar}))
leakOpt := goleak.IgnoreCurrent()
rc := newRestartCoordinator(time.Hour, nil)
runErr := make(chan error, 1)
go func() { runErr <- run(log, logBuf, levelVar, rc) }()
// Wait for the server to actually serve before requesting the restart.
healthURL := fmt.Sprintf("http://127.0.0.1:%d/health", port)
up := false
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); {
resp, healthErr := http.Get(healthURL) //nolint:gosec // G107: loopback URL built from the test's own port
if healthErr == nil {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
up = true
break
}
select {
case err := <-runErr:
t.Fatalf("run() exited before serving: %v", err)
case <-time.After(50 * time.Millisecond):
}
}
if !up {
t.Fatal("server never became reachable on /health")
}
rc.Request("test-restart")
select {
case err := <-runErr:
if err != nil {
t.Fatalf("run() after restart request = %v, want nil (clean drain)", err)
}
case <-time.After(60 * time.Second):
t.Fatal("run() did not return after the restart request")
}
rc.disarm()
if reason, ok := rc.Requested(); !ok || reason != "test-restart" {
t.Errorf("Requested() = %q, %v; want the recorded restart", reason, ok)
}
// Everything must have drained: this is the guarantee that lets main()
// start the successor with zero lock/port contention.
if err := goleak.Find(leakOpt); err != nil {
t.Errorf("goroutines leaked after a restart-request drain: %v", err)
}
// The port must actually be free again.
relisten, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if err != nil {
t.Errorf("port still held after drain: %v", err)
} else {
_ = relisten.Close()
}
}