mirror of
https://github.com/J3vb/OwnCord.git
synced 2026-09-03 03:50:00 +03:00
* feat(server): supervisor detection and server.restart_mode config key RunningUnderSupervisor detects systemd (INVOCATION_ID) and, best-effort, NSSM (NSSM_SERVICE_NAME — 2.24 does not set it, so NSSM deployments set the mode explicitly). server.restart_mode (auto|spawn|supervised, default auto, env OWNCORD_SERVER_RESTART_MODE) selects how a self-restart hands off after the server drains: exit for the supervisor to relaunch, or spawn the replacement directly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * fix(server): make the self-restart handoff drain fully before starting the successor The update/restore/wizard restart previously spawned the replacement while the old server was still serving, then SIGTERMed itself and hard-exited after 10s. That design failed in every documented deployment mode: under the shipped systemd unit the spawned child (same cgroup) was killed when the old main process exited and Restart=on-failure never relaunched a clean exit; on Windows the self-SIGTERM is unsupported and silently dropped, so graceful shutdown never ran — hub.GracefulStop (the only caller of LiveKitProcess.Stop) was skipped, orphaning livekit-server on TCP 7880/UDP 50000-60000 and dropping queued event/audit batches; and NSSM's relaunch raced the self-spawned replacement for the database lock. Admin handlers now perform only the on-disk swap and request a restart through an injected hook (admin.SetRestartHandoff). The main package's restart coordinator cancels the parent of run()'s signal.NotifyContext — the exact drain a SIGTERM triggers, on every platform — and after run() has fully torn down (listeners closed, hub and LiveKit stopped, queues flushed, DB closed and its lock released) main() performs the handoff: spawn the replacement in spawn mode, or exit 0 for the supervisor in supervised mode. A 90s backstop force-exits a wedged teardown; the DB-lock and bind retries demote to safety nets. A three-state guard (idle/busy/restart-pending) serializes update apply, backup restore, and setup-wizard restarts against each other: concurrent applies no longer race the same staged .new file or broadcast a spurious update_aborted, and conflicting requests get 409 UPDATE_IN_PROGRESS / RESTART_PENDING. The swap being free of process side effects also makes the apply success path unit-testable for the first time. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * fix(server): errno-based bind-conflict detection, ACME bind retry, LiveKit Pdeathsig isAddrInUse now unwraps to the platform errno (EADDRINUSE; WSAEADDRINUSE 10048 on Windows) with the English strings kept only as fallback — the string-only match never fired on localized Windows, silently disabling the bind retry. The retry loop is extracted into serveWithBindRetry and now also covers the ACME :80 challenge server, which previously gave up on first conflict and stayed dead (breaking HTTP-01 renewals) until the next restart. The .old-binary boot cleanup retries briefly for the window where a spawn-mode predecessor has not fully exited. The companion livekit-server gets Pdeathsig SIGKILL on Linux so a parent killed without teardown (kill -9, OOM, backstop exit) cannot orphan it with the voice ports held. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * docs(deploy): Restart=always unit and per-supervisor restart-mode guidance Restart=always is what lets the deliberate clean exit after a self-update/restore relaunch under systemd (systemctl stop is never auto-restarted; failure exits behave as before). Deployment docs gain the required NSSM AppEnvironmentExtra line, the Task Scheduler and Docker restart-policy notes, and the new drain-then-handoff update flow. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp --------- Co-authored-by: Claude <noreply@anthropic.com>
261 lines
7.8 KiB
Go
261 lines
7.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/goleak"
|
|
|
|
"github.com/owncord/server/admin"
|
|
)
|
|
|
|
func TestRestartCoordinator_RequestIdempotent(t *testing.T) {
|
|
rc := newRestartCoordinator(time.Hour, nil)
|
|
defer rc.disarm()
|
|
|
|
if reason, ok := rc.Requested(); ok || reason != "" {
|
|
t.Fatalf("fresh coordinator Requested() = %q, %v; want none", reason, ok)
|
|
}
|
|
|
|
rc.Request("first")
|
|
rc.Request("second")
|
|
|
|
reason, ok := rc.Requested()
|
|
if !ok || reason != "first" {
|
|
t.Errorf("Requested() = %q, %v; want first reason to win", reason, ok)
|
|
}
|
|
select {
|
|
case <-rc.Context().Done():
|
|
default:
|
|
t.Error("coordinator context not cancelled after Request")
|
|
}
|
|
}
|
|
|
|
// Cancelling the coordinator's context must drive a signal.NotifyContext
|
|
// built on top of it — that is the whole mechanism by which a restart
|
|
// request drains run() exactly like a SIGTERM, on every platform.
|
|
func TestRestartCoordinator_CancelDrivesNotifyContext(t *testing.T) {
|
|
rc := newRestartCoordinator(time.Hour, nil)
|
|
defer rc.disarm()
|
|
|
|
ctx, stop := signal.NotifyContext(rc.Context(), os.Interrupt)
|
|
defer stop()
|
|
|
|
rc.Request("test")
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("NotifyContext not done after coordinator Request — restart requests would never drain the server")
|
|
}
|
|
}
|
|
|
|
func TestRestartCoordinator_Backstop(t *testing.T) {
|
|
t.Run("fires after the delay", func(t *testing.T) {
|
|
fired := make(chan struct{})
|
|
rc := newRestartCoordinator(10*time.Millisecond, func() { close(fired) })
|
|
rc.Request("test")
|
|
select {
|
|
case <-fired:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("backstop did not fire")
|
|
}
|
|
})
|
|
|
|
t.Run("disarm stops it", func(t *testing.T) {
|
|
fired := make(chan struct{})
|
|
rc := newRestartCoordinator(50*time.Millisecond, func() { close(fired) })
|
|
rc.Request("test")
|
|
rc.disarm()
|
|
select {
|
|
case <-fired:
|
|
t.Fatal("backstop fired after disarm")
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
})
|
|
|
|
t.Run("not armed without a request", func(t *testing.T) {
|
|
fired := make(chan struct{})
|
|
_ = newRestartCoordinator(10*time.Millisecond, func() { close(fired) })
|
|
select {
|
|
case <-fired:
|
|
t.Fatal("backstop fired without a restart request")
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestResolveRestartMode(t *testing.T) {
|
|
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
|
|
// Pin every detector input: CI runners themselves can execute under
|
|
// systemd and carry a real INVOCATION_ID into the test process.
|
|
// present-but-empty counts as unset for all three.
|
|
pinBare := func(t *testing.T) {
|
|
t.Setenv("INVOCATION_ID", "")
|
|
t.Setenv("NSSM_SERVICE_NAME", "")
|
|
t.Setenv("OWNCORD_CONTAINER", "0")
|
|
}
|
|
|
|
t.Run("explicit values win over detection", func(t *testing.T) {
|
|
t.Setenv("INVOCATION_ID", "detected-supervisor")
|
|
if got := resolveRestartMode("spawn", log); got != restartModeSpawn {
|
|
t.Errorf("resolveRestartMode(spawn) = %q under systemd, want spawn", got)
|
|
}
|
|
pinBare(t)
|
|
if got := resolveRestartMode("supervised", log); got != restartModeSupervised {
|
|
t.Errorf("resolveRestartMode(supervised) = %q on bare metal, want supervised", got)
|
|
}
|
|
})
|
|
|
|
t.Run("auto on bare metal spawns", func(t *testing.T) {
|
|
pinBare(t)
|
|
for _, v := range []string{"auto", "", "bogus-mode"} {
|
|
if got := resolveRestartMode(v, log); got != restartModeSpawn {
|
|
t.Errorf("resolveRestartMode(%q) = %q, want spawn", v, got)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("auto under systemd is supervised", func(t *testing.T) {
|
|
pinBare(t)
|
|
t.Setenv("INVOCATION_ID", "4a1f3b0e9c8d4e2f")
|
|
if got := resolveRestartMode("auto", log); got != restartModeSupervised {
|
|
t.Errorf("resolveRestartMode(auto) = %q, want supervised", got)
|
|
}
|
|
})
|
|
|
|
t.Run("auto in a container is supervised", func(t *testing.T) {
|
|
pinBare(t)
|
|
t.Setenv("OWNCORD_CONTAINER", "1")
|
|
if got := resolveRestartMode("auto", log); got != restartModeSupervised {
|
|
t.Errorf("resolveRestartMode(auto) = %q, want supervised (restore/wizard restarts in Docker rely on the restart policy)", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPerformRestartHandoff(t *testing.T) {
|
|
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
|
|
type call struct {
|
|
exe string
|
|
args []string
|
|
}
|
|
var calls []call
|
|
prev := spawnReplacement
|
|
spawnReplacement = func(exePath string, args []string) error {
|
|
calls = append(calls, call{exePath, args})
|
|
return nil
|
|
}
|
|
defer func() { spawnReplacement = prev }()
|
|
|
|
performRestartHandoff("update", restartModeSupervised, log)
|
|
if len(calls) != 0 {
|
|
t.Fatalf("supervised handoff spawned a process: %+v — the supervisor owns the relaunch", calls)
|
|
}
|
|
|
|
performRestartHandoff("update", restartModeSpawn, log)
|
|
if len(calls) != 1 {
|
|
t.Fatalf("spawn handoff made %d spawn calls, want 1", len(calls))
|
|
}
|
|
if calls[0].exe == "" {
|
|
t.Error("spawn handoff passed an empty executable path")
|
|
}
|
|
|
|
// A failing spawn must be survivable (logged, no panic) — there is no
|
|
// hub left to notify at this point.
|
|
spawnReplacement = func(string, []string) error { return fmt.Errorf("injected spawn failure") }
|
|
performRestartHandoff("update", restartModeSpawn, log)
|
|
}
|
|
|
|
// TestRun_RestartRequest_DrainsCleanly drives run() end to end: boot on a
|
|
// real port, request a restart through the coordinator (exactly what an
|
|
// admin update/restore does via admin.SetRestartHandoff), and assert run()
|
|
// drains and returns nil with no leaked goroutines — the property main()'s
|
|
// post-run handoff depends on (DB closed and lock released, port free,
|
|
// LiveKit stopped) before it starts the successor.
|
|
func TestRun_RestartRequest_DrainsCleanly(t *testing.T) {
|
|
t.Chdir(t.TempDir())
|
|
|
|
// Grab a free port, release it, and hand it to run(). The tiny window
|
|
// where something else could take it is absorbed by run()'s bind retry.
|
|
l, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("probe listen: %v", err)
|
|
}
|
|
port := l.Addr().(*net.TCPAddr).Port
|
|
_ = l.Close()
|
|
|
|
t.Setenv("OWNCORD_SERVER_PORT", fmt.Sprint(port))
|
|
t.Setenv("OWNCORD_TLS_MODE", "off")
|
|
t.Setenv("OWNCORD_VOICE_AUTO_DOWNLOAD_LIVEKIT", "false") // keep the test offline
|
|
|
|
logBuf := admin.NewRingBuffer(64)
|
|
levelVar := new(slog.LevelVar)
|
|
log := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: levelVar}))
|
|
|
|
leakOpt := goleak.IgnoreCurrent()
|
|
|
|
rc := newRestartCoordinator(time.Hour, nil)
|
|
runErr := make(chan error, 1)
|
|
go func() { runErr <- run(log, logBuf, levelVar, rc) }()
|
|
|
|
// Wait for the server to actually serve before requesting the restart.
|
|
healthURL := fmt.Sprintf("http://127.0.0.1:%d/health", port)
|
|
up := false
|
|
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); {
|
|
resp, healthErr := http.Get(healthURL) //nolint:gosec // G107: loopback URL built from the test's own port
|
|
if healthErr == nil {
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
_ = resp.Body.Close()
|
|
up = true
|
|
break
|
|
}
|
|
select {
|
|
case err := <-runErr:
|
|
t.Fatalf("run() exited before serving: %v", err)
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
}
|
|
if !up {
|
|
t.Fatal("server never became reachable on /health")
|
|
}
|
|
|
|
rc.Request("test-restart")
|
|
|
|
select {
|
|
case err := <-runErr:
|
|
if err != nil {
|
|
t.Fatalf("run() after restart request = %v, want nil (clean drain)", err)
|
|
}
|
|
case <-time.After(60 * time.Second):
|
|
t.Fatal("run() did not return after the restart request")
|
|
}
|
|
rc.disarm()
|
|
|
|
if reason, ok := rc.Requested(); !ok || reason != "test-restart" {
|
|
t.Errorf("Requested() = %q, %v; want the recorded restart", reason, ok)
|
|
}
|
|
|
|
// Everything must have drained: this is the guarantee that lets main()
|
|
// start the successor with zero lock/port contention.
|
|
if err := goleak.Find(leakOpt); err != nil {
|
|
t.Errorf("goroutines leaked after a restart-request drain: %v", err)
|
|
}
|
|
|
|
// The port must actually be free again.
|
|
relisten, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
|
|
if err != nil {
|
|
t.Errorf("port still held after drain: %v", err)
|
|
} else {
|
|
_ = relisten.Close()
|
|
}
|
|
}
|