mirror of
https://github.com/J3vb/OwnCord.git
synced 2026-09-03 03:50:00 +03:00
* feat(server): supervisor detection and server.restart_mode config key RunningUnderSupervisor detects systemd (INVOCATION_ID) and, best-effort, NSSM (NSSM_SERVICE_NAME — 2.24 does not set it, so NSSM deployments set the mode explicitly). server.restart_mode (auto|spawn|supervised, default auto, env OWNCORD_SERVER_RESTART_MODE) selects how a self-restart hands off after the server drains: exit for the supervisor to relaunch, or spawn the replacement directly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * fix(server): make the self-restart handoff drain fully before starting the successor The update/restore/wizard restart previously spawned the replacement while the old server was still serving, then SIGTERMed itself and hard-exited after 10s. That design failed in every documented deployment mode: under the shipped systemd unit the spawned child (same cgroup) was killed when the old main process exited and Restart=on-failure never relaunched a clean exit; on Windows the self-SIGTERM is unsupported and silently dropped, so graceful shutdown never ran — hub.GracefulStop (the only caller of LiveKitProcess.Stop) was skipped, orphaning livekit-server on TCP 7880/UDP 50000-60000 and dropping queued event/audit batches; and NSSM's relaunch raced the self-spawned replacement for the database lock. Admin handlers now perform only the on-disk swap and request a restart through an injected hook (admin.SetRestartHandoff). The main package's restart coordinator cancels the parent of run()'s signal.NotifyContext — the exact drain a SIGTERM triggers, on every platform — and after run() has fully torn down (listeners closed, hub and LiveKit stopped, queues flushed, DB closed and its lock released) main() performs the handoff: spawn the replacement in spawn mode, or exit 0 for the supervisor in supervised mode. A 90s backstop force-exits a wedged teardown; the DB-lock and bind retries demote to safety nets. A three-state guard (idle/busy/restart-pending) serializes update apply, backup restore, and setup-wizard restarts against each other: concurrent applies no longer race the same staged .new file or broadcast a spurious update_aborted, and conflicting requests get 409 UPDATE_IN_PROGRESS / RESTART_PENDING. The swap being free of process side effects also makes the apply success path unit-testable for the first time. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * fix(server): errno-based bind-conflict detection, ACME bind retry, LiveKit Pdeathsig isAddrInUse now unwraps to the platform errno (EADDRINUSE; WSAEADDRINUSE 10048 on Windows) with the English strings kept only as fallback — the string-only match never fired on localized Windows, silently disabling the bind retry. The retry loop is extracted into serveWithBindRetry and now also covers the ACME :80 challenge server, which previously gave up on first conflict and stayed dead (breaking HTTP-01 renewals) until the next restart. The .old-binary boot cleanup retries briefly for the window where a spawn-mode predecessor has not fully exited. The companion livekit-server gets Pdeathsig SIGKILL on Linux so a parent killed without teardown (kill -9, OOM, backstop exit) cannot orphan it with the voice ports held. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp * docs(deploy): Restart=always unit and per-supervisor restart-mode guidance Restart=always is what lets the deliberate clean exit after a self-update/restore relaunch under systemd (systemctl stop is never auto-restarted; failure exits behave as before). Deployment docs gain the required NSSM AppEnvironmentExtra line, the Task Scheduler and Docker restart-policy notes, and the new drain-then-handoff update flow. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ngzj2Rx9UGC35uLHAfErMp --------- Co-authored-by: Claude <noreply@anthropic.com>
169 lines
6.5 KiB
Go
169 lines
6.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
"go.uber.org/goleak"
|
|
|
|
"github.com/owncord/server/admin"
|
|
"github.com/owncord/server/auth"
|
|
"github.com/owncord/server/db"
|
|
"github.com/owncord/server/ws"
|
|
)
|
|
|
|
// TestRun_ServeErrorReturn_StopsHubDispatchGoroutine pins OC-0027:
|
|
// hub.GracefulStop() (the only caller of LiveKitProcess.Stop(), and what
|
|
// closes the hub's dispatch goroutine) is a plain statement reached only on
|
|
// the graceful-shutdown path. The serve-error branch — `case err :=
|
|
// <-serveErr: ... return fmt.Errorf(...)` — returns from run() before ever
|
|
// reaching it, so the hub's `go hub.Run()` dispatch goroutine (started by
|
|
// api.NewRouter) is left running, and in production the companion
|
|
// livekit-server process it owns is left running with it.
|
|
//
|
|
// An out-of-range port fails the first listen attempt with an error that
|
|
// isAddrInUse does not recognize, so run() takes the servErr branch
|
|
// immediately instead of retrying for ~10s.
|
|
func TestRun_ServeErrorReturn_StopsHubDispatchGoroutine(t *testing.T) {
|
|
t.Chdir(t.TempDir())
|
|
|
|
t.Setenv("OWNCORD_SERVER_PORT", "99999") // out of range: immediate, non-retryable listen error
|
|
t.Setenv("OWNCORD_TLS_MODE", "off") // skip self-signed cert generation
|
|
t.Setenv("OWNCORD_VOICE_AUTO_DOWNLOAD_LIVEKIT", "false") // the generated default config.yaml turns this on; keep the test offline
|
|
|
|
logBuf := admin.NewRingBuffer(64)
|
|
levelVar := new(slog.LevelVar)
|
|
log := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: levelVar}))
|
|
|
|
leakOpt := goleak.IgnoreCurrent()
|
|
|
|
rc := newRestartCoordinator(time.Hour, nil)
|
|
if err := run(log, logBuf, levelVar, rc); err == nil {
|
|
t.Fatal("expected run() to return an error for an out-of-range port")
|
|
}
|
|
if _, requested := rc.Requested(); requested {
|
|
t.Error("no restart was requested, but the coordinator reports one")
|
|
}
|
|
|
|
// hub.Run's dispatch goroutine only exits once hub.stop is closed, which
|
|
// only happens inside hub.GracefulStop(). If run() returned without
|
|
// calling it, this goroutine is still alive here.
|
|
if err := goleak.Find(leakOpt); err != nil {
|
|
t.Fatalf("hub dispatch goroutine (and, in production, its LiveKit process) leaked after run() returned early: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestSeedHubReplayState_ForcesFullResyncForOfflineClient pins OC-0204:
|
|
// h.seq is persisted (events table) and restored at startup via SeedSeq, but
|
|
// its paired in-memory watermark (visibilityChangeSeq) always starts at 0 on
|
|
// a fresh process. mustFullResync short-circuits on `w > 0`, so without also
|
|
// forcing the watermark forward at startup, every client resuming from a
|
|
// last_seq at or before the just-restored max sails through mustFullResync
|
|
// and gets an ordinary tiered replay — even though a channel-visibility
|
|
// change made to it while offline (RefreshChannelVisibility,
|
|
// revokeUnreadableChannels) was sent only as a targeted, unsequenced message
|
|
// that was never persisted and can never be recovered by that replay.
|
|
//
|
|
// This seeds a DB with a contiguous run of persisted events (simulating a
|
|
// prior boot that reached seq 520), then calls seedHubReplayState exactly as
|
|
// run() does, then reconnects a client with last_seq=500 (<= the restored
|
|
// max) and asserts the resume is forced onto the full-ready tier. Before the
|
|
// fix, last_seq=500 converges via the ordinary DB cold-tier replay instead
|
|
// (the persisted run 501..520 is contiguous and complete), silently proving
|
|
// the bug: a resume that must be forced full sails through unforced.
|
|
func TestSeedHubReplayState_ForcesFullResyncForOfflineClient(t *testing.T) {
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("db.Open: %v", err)
|
|
}
|
|
defer database.Close() //nolint:errcheck
|
|
if err := db.Migrate(database); err != nil {
|
|
t.Fatalf("db.Migrate: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
// Simulate the prior boot: 20 persisted global (channel_id=0) events at
|
|
// seqs 501..520, contiguous and complete — exactly the shape that lets
|
|
// handleReconnect's DB-tier contiguity/tail checks succeed today.
|
|
for seq := int64(501); seq <= 520; seq++ {
|
|
payload := fmt.Appendf(nil, `{"seq":%d,"type":"broadcast"}`, seq)
|
|
if err := database.PersistEvent(ctx, seq, "broadcast", 0, payload); err != nil {
|
|
t.Fatalf("PersistEvent seq=%d: %v", seq, err)
|
|
}
|
|
}
|
|
|
|
userID, err := database.CreateUser(ctx, "seed-replay-user", "hash", 1)
|
|
if err != nil {
|
|
t.Fatalf("CreateUser: %v", err)
|
|
}
|
|
token, err := auth.GenerateToken()
|
|
if err != nil {
|
|
t.Fatalf("GenerateToken: %v", err)
|
|
}
|
|
if _, err := database.CreateSession(ctx, userID, auth.HashToken(token), "test", "127.0.0.1"); err != nil {
|
|
t.Fatalf("CreateSession: %v", err)
|
|
}
|
|
|
|
limiter := auth.NewRateLimiter()
|
|
hub := ws.NewHub(database, limiter, nil)
|
|
go hub.Run()
|
|
defer hub.Stop()
|
|
|
|
// The exact startup call run() makes once event persistence is enabled —
|
|
// no ring-buffer events are pushed, so a resuming client's replay can
|
|
// only be satisfied via the DB cold tier or forced full.
|
|
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
seedHubReplayState(ctx, hub, database, log)
|
|
hub.SetEventStore(database)
|
|
|
|
handler := ws.ServeWS(hub, database, []string{"*"}, 0)
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
|
|
wsURL := "ws" + strings.TrimPrefix(srv.URL, "http")
|
|
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
|
|
conn, dialResp, dialErr := websocket.Dial(dialCtx, wsURL, nil)
|
|
if dialResp != nil && dialResp.Body != nil {
|
|
_ = dialResp.Body.Close()
|
|
}
|
|
if dialErr != nil {
|
|
t.Fatalf("websocket.Dial: %v", dialErr)
|
|
}
|
|
defer func() { _ = conn.Close(websocket.StatusNormalClosure, "") }()
|
|
|
|
// last_seq=500 predates the restored max (520): a client whose sidebar
|
|
// missed a targeted visibility change while offline must be forced onto
|
|
// the full-ready path to converge.
|
|
authMsg := map[string]any{
|
|
"type": "auth",
|
|
"payload": map[string]any{
|
|
"token": token,
|
|
"last_seq": uint64(500),
|
|
},
|
|
}
|
|
raw, _ := json.Marshal(authMsg)
|
|
if err := conn.Write(dialCtx, websocket.MessageText, raw); err != nil {
|
|
t.Fatalf("write auth: %v", err)
|
|
}
|
|
if _, _, err := conn.Read(dialCtx); err != nil {
|
|
t.Fatalf("read handshake response: %v", err)
|
|
}
|
|
|
|
bufTier, dbTier, fullTier := hub.ReconnectTierStats()
|
|
if fullTier != 1 {
|
|
t.Fatalf("reconnect tiers (buffer=%d db=%d full=%d): want full=1 — a client resuming from before a restart-restored seq must be forced onto the full-ready path, since an offline visibility change is never recoverable by replay",
|
|
bufTier, dbTier, fullTier)
|
|
}
|
|
}
|