2026-03-21 10:08:44 +01:00
|
|
|
package api
|
|
|
|
|
|
|
|
|
|
import (
|
2026-04-03 23:18:06 +02:00
|
|
|
"context"
|
2026-08-15 20:50:47 +02:00
|
|
|
"database/sql"
|
2026-03-21 10:08:44 +01:00
|
|
|
"net/http"
|
|
|
|
|
"runtime"
|
|
|
|
|
"time"
|
|
|
|
|
)
|
|
|
|
|
|
2026-08-15 20:50:47 +02:00
|
|
|
// EventPersisterMetrics is the nested event-persistence block of ServerMetrics.
|
|
|
|
|
// Present only when event persistence is enabled.
|
|
|
|
|
type EventPersisterMetrics struct {
|
|
|
|
|
Persisted uint64 `json:"persisted"`
|
|
|
|
|
Dropped uint64 `json:"dropped"`
|
|
|
|
|
Flushes uint64 `json:"flushes"`
|
|
|
|
|
Errors uint64 `json:"errors"`
|
|
|
|
|
}
|
|
|
|
|
|
2026-03-21 10:08:44 +01:00
|
|
|
// ServerMetrics holds runtime metrics for the /api/v1/metrics endpoint.
|
2026-08-15 20:50:47 +02:00
|
|
|
// The shape is documented in docs/deployment.md — keep the two in sync.
|
2026-03-21 10:08:44 +01:00
|
|
|
type ServerMetrics struct {
|
|
|
|
|
Uptime string `json:"uptime"`
|
|
|
|
|
UptimeSeconds float64 `json:"uptime_seconds"`
|
|
|
|
|
GoRoutines int `json:"goroutines"`
|
|
|
|
|
HeapAllocMB float64 `json:"heap_alloc_mb"`
|
|
|
|
|
HeapSysMB float64 `json:"heap_sys_mb"`
|
|
|
|
|
NumGC uint32 `json:"num_gc"`
|
|
|
|
|
ConnectedUsers int `json:"connected_users"`
|
2026-03-29 00:51:54 +01:00
|
|
|
VoiceSessions int `json:"voice_sessions"`
|
2026-08-15 20:50:47 +02:00
|
|
|
// BroadcastDrops counts messages dropped because the hub-wide broadcast
|
|
|
|
|
// queue was full — a hub-level overload signal, distinct from the
|
|
|
|
|
// per-client backpressure counters below. Any nonzero growth here means
|
|
|
|
|
// sequenced events were lost before delivery and is worth alerting on.
|
|
|
|
|
BroadcastDrops uint64 `json:"broadcast_drops"`
|
|
|
|
|
LiveKitHealthy *bool `json:"livekit_healthy,omitempty"`
|
|
|
|
|
|
|
|
|
|
// Reconnect replay tier hits. A rising full-resync share means the replay
|
|
|
|
|
// budget (ring size / cold cap) is too small for observed disconnect gaps.
|
|
|
|
|
ReconnectTierBuffer uint64 `json:"reconnect_tier_buffer"`
|
|
|
|
|
ReconnectTierDB uint64 `json:"reconnect_tier_db"`
|
|
|
|
|
ReconnectTierFull uint64 `json:"reconnect_tier_full"`
|
|
|
|
|
|
|
|
|
|
// Per-client send-queue backpressure totals.
|
|
|
|
|
BackpressureQueueDisconnects uint64 `json:"backpressure_queue_disconnects"`
|
|
|
|
|
BackpressureHighFallbacks uint64 `json:"backpressure_high_fallbacks"`
|
|
|
|
|
BackpressureLowDrops uint64 `json:"backpressure_low_drops"`
|
|
|
|
|
|
|
|
|
|
// WSConnRejects counts upgrades refused by the max_ws_connections cap.
|
|
|
|
|
WSConnRejects uint64 `json:"ws_conn_rejects"`
|
|
|
|
|
|
|
|
|
|
// DiskFreeMB is free space on the data volume; omitted when unknown.
|
|
|
|
|
DiskFreeMB *float64 `json:"disk_free_mb,omitempty"`
|
|
|
|
|
|
|
|
|
|
// SQLite writer-pool saturation: time spent queueing for the single write
|
|
|
|
|
// connection. The most direct signal for the documented single-writer
|
|
|
|
|
// bottleneck.
|
|
|
|
|
DBWriterWaitCount int64 `json:"db_writer_wait_count"`
|
|
|
|
|
DBWriterWaitSeconds float64 `json:"db_writer_wait_seconds"`
|
|
|
|
|
|
|
|
|
|
// Permission cache effectiveness.
|
|
|
|
|
PermCacheHits uint64 `json:"perm_cache_hits"`
|
|
|
|
|
PermCacheMisses uint64 `json:"perm_cache_misses"`
|
|
|
|
|
|
|
|
|
|
EventPersister *EventPersisterMetrics `json:"event_persister,omitempty"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// MetricsSources provides the live data feeds for handleMetrics. Any nil
|
|
|
|
|
// field is skipped, leaving that metric at its zero value (or absent for
|
|
|
|
|
// pointer-typed output), so tests and partial wirings stay cheap.
|
|
|
|
|
type MetricsSources struct {
|
|
|
|
|
ConnectedUsers func() int
|
|
|
|
|
VoiceSessions func() int
|
|
|
|
|
BroadcastDrops func() uint64
|
|
|
|
|
LiveKitHealth func(context.Context) (bool, error)
|
|
|
|
|
ReconnectTiers func() (buffer, db, full uint64)
|
|
|
|
|
Backpressure func() (queueDisconnects, highFallbacks, lowDrops uint64)
|
|
|
|
|
ConnRejects func() uint64
|
|
|
|
|
PersisterStats func() (persisted, dropped, flushes, errs uint64, ok bool)
|
|
|
|
|
DBStats func() sql.DBStats // writer pool
|
|
|
|
|
PermCache func() (hits, misses uint64)
|
|
|
|
|
DiskFree func() (uint64, error)
|
2026-03-21 10:08:44 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// handleMetrics returns an HTTP handler that reports runtime server metrics.
|
2026-08-15 20:50:47 +02:00
|
|
|
func handleMetrics(src MetricsSources) http.HandlerFunc {
|
2026-03-21 10:08:44 +01:00
|
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
var m runtime.MemStats
|
|
|
|
|
runtime.ReadMemStats(&m)
|
|
|
|
|
|
|
|
|
|
uptime := time.Since(serverStartTime)
|
|
|
|
|
metrics := ServerMetrics{
|
2026-08-15 20:50:47 +02:00
|
|
|
Uptime: uptime.Truncate(time.Second).String(),
|
|
|
|
|
UptimeSeconds: uptime.Seconds(),
|
|
|
|
|
GoRoutines: runtime.NumGoroutine(),
|
|
|
|
|
HeapAllocMB: float64(m.HeapAlloc) / 1024 / 1024,
|
|
|
|
|
HeapSysMB: float64(m.HeapSys) / 1024 / 1024,
|
|
|
|
|
NumGC: m.NumGC,
|
2026-03-21 10:08:44 +01:00
|
|
|
}
|
|
|
|
|
|
2026-08-15 20:50:47 +02:00
|
|
|
if src.ConnectedUsers != nil {
|
|
|
|
|
metrics.ConnectedUsers = src.ConnectedUsers()
|
|
|
|
|
}
|
|
|
|
|
if src.VoiceSessions != nil {
|
|
|
|
|
metrics.VoiceSessions = src.VoiceSessions()
|
|
|
|
|
}
|
|
|
|
|
if src.BroadcastDrops != nil {
|
|
|
|
|
metrics.BroadcastDrops = src.BroadcastDrops()
|
|
|
|
|
}
|
|
|
|
|
if src.LiveKitHealth != nil {
|
|
|
|
|
healthy, _ := src.LiveKitHealth(r.Context())
|
2026-03-22 19:29:42 +01:00
|
|
|
metrics.LiveKitHealthy = &healthy
|
|
|
|
|
}
|
2026-08-15 20:50:47 +02:00
|
|
|
if src.ReconnectTiers != nil {
|
|
|
|
|
metrics.ReconnectTierBuffer, metrics.ReconnectTierDB, metrics.ReconnectTierFull = src.ReconnectTiers()
|
|
|
|
|
}
|
|
|
|
|
if src.Backpressure != nil {
|
|
|
|
|
metrics.BackpressureQueueDisconnects, metrics.BackpressureHighFallbacks, metrics.BackpressureLowDrops = src.Backpressure()
|
|
|
|
|
}
|
|
|
|
|
if src.PersisterStats != nil {
|
|
|
|
|
if persisted, dropped, flushes, errs, ok := src.PersisterStats(); ok {
|
|
|
|
|
metrics.EventPersister = &EventPersisterMetrics{
|
|
|
|
|
Persisted: persisted,
|
|
|
|
|
Dropped: dropped,
|
|
|
|
|
Flushes: flushes,
|
|
|
|
|
Errors: errs,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if src.DBStats != nil {
|
|
|
|
|
st := src.DBStats()
|
|
|
|
|
metrics.DBWriterWaitCount = st.WaitCount
|
|
|
|
|
metrics.DBWriterWaitSeconds = st.WaitDuration.Seconds()
|
|
|
|
|
}
|
|
|
|
|
if src.PermCache != nil {
|
|
|
|
|
metrics.PermCacheHits, metrics.PermCacheMisses = src.PermCache()
|
|
|
|
|
}
|
|
|
|
|
if src.ConnRejects != nil {
|
|
|
|
|
metrics.WSConnRejects = src.ConnRejects()
|
|
|
|
|
}
|
|
|
|
|
if src.DiskFree != nil {
|
|
|
|
|
if free, err := src.DiskFree(); err == nil {
|
|
|
|
|
mb := float64(free) / 1024 / 1024
|
|
|
|
|
metrics.DiskFreeMB = &mb
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-03-22 19:29:42 +01:00
|
|
|
|
2026-03-21 10:08:44 +01:00
|
|
|
writeJSON(w, http.StatusOK, metrics)
|
|
|
|
|
}
|
|
|
|
|
}
|