implement full three-tier priority queue system

Client now has three send channels:
- sendHigh (64 slots): DMs, mentions — drained first by writePump
- send (256 slots): chat messages, reactions — drained second
- sendLow (64 slots): typing, presence — drained last, dropped on overflow

writePump drains high-priority messages before checking normal/low.
PubSub gains PublishHigh/PublishLow alongside existing Publish.
EmitEvents routes events by priority:
- High: SequencedDMEvent, UserTargetedEvent
- Normal: ChannelEvent, VoiceChannelEvent
- Low: ExcludeSenderEvent (typing), PresenceEvent

Slow clients get typing/presence dropped first (sendLowMsg silently
drops), then disconnect on normal buffer overflow, ensuring DMs are
never lost to typing indicator backpressure.

https://claude.ai/code/session_01CBFF3r84ywkJRWwuqw8zD8
This commit is contained in:
Claude
2026-04-05 21:12:08 +00:00
parent c2bf9304e4
commit 1bf3ca5de3
7 changed files with 268 additions and 76 deletions
+81 -16
View File
@@ -9,7 +9,11 @@ import (
"github.com/owncord/server/syncutil"
)
const sendBufSize = 256 // per-client outbound send-channel capacity
const (
sendBufSize = 256 // per-client outbound send-channel capacity (normal priority)
sendHighBufSize = 64 // high-priority buffer (DMs, mentions)
sendLowBufSize = 64 // low-priority buffer (typing, presence)
)
// SessionCheckInterval is the number of messages processed between periodic
// session-expiry checks in readPump. Exported so tests can trigger the check
@@ -39,8 +43,10 @@ type Client struct {
msgsDropped int64 // messages dropped due to full send buffer
invalidCount int // consecutive invalid messages; reset on valid parse
lastActivity time.Time // last message received from this client; guarded by mu
sendClosed bool // true after the send channel has been closed
send chan []byte
sendClosed bool // true after all send channels have been closed
send chan []byte // normal-priority outbound messages (chat messages, reactions)
sendHigh chan []byte // high-priority outbound messages (DMs, mentions)
sendLow chan []byte // low-priority outbound messages (typing, presence) — dropped on overflow
mu syncutil.Mutex // guards sendClosed, msgCount, channelID, lastActivity, msgsReceived, msgsSent, msgsDropped
voiceMu syncutil.Mutex // guards voiceChID and voiceJoinToken
}
@@ -66,6 +72,8 @@ func newClient(hub *Hub, conn wsConn, user *db.User, tokenHash string, lastSeq u
connectedAt: now,
lastActivity: now,
send: make(chan []byte, sendBufSize),
sendHigh: make(chan []byte, sendHighBufSize),
sendLow: make(chan []byte, sendLowBufSize),
}
}
@@ -79,10 +87,12 @@ func (c *Client) GetTokenHash() string {
// Intended for unit tests only — conn is nil.
func NewTestClient(hub *Hub, userID int64, send chan []byte) *Client {
return &Client{
hub: hub,
ctx: context.Background(),
userID: userID,
send: send,
hub: hub,
ctx: context.Background(),
userID: userID,
send: send,
sendHigh: make(chan []byte, sendHighBufSize),
sendLow: make(chan []byte, sendLowBufSize),
}
}
@@ -94,6 +104,8 @@ func NewTestClientWithChannel(hub *Hub, userID, channelID int64, send chan []byt
userID: userID,
channelID: channelID,
send: send,
sendHigh: make(chan []byte, sendHighBufSize),
sendLow: make(chan []byte, sendLowBufSize),
}
}
@@ -107,6 +119,8 @@ func NewTestClientWithUser(hub *Hub, user *db.User, channelID int64, send chan [
user: user,
channelID: channelID,
send: send,
sendHigh: make(chan []byte, sendHighBufSize),
sendLow: make(chan []byte, sendLowBufSize),
}
}
@@ -150,6 +164,8 @@ func NewTestClientWithTokenHash(hub *Hub, user *db.User, tokenHash string, chann
tokenHash: tokenHash,
channelID: channelID,
send: send,
sendHigh: make(chan []byte, sendHighBufSize),
sendLow: make(chan []byte, sendLowBufSize),
}
}
@@ -242,8 +258,7 @@ func (c *Client) getE2EEPubKey() string {
return c.e2eePubKey
}
// sendMsg queues a message to this client's send buffer without blocking.
// It is a no-op if the send channel has already been closed.
// sendMsg queues a normal-priority message (chat messages, reactions, channel events).
// If the buffer is full, the client is disconnected to force a reconnect
// with replay recovery instead of silently losing messages (BUG-124).
func (c *Client) sendMsg(msg []byte) {
@@ -259,13 +274,57 @@ func (c *Client) sendMsg(msg []byte) {
c.msgsDropped++
slog.Warn("ws: client send buffer full, closing connection to force reconnect",
"user_id", c.userID)
c.sendClosed = true
close(c.send)
c.closeAllSendLocked()
}
}
// trySendMsg queues a message and returns true if it was accepted, false if
// the buffer is full or the channel is closed.
// sendHighMsg queues a high-priority message (DMs, direct mentions).
// High-priority messages are drained before normal and low-priority messages
// by writePump. If the high-priority buffer is full, falls back to the normal
// buffer. If both are full, disconnects the client.
func (c *Client) sendHighMsg(msg []byte) {
c.mu.Lock()
defer c.mu.Unlock()
if c.sendClosed {
return
}
select {
case c.sendHigh <- msg:
c.msgsSent++
default:
// Fall back to normal priority channel.
select {
case c.send <- msg:
c.msgsSent++
default:
c.msgsDropped++
slog.Warn("ws: client high+normal buffers full, closing connection",
"user_id", c.userID)
c.closeAllSendLocked()
}
}
}
// sendLowMsg queues a low-priority message (typing indicators, presence updates).
// If the buffer is full the message is silently dropped — the client is NOT
// disconnected, since these events are ephemeral and can be safely lost.
func (c *Client) sendLowMsg(msg []byte) {
c.mu.Lock()
defer c.mu.Unlock()
if c.sendClosed {
return
}
select {
case c.sendLow <- msg:
c.msgsSent++
default:
c.msgsDropped++
// Do NOT disconnect — low-priority messages are safely droppable.
}
}
// trySendMsg queues a normal-priority message and returns true if it was
// accepted, false if the buffer is full or the channel is closed.
// On buffer overflow, the client is disconnected to force a reconnect (BUG-124).
func (c *Client) trySendMsg(msg []byte) bool {
c.mu.Lock()
@@ -281,19 +340,25 @@ func (c *Client) trySendMsg(msg []byte) bool {
c.msgsDropped++
slog.Warn("ws: client send buffer full (trySend), closing connection to force reconnect",
"user_id", c.userID)
c.sendClosed = true
close(c.send)
c.closeAllSendLocked()
return false
}
}
// closeSend marks the send channel closed and closes it exactly once.
// closeSend marks all send channels closed and closes them exactly once.
// Safe to call from any goroutine.
func (c *Client) closeSend() {
c.mu.Lock()
defer c.mu.Unlock()
c.closeAllSendLocked()
}
// closeAllSendLocked closes all three send channels. Caller must hold c.mu.
func (c *Client) closeAllSendLocked() {
if !c.sendClosed {
c.sendClosed = true
close(c.send)
close(c.sendHigh)
close(c.sendLow)
}
}
+12 -4
View File
@@ -19,17 +19,25 @@ func (h *Hub) EmitEvents(events []Event) {
for _, ev := range events {
switch e := ev.(type) {
case SequencedDMEvent:
h.sendSequencedToUsers(e.ChannelID(), e.ParticipantIDs(), e.Payload())
// High priority: DMs are time-sensitive.
h.sendSequencedToUsersHigh(e.ChannelID(), e.ParticipantIDs(), e.Payload())
case VoiceChannelGuardedEvent:
h.sendToUserIfInVoiceChannel(e.VoiceChannelID(), e.TargetUserID(), e.Payload())
case VoiceChannelEvent:
h.sendToVoiceChannelExcept(e.VoiceChannelID(), e.ExcludeUserID(), e.Payload())
case ExcludeSenderEvent:
h.broadcastExclude(e.ChannelID(), e.ExcludeUserID(), e.Payload())
// Low priority: typing indicators are ephemeral.
h.broadcastExcludeLow(e.ChannelID(), e.ExcludeUserID(), e.Payload())
case UserTargetedEvent:
h.SendToUser(e.TargetUserID(), e.Payload())
// High priority: targeted events (DM opens, mentions).
h.SendToUserHigh(e.TargetUserID(), e.Payload())
case BroadcastAllEvent:
h.BroadcastToAll(e.Payload())
// Check concrete type: presence is low-priority, others are normal.
if _, isPresence := ev.(PresenceEvent); isPresence {
h.BroadcastToAllLow(e.Payload())
} else {
h.BroadcastToAll(e.Payload())
}
case ChannelEvent:
h.BroadcastToChannel(e.ChannelID(), e.Payload())
default:
+10 -2
View File
@@ -233,14 +233,22 @@ func (h *Hub) requireChannelPerm(c *Client, channelID int64, perm int64, permLab
// that should survive reconnection replay.
func (h *Hub) broadcastExclude(channelID, excludeUserID int64, msg []byte) {
if channelID == 0 {
// Global broadcast excluding one user — use the global topic.
h.pubsub.Publish(TopicGlobal, msg, excludeUserID)
return
}
// Channel-scoped broadcast excluding one user.
h.pubsub.Publish(ChannelTopic(channelID), msg, excludeUserID)
}
// broadcastExcludeLow is like broadcastExclude but at low priority.
// Used for typing indicators — dropped on overflow instead of disconnecting.
func (h *Hub) broadcastExcludeLow(channelID, excludeUserID int64, msg []byte) {
if channelID == 0 {
h.pubsub.PublishLow(TopicGlobal, msg, excludeUserID)
return
}
h.pubsub.PublishLow(ChannelTopic(channelID), msg, excludeUserID)
}
// broadcastToDMParticipants sends a message to all participants of a DM channel
// while preserving DM semantics (delivery is by participant, not channel focus).
// Unlike broadcastToDMParticipantsExclude, this path is sequenced and replayable.
+34
View File
@@ -512,6 +512,26 @@ func (h *Hub) SendToUser(userID int64, msg []byte) bool {
return c.trySendMsg(msg)
}
// SendToUserHigh sends a high-priority message to a specific user.
func (h *Hub) SendToUserHigh(userID int64, msg []byte) bool {
h.mu.RLock()
c, ok := h.clients[userID]
h.mu.RUnlock()
if !ok {
return false
}
c.sendHighMsg(msg)
return true
}
// BroadcastToAllLow enqueues a low-priority global broadcast.
// Low-priority messages are silently dropped if a client's buffer is full.
func (h *Hub) BroadcastToAllLow(msg []byte) {
// Low-priority global broadcasts bypass the sequenced broadcast channel
// and go directly through pub/sub — they don't need replay or seq numbering.
h.pubsub.PublishGlobalLow(msg)
}
// sendSequencedToUsers stamps msg with a monotonic seq, stores it in the replay
// buffer under channelID, and fanouts the wrapped payload to the provided users.
func (h *Hub) sendSequencedToUsers(channelID int64, userIDs []int64, msg []byte) {
@@ -530,6 +550,20 @@ func (h *Hub) sendSequencedToUsers(channelID int64, userIDs []int64, msg []byte)
}
}
// sendSequencedToUsersHigh is like sendSequencedToUsers but uses high-priority delivery.
func (h *Hub) sendSequencedToUsersHigh(channelID int64, userIDs []int64, msg []byte) {
h.seqMu.Lock()
defer h.seqMu.Unlock()
seq := h.nextSeq()
wrapped := wrapWithSeq(msg, seq)
h.replayBuf.Push(seq, channelID, wrapped)
for _, userID := range userIDs {
h.SendToUserHigh(userID, wrapped)
}
}
// ClientCount returns the number of currently registered clients (test helper).
func (h *Hub) ClientCount() int {
h.mu.RLock()
+47 -37
View File
@@ -130,10 +130,45 @@ func (ps *PubSub) UnsubscribeAll(client *Client) {
delete(ps.clients, client.userID)
}
// Publish sends msg to all subscribers of topic, except the client identified
// by excludeUserID (pass 0 to exclude nobody). Returns the number of clients
// the message was delivered to.
// Priority levels for pub/sub delivery.
const (
PriorityHigh = 0 // DMs, direct mentions — drained first by writePump
PriorityNormal = 1 // chat messages, reactions, channel events
PriorityLow = 2 // typing indicators, presence updates — dropped on overflow
)
// Publish sends msg to all subscribers of topic at normal priority.
// If a client's buffer is full, it is disconnected.
func (ps *PubSub) Publish(topic Topic, msg []byte, excludeUserID int64) int {
return ps.publishWithPriority(topic, msg, excludeUserID, PriorityNormal)
}
// PublishHigh sends msg at high priority (DMs, mentions).
// High-priority messages are drained before normal/low by writePump.
func (ps *PubSub) PublishHigh(topic Topic, msg []byte, excludeUserID int64) int {
return ps.publishWithPriority(topic, msg, excludeUserID, PriorityHigh)
}
// PublishLow sends msg at low priority (typing, presence).
// If a client's buffer is full the message is silently dropped.
func (ps *PubSub) PublishLow(topic Topic, msg []byte, excludeUserID int64) int {
return ps.publishWithPriority(topic, msg, excludeUserID, PriorityLow)
}
// PublishGlobal sends msg to every client subscribed to the "global" topic
// at normal priority.
func (ps *PubSub) PublishGlobal(msg []byte) int {
return ps.Publish(TopicGlobal, msg, 0)
}
// PublishGlobalLow sends msg to all global subscribers at low priority.
func (ps *PubSub) PublishGlobalLow(msg []byte) int {
return ps.PublishLow(TopicGlobal, msg, 0)
}
// publishWithPriority is the core publish method routing to the appropriate
// client send method based on priority level.
func (ps *PubSub) publishWithPriority(topic Topic, msg []byte, excludeUserID int64, priority int) int {
ps.mu.RLock()
subs := ps.topics[topic]
// Snapshot the subscriber slice under read lock to avoid holding the lock
@@ -146,48 +181,23 @@ func (ps *PubSub) Publish(topic Topic, msg []byte, excludeUserID int64) int {
}
ps.mu.RUnlock()
for _, c := range clients {
c.sendMsg(msg)
}
return len(clients)
}
// PublishLowPriority sends msg to subscribers of topic using trySendMsg, which
// silently drops the message if a client's buffer is full instead of
// disconnecting them. Use this for ephemeral events like typing indicators
// and presence updates that can be safely lost.
func (ps *PubSub) PublishLowPriority(topic Topic, msg []byte, excludeUserID int64) int {
ps.mu.RLock()
subs := ps.topics[topic]
clients := make([]*Client, 0, len(subs))
for uid, c := range subs {
if uid != excludeUserID {
clients = append(clients, c)
}
}
ps.mu.RUnlock()
delivered := 0
for _, c := range clients {
if c.trySendMsg(msg) {
switch priority {
case PriorityHigh:
c.sendHighMsg(msg)
delivered++
case PriorityLow:
c.sendLowMsg(msg)
delivered++ // count attempt, even if dropped
default:
c.sendMsg(msg)
delivered++
}
}
return delivered
}
// PublishGlobal sends msg to every client subscribed to the "global" topic.
// This is equivalent to Publish(TopicGlobal, msg, 0) but makes intent explicit.
func (ps *PubSub) PublishGlobal(msg []byte) int {
return ps.Publish(TopicGlobal, msg, 0)
}
// PublishGlobalLowPriority sends msg to all global subscribers using
// trySendMsg (drop instead of disconnect on full buffer).
func (ps *PubSub) PublishGlobalLowPriority(msg []byte) int {
return ps.PublishLowPriority(TopicGlobal, msg, 0)
}
// SubscriberCount returns the number of subscribers for a topic.
func (ps *PubSub) SubscriberCount(topic Topic) int {
ps.mu.RLock()
+35 -11
View File
@@ -13,8 +13,10 @@ func newTestPubSub() *PubSub {
func makeTestClient(userID int64) *Client {
return &Client{
userID: userID,
send: make(chan []byte, 16),
userID: userID,
send: make(chan []byte, 16),
sendHigh: make(chan []byte, 16),
sendLow: make(chan []byte, 16),
}
}
@@ -165,25 +167,47 @@ func TestPubSub_PublishGlobal(t *testing.T) {
assertChanMsg(t, c2.send, msg)
}
func TestPubSub_PublishLowPriority(t *testing.T) {
func TestPubSub_PublishLow(t *testing.T) {
ps := newTestPubSub()
c1 := makeTestClient(1)
c2 := &Client{userID: 2, send: make(chan []byte, 1)} // tiny buffer
// c2 has a tiny low-priority buffer that we pre-fill.
c2 := &Client{userID: 2, send: make(chan []byte, 16), sendHigh: make(chan []byte, 4), sendLow: make(chan []byte, 1)}
ps.Subscribe(c1, "channel:1")
ps.Subscribe(c2, "channel:1")
// Fill c2's buffer so it drops low-priority messages.
c2.send <- []byte(`filler`)
// Fill c2's low-priority buffer so it drops.
c2.sendLow <- []byte(`filler`)
msg := []byte(`{"type":"typing"}`)
delivered := ps.PublishLowPriority("channel:1", msg, 0)
delivered := ps.PublishLow("channel:1", msg, 0)
// c1 should receive, c2 should be dropped (buffer full).
if delivered != 1 {
t.Fatalf("delivered = %d, want 1 (c2 buffer full)", delivered)
// Both get counted (sendLowMsg counts the attempt), but c2's message was dropped.
if delivered != 2 {
t.Fatalf("delivered = %d, want 2", delivered)
}
assertChanMsg(t, c1.send, msg)
assertChanMsg(t, c1.sendLow, msg)
// c2's sendLow only has the filler, not the typing msg.
assertChanMsg(t, c2.sendLow, []byte(`filler`))
assertChanEmpty(t, c2.sendLow)
}
func TestPubSub_PublishHigh(t *testing.T) {
ps := newTestPubSub()
c1 := makeTestClient(1)
c2 := makeTestClient(2)
ps.Subscribe(c1, UserTopic(1))
ps.Subscribe(c2, UserTopic(1))
msg := []byte(`{"type":"dm"}`)
delivered := ps.PublishHigh(UserTopic(1), msg, 0)
if delivered != 2 {
t.Fatalf("delivered = %d, want 2", delivered)
}
assertChanMsg(t, c1.sendHigh, msg)
assertChanMsg(t, c2.sendHigh, msg)
}
// ─── TopicsForClient ─────────────────────────────────────────────────────────
+49 -6
View File
@@ -303,20 +303,63 @@ func (h *Hub) handleFreshConnect(
return nil
}
// writePump drains the client's send channel and writes to the WebSocket.
// writePump drains the client's send channels and writes to the WebSocket.
// Priority ordering: high > normal > low. High-priority messages (DMs, mentions)
// are drained first. Normal messages (chat, reactions) come next. Low-priority
// messages (typing, presence) are only sent when no higher-priority work is pending.
func writePump(ctx context.Context, conn *websocket.Conn, c *Client) {
writeMsg := func(msg []byte) bool {
wCtx, cancel := context.WithTimeout(ctx, writeTimeout)
err := conn.Write(wCtx, websocket.MessageText, msg)
cancel()
if err != nil {
slog.Warn("ws writePump error", "user_id", c.userID, "err", err)
return false
}
return true
}
for {
// Priority 1: drain all pending high-priority messages first.
select {
case msg, ok := <-c.sendHigh:
if !ok {
_ = conn.Close(websocket.StatusNormalClosure, "")
return
}
if !writeMsg(msg) {
return
}
continue
default:
}
// Priority 2: try high or normal (high still gets priority via the
// first case in the select, but Go's select is random when both are
// ready — the outer drain-high loop above ensures high is truly first).
select {
case msg, ok := <-c.sendHigh:
if !ok {
_ = conn.Close(websocket.StatusNormalClosure, "")
return
}
if !writeMsg(msg) {
return
}
case msg, ok := <-c.send:
if !ok {
_ = conn.Close(websocket.StatusNormalClosure, "")
return
}
wCtx, cancel := context.WithTimeout(ctx, writeTimeout)
err := conn.Write(wCtx, websocket.MessageText, msg)
cancel()
if err != nil {
slog.Warn("ws writePump error", "user_id", c.userID, "err", err)
if !writeMsg(msg) {
return
}
case msg, ok := <-c.sendLow:
if !ok {
_ = conn.Close(websocket.StatusNormalClosure, "")
return
}
if !writeMsg(msg) {
return
}
case <-ctx.Done():