Stirling-Tools/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2026-09-03 05:10:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add per-endpoint concurrency so one address can front a shared unoserver fleet

Browse Source
This commit is contained in:
Anthony Stirling
2026-08-31 00:24:38 +01:00
parent 34694c6f5e
commit 1aaf889135
15 changed files with 647 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java
+9 -5
View File
@@ -292,12 +292,16 @@ public class RuntimePathConfig {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> configured =
sanitizeUnoServerEndpoints(processExecutor.getUnoServerEndpoints());
if (!configured.isEmpty()) {
// Warn if manual endpoint count doesn't match sessionLimit
if (configured.size() != sessionLimit) {
int slots = 0;
for (ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint :
configured) {
slots += Math.max(1, endpoint.getConcurrency());
}
if (slots != sessionLimit) {
log.warn(
"Manual UNO endpoint count ({}) differs from libreOfficeSessionLimit ({}). "
+ "Concurrency will be limited by endpoint count, not sessionLimit.",
configured.size(),
"Manual UNO endpoints allow {} concurrent conversion(s) but libreOfficeSessionLimit is {}. "
+ "Concurrency will be limited by the endpoints, not sessionLimit.",
slots,
sessionLimit);
}
return configured;
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+1
View File
@@ -1612,6 +1612,7 @@ public class ApplicationProperties {
private int port = 2003;
private String hostLocation = "auto"; // auto|local|remote
private String protocol = "http"; // http|https
private int concurrency = 1;
}
@Data
app/common/src/main/java/stirling/software/common/util/UnoServerPool.java
+15 -2
View File
@@ -24,9 +24,10 @@ public class UnoServerPool {
} else {
this.endpoints = new ArrayList<>(endpoints);
this.availableIndices = new LinkedBlockingQueue<>();
// Initialize queue with all endpoint indices
for (int i = 0; i < this.endpoints.size(); i++) {
this.availableIndices.offer(i);
for (int slot = 0; slot < slotsFor(this.endpoints.get(i)); slot++) {
this.availableIndices.offer(i);
}
}
}
}
@@ -35,6 +36,18 @@ public class UnoServerPool {
return endpoints.isEmpty();
}
public int totalSlots() {
int total = 0;
for (ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint : endpoints) {
total += slotsFor(endpoint);
}
return total;
}
private static int slotsFor(ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint) {
return Math.max(1, endpoint.getConcurrency());
}
public UnoServerLease acquireEndpoint() throws InterruptedException {
if (endpoints.isEmpty()) {
return new UnoServerLease(defaultEndpoint(), null, this);
app/common/src/test/java/stirling/software/common/util/UnoServerPoolTest.java
+44
View File
@@ -259,6 +259,50 @@ public class UnoServerPoolTest {
}
}
@Test
void defaultConcurrencyKeepsOneSlotPerEndpoint() {
UnoServerPool pool = new UnoServerPool(createEndpoints(3));
assertEquals(3, pool.totalSlots(), "Unset concurrency must behave as one slot each");
}
@Test
void loadBalancerEntryHandsOutItsConcurrencyInParallel() throws InterruptedException {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
createEndpoints(1);
endpoints.getFirst().setHost("unoserver-lb");
endpoints.getFirst().setConcurrency(4);
UnoServerPool pool = new UnoServerPool(endpoints);
assertEquals(4, pool.totalSlots());
List<UnoServerPool.UnoServerLease> held = new ArrayList<>();
for (int i = 0; i < 4; i++) {
UnoServerPool.UnoServerLease lease = pool.acquireEndpoint();
assertEquals("unoserver-lb", lease.getEndpoint().getHost());
held.add(lease);
}
assertThrows(
TimeoutException.class,
() -> pool.acquireEndpoint(100, TimeUnit.MILLISECONDS),
"The fifth acquire must wait: concurrency is a cap, not a hint");
held.forEach(UnoServerPool.UnoServerLease::close);
try (UnoServerPool.UnoServerLease reacquired = pool.acquireEndpoint()) {
assertEquals("unoserver-lb", reacquired.getEndpoint().getHost());
}
}
@Test
void slotsSumAcrossMixedEndpoints() {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints =
createEndpoints(2);
endpoints.get(0).setConcurrency(3);
endpoints.get(1).setConcurrency(0);
UnoServerPool pool = new UnoServerPool(endpoints);
assertEquals(4, pool.totalSlots(), "3 + a clamped 1");
}
private List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> createEndpoints(
int count) {
List<ApplicationProperties.ProcessExecutor.UnoServerEndpoint> endpoints = new ArrayList<>();
app/core/src/main/resources/settings.yml.template
+1
View File
@@ -343,6 +343,7 @@ processExecutor:
# port: 2003
# hostLocation: "auto" # auto|local|remote (use "remote" for port-forwarded servers)
# protocol: "http" # http|https
# concurrency: 1
# - host: "remote-server.local"
# port: 8080
# hostLocation: "remote"
docker/unoserver/deploy/compose-direct.yml
+44
View File
@@ -0,0 +1,44 @@
x-uno: &uno
image: ghcr.io/stirling-tools/stirling-unoserver:latest
environment:
UNOSERVER_PORT: "2003"
UNOSERVER_CONVERSION_TIMEOUT: "1800"
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
healthcheck:
test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"]
interval: 10s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
networks: [stirling]
services:
stirling-pdf:
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
ports:
- "8080:8080"
environment:
PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-1"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_HOST: "unoserver-2"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_HOST_LOCATION: "remote"
PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "2"
depends_on:
unoserver-1:
condition: service_healthy
unoserver-2:
condition: service_healthy
restart: unless-stopped
networks: [stirling]
unoserver-1:
<<: *uno
unoserver-2:
<<: *uno
networks:
stirling:
docker/unoserver/deploy/compose-haproxy.yml
+43
View File
@@ -0,0 +1,43 @@
services:
stirling-pdf:
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
ports:
- "8080:8080"
environment:
PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "4"
PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "4"
depends_on:
- unoserver-lb
restart: unless-stopped
networks: [stirling]
unoserver-lb:
image: haproxy:3.0-alpine
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "8404:8404"
restart: unless-stopped
networks: [stirling]
unoserver:
image: ghcr.io/stirling-tools/stirling-unoserver:latest
environment:
UNOSERVER_PORT: "2003"
UNOSERVER_CONVERSION_TIMEOUT: "1800"
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
healthcheck:
test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"]
interval: 10s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
networks: [stirling]
networks:
stirling:
docker/unoserver/deploy/compose-nginx.yml
+48
View File
@@ -0,0 +1,48 @@
x-uno: &uno
image: ghcr.io/stirling-tools/stirling-unoserver:latest
environment:
UNOSERVER_PORT: "2003"
UNOSERVER_CONVERSION_TIMEOUT: "1800"
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
healthcheck:
test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"]
interval: 10s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
networks: [stirling]
services:
stirling-pdf:
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
ports:
- "8080:8080"
environment:
PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "3"
PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "3"
depends_on:
- unoserver-lb
restart: unless-stopped
networks: [stirling]
unoserver-lb:
image: nginx:1.27-alpine
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
restart: unless-stopped
networks: [stirling]
unoserver-1:
<<: *uno
unoserver-2:
<<: *uno
unoserver-3:
<<: *uno
networks:
stirling:
docker/unoserver/deploy/compose-sharedfs.yml
+72
View File
@@ -0,0 +1,72 @@
x-uno: &uno
image: ghcr.io/stirling-tools/stirling-unoserver:latest
environment:
UNOSERVER_PORT: "2003"
UNOSERVER_CONVERSION_TIMEOUT: "1800"
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
volumes:
- uno-shared:/shared/stirling-pdf
healthcheck:
test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"]
interval: 10s
timeout: 10s
retries: 3
start_period: 30s
depends_on:
uno-shared-init:
condition: service_completed_successfully
restart: unless-stopped
networks: [stirling]
services:
uno-shared-init:
image: alpine:3.20
user: "0:0"
command: ["sh", "-c", "mkdir -p /shared/stirling-pdf && chown -R 1001:1001 /shared/stirling-pdf && chmod 775 /shared/stirling-pdf"]
volumes:
- uno-shared:/shared/stirling-pdf
networks: [stirling]
stirling-pdf:
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
ports:
- "8080:8080"
environment:
PUID: "1001"
PGID: "1001"
SYSTEM_TEMPFILEMANAGEMENT_BASETMPDIR: "/shared/stirling-pdf"
PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "local"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "3"
PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "3"
volumes:
- uno-shared:/shared/stirling-pdf
depends_on:
uno-shared-init:
condition: service_completed_successfully
unoserver-lb:
condition: service_started
restart: unless-stopped
networks: [stirling]
unoserver-lb:
image: nginx:1.27-alpine
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
restart: unless-stopped
networks: [stirling]
unoserver-1:
<<: *uno
unoserver-2:
<<: *uno
unoserver-3:
<<: *uno
volumes:
uno-shared:
networks:
stirling:
docker/unoserver/deploy/haproxy.cfg
+38
View File
@@ -0,0 +1,38 @@
global
log stdout format raw local0 info
maxconn 4000
defaults
mode http
log global
option httplog
timeout connect 5s
timeout client 1830s
timeout server 1830s
timeout queue 300s
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 5s
hold other 5s
hold refused 5s
hold nx 5s
hold timeout 5s
frontend uno_in
bind *:2003
default_backend uno_pool
frontend uno_stats
bind *:8404
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /
stats refresh 5s
backend uno_pool
balance leastconn
server-template uno 20 unoserver:2003 check resolvers docker init-addr none maxconn 1
docker/unoserver/deploy/kubernetes.yaml
+102
View File
@@ -0,0 +1,102 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: unoserver
labels:
app: unoserver
spec:
replicas: 5
selector:
matchLabels:
app: unoserver
template:
metadata:
labels:
app: unoserver
spec:
terminationGracePeriodSeconds: 1800
containers:
- name: unoserver
image: ghcr.io/stirling-tools/stirling-unoserver:latest
ports:
- containerPort: 2003
name: rpc
env:
- name: UNOSERVER_PORT
value: "2003"
- name: UNOSERVER_CONVERSION_TIMEOUT
value: "1800"
- name: UNOSERVER_RECYCLE_INTERVAL_SECONDS
value: "3600"
resources:
requests:
cpu: "500m"
memory: "1Gi"
limits:
memory: "2Gi"
readinessProbe:
exec:
command: ["sh", "-c", "unoping --host 127.0.0.1 --port 2003"]
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 10
livenessProbe:
exec:
command: ["sh", "-c", "unoping --host 127.0.0.1 --port 2003"]
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 3
---
apiVersion: v1
kind: Service
metadata:
name: unoserver
spec:
selector:
app: unoserver
ports:
- port: 2003
targetPort: 2003
name: rpc
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: stirling-pdf
labels:
app: stirling-pdf
spec:
replicas: 3
selector:
matchLabels:
app: stirling-pdf
template:
metadata:
labels:
app: stirling-pdf
spec:
containers:
- name: stirling-pdf
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
ports:
- containerPort: 8080
env:
- name: PROCESS_EXECUTOR_AUTO_UNO_SERVER
value: "false"
- name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST
value: "unoserver"
- name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT
value: "2003"
- name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION
value: "remote"
- name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY
value: "2"
- name: PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT
value: "2"
readinessProbe:
httpGet:
path: /api/v1/info/status
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
docker/unoserver/deploy/nginx.conf
+29
View File
@@ -0,0 +1,29 @@
worker_processes 1;
events { worker_connections 1024; }
http {
upstream uno_pool {
least_conn;
server unoserver-1:2003 max_fails=1 fail_timeout=10s;
server unoserver-2:2003 max_fails=1 fail_timeout=10s;
server unoserver-3:2003 max_fails=1 fail_timeout=10s;
}
client_max_body_size 0;
proxy_request_buffering off;
proxy_buffering off;
server {
listen 2003;
location / {
proxy_pass http://uno_pool;
proxy_http_version 1.1;
proxy_connect_timeout 5s;
proxy_send_timeout 1830s;
proxy_read_timeout 1830s;
proxy_next_upstream off;
}
}
}
testing/compose/docker-compose-multinode-unoserver.override.yml
+69
View File
@@ -0,0 +1,69 @@
# Shared unoserver pool for the multi-node stack: every Stirling node converts through
# one HAProxy-fronted fleet instead of running its own LibreOffice.
#
# Bring up: docker compose -f docker-compose-multinode.yml -f docker-compose-multinode-unoserver.override.yml up -d --scale unoserver=3
# Resize: docker compose ... up -d --scale unoserver=6 --no-recreate (nothing restarts)
# Watch: http://localhost:8404/ (queue depth, per-server sessions)
#
# Why a proxy rather than listing unoserver hosts on each node: unoserver converts
# strictly in series (measured: 1 job 6.0s, 2 jobs 10.5s, 4 jobs 20.8s against one
# instance) and queues without bound instead of rejecting. UnoServerPool's semaphore
# is per-JVM, so N nodes each believing they may run M conversions oversubscribes the
# fleet by N and the excess piles up invisibly inside unoserver. One shared queue in
# front is the only place that limit can actually be enforced.
x-uno-endpoints: &uno-endpoints
PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false"
# One entry for the whole fleet. concurrency is this node's in-flight cap, not a host
# count: real capacity is the replica count, and HAProxy queues the difference.
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote"
PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "4"
# Matches the slot count so RuntimePathConfig does not log a mismatch warning.
PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "4"
services:
stirling-1:
environment: *uno-endpoints
depends_on:
unoserver-lb:
condition: service_started
stirling-2:
environment: *uno-endpoints
depends_on:
unoserver-lb:
condition: service_started
# No container_name: that is what allows --scale.
unoserver:
image: ghcr.io/stirling-tools/stirling-unoserver:latest
environment:
UNOSERVER_PORT: "2003"
UNOSERVER_UNO_PORT: "2002"
UNOSERVER_CONVERSION_TIMEOUT: "1800"
# Hourly restart bounds LibreOffice memory growth. Safe to leave on here only
# because HAProxy health checks pull a recycling instance out of rotation.
UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600"
healthcheck:
# unoping is a real RPC round-trip: the TCP port stays bound while LibreOffice
# is wedged, so a port check would keep a dead instance in the pool.
test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"]
interval: 10s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
networks:
- stirling-multinode
unoserver-lb:
image: haproxy:3.0-alpine
volumes:
- ./multinode/haproxy-unoserver.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "8404:8404" # stats + /metrics
restart: unless-stopped
networks:
- stirling-multinode
testing/compose/multinode/haproxy-unoserver.cfg
+55
View File
@@ -0,0 +1,55 @@
# Shared unoserver pool for a multi-node Stirling cluster.
#
# Every Stirling node points every one of its PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_n
# entries at this one address, so the whole cluster draws from a single queue instead
# of each node guessing at capacity it cannot see.
global
log stdout format raw local0 info
maxconn 4000
defaults
mode http
log global
option httplog
timeout connect 5s
# A conversion holds its connection open for the whole job, so these must exceed
# UNOSERVER_CONVERSION_TIMEOUT or HAProxy cuts jobs unoserver would have finished.
timeout client 1830s
timeout server 1830s
# Bounded wait for a free backend. Past this the caller gets 503 rather than
# queueing forever behind a fleet that is too small.
timeout queue 300s
# Docker's embedded DNS. Re-resolved on the hold timers below, which is what lets
# `docker compose up --scale unoserver=N` change capacity with nothing restarted.
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 5s
hold other 5s
hold refused 5s
hold nx 5s
hold timeout 5s
frontend uno_in
bind *:2003
default_backend uno_pool
# Stats page plus a Prometheus endpoint: queue depth and per-server session counts
# are the two numbers that say whether the pool is the bottleneck.
frontend uno_stats
bind *:8404
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /
stats refresh 5s
backend uno_pool
balance leastconn
# maxconn 1 because unoserver runs one LibreOffice and converts strictly in
# series: a second request does not run faster, it waits. Holding that queue
# here makes it bounded, measurable, and shared across every Stirling node.
server-template uno 20 unoserver:2003 check resolvers docker init-addr none maxconn 1
testing/compose/validate-unoserver-pool.sh
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Proves the shared unoserver pool scales without restarting anything: capacity tracks
# the replica count, a killed instance leaves rotation, and the queue is observable.
# Run against a stack started with docker-compose-multinode-unoserver.override.yml.
# Non-destructive apart from killing one unoserver replica, which restarts on its own.
set -uo pipefail
cd "$(dirname "$0")"
COMPOSE="docker compose -f docker-compose-multinode.yml -f docker-compose-multinode-unoserver.override.yml"
STATS="http://localhost:8404/metrics"
pass=0; fail=0
ok() { echo " PASS - $*"; pass=$((pass+1)); }
bad() { echo " FAIL - $*"; fail=$((fail+1)); }
# Servers HAProxy has resolved an address for and considers UP.
up_backends() {
curl -s "$STATS" \
| awk -F'server="' '/haproxy_server_status\{proxy="uno_pool".*state="UP"\} 1$/ {split($2,a,"\""); print a[1]}' \
| sort -u
}
up_count() { up_backends | grep -c . ; }
wait_for_backends() { # $1 = expected count, $2 = seconds
for _ in $(seq 1 "$2"); do
[ "$(up_count)" -ge "$1" ] && return 0
sleep 1
done
return 1
}
echo "== 1. HAProxy discovered the running replicas =="
replicas=$($COMPOSE ps -q unoserver | grep -c .)
if wait_for_backends "$replicas" 60; then
ok "$(up_count) backends UP for $replicas replicas: $(up_backends | paste -sd, -)"
else
bad "only $(up_count) backends UP, expected $replicas"
fi
echo "== 2. Scaling up is picked up with nothing restarted =="
lb_before=$($COMPOSE ps -q unoserver-lb)
target=$((replicas + 1))
$COMPOSE up -d --scale unoserver=$target --no-recreate >/dev/null 2>&1
if wait_for_backends "$target" 90; then
ok "scaled $replicas -> $target, HAProxy sees $(up_count)"
else
bad "HAProxy still sees $(up_count) after scaling to $target"
fi
[ "$lb_before" = "$($COMPOSE ps -q unoserver-lb)" ] \
&& ok "HAProxy container never restarted" \
|| bad "HAProxy was recreated (it should not need to be)"
echo "== 3. A dead instance leaves rotation =="
victim=$($COMPOSE ps -q unoserver | tail -1)
docker kill "$victim" >/dev/null 2>&1
dropped=1
for _ in $(seq 1 30); do
[ "$(up_count)" -lt "$target" ] && { dropped=0; break; }
sleep 1
done
[ "$dropped" -eq 0 ] && ok "backend removed, $(up_count) left serving" \
|| bad "dead backend still marked UP after 30s"
docker start "$victim" >/dev/null 2>&1
echo "== 4. Queue depth is exported for alerting and autoscaling =="
# Scraped into a variable rather than piped into grep -q: under pipefail an early
# grep exit makes curl fail on SIGPIPE and the check reports a false negative.
metrics=$(curl -s "$STATS")
q=$(printf '%s\n' "$metrics" | awk '/haproxy_backend_current_queue\{proxy="uno_pool"\}/ {print $2}')
if [ -n "$q" ]; then
ok "haproxy_backend_current_queue is exported (currently $q)"
else
bad "queue metric missing; is the prometheus-exporter line in haproxy-unoserver.cfg?"
fi
echo
echo "passed: $pass failed: $fail"
[ "$fail" -eq 0 ]