diff --git a/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java b/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java index 5be836f82a..c9870e8ff2 100644 --- a/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java +++ b/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java @@ -292,12 +292,16 @@ public class RuntimePathConfig { List configured = sanitizeUnoServerEndpoints(processExecutor.getUnoServerEndpoints()); if (!configured.isEmpty()) { - // Warn if manual endpoint count doesn't match sessionLimit - if (configured.size() != sessionLimit) { + int slots = 0; + for (ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint : + configured) { + slots += Math.max(1, endpoint.getConcurrency()); + } + if (slots != sessionLimit) { log.warn( - "Manual UNO endpoint count ({}) differs from libreOfficeSessionLimit ({}). " - + "Concurrency will be limited by endpoint count, not sessionLimit.", - configured.size(), + "Manual UNO endpoints allow {} concurrent conversion(s) but libreOfficeSessionLimit is {}. " + + "Concurrency will be limited by the endpoints, not sessionLimit.", + slots, sessionLimit); } return configured; diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index 216e0c1e66..c151e579db 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -1612,6 +1612,7 @@ public class ApplicationProperties { private int port = 2003; private String hostLocation = "auto"; // auto|local|remote private String protocol = "http"; // http|https + private int concurrency = 1; } @Data diff --git a/app/common/src/main/java/stirling/software/common/util/UnoServerPool.java b/app/common/src/main/java/stirling/software/common/util/UnoServerPool.java index 3863bd5310..2d960d2cab 100644 --- a/app/common/src/main/java/stirling/software/common/util/UnoServerPool.java +++ b/app/common/src/main/java/stirling/software/common/util/UnoServerPool.java @@ -24,9 +24,10 @@ public class UnoServerPool { } else { this.endpoints = new ArrayList<>(endpoints); this.availableIndices = new LinkedBlockingQueue<>(); - // Initialize queue with all endpoint indices for (int i = 0; i < this.endpoints.size(); i++) { - this.availableIndices.offer(i); + for (int slot = 0; slot < slotsFor(this.endpoints.get(i)); slot++) { + this.availableIndices.offer(i); + } } } } @@ -35,6 +36,18 @@ public class UnoServerPool { return endpoints.isEmpty(); } + public int totalSlots() { + int total = 0; + for (ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint : endpoints) { + total += slotsFor(endpoint); + } + return total; + } + + private static int slotsFor(ApplicationProperties.ProcessExecutor.UnoServerEndpoint endpoint) { + return Math.max(1, endpoint.getConcurrency()); + } + public UnoServerLease acquireEndpoint() throws InterruptedException { if (endpoints.isEmpty()) { return new UnoServerLease(defaultEndpoint(), null, this); diff --git a/app/common/src/test/java/stirling/software/common/util/UnoServerPoolTest.java b/app/common/src/test/java/stirling/software/common/util/UnoServerPoolTest.java index 06d1575ef5..dac438701d 100644 --- a/app/common/src/test/java/stirling/software/common/util/UnoServerPoolTest.java +++ b/app/common/src/test/java/stirling/software/common/util/UnoServerPoolTest.java @@ -259,6 +259,50 @@ public class UnoServerPoolTest { } } + @Test + void defaultConcurrencyKeepsOneSlotPerEndpoint() { + UnoServerPool pool = new UnoServerPool(createEndpoints(3)); + assertEquals(3, pool.totalSlots(), "Unset concurrency must behave as one slot each"); + } + + @Test + void loadBalancerEntryHandsOutItsConcurrencyInParallel() throws InterruptedException { + List endpoints = + createEndpoints(1); + endpoints.getFirst().setHost("unoserver-lb"); + endpoints.getFirst().setConcurrency(4); + + UnoServerPool pool = new UnoServerPool(endpoints); + assertEquals(4, pool.totalSlots()); + + List held = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + UnoServerPool.UnoServerLease lease = pool.acquireEndpoint(); + assertEquals("unoserver-lb", lease.getEndpoint().getHost()); + held.add(lease); + } + assertThrows( + TimeoutException.class, + () -> pool.acquireEndpoint(100, TimeUnit.MILLISECONDS), + "The fifth acquire must wait: concurrency is a cap, not a hint"); + + held.forEach(UnoServerPool.UnoServerLease::close); + try (UnoServerPool.UnoServerLease reacquired = pool.acquireEndpoint()) { + assertEquals("unoserver-lb", reacquired.getEndpoint().getHost()); + } + } + + @Test + void slotsSumAcrossMixedEndpoints() { + List endpoints = + createEndpoints(2); + endpoints.get(0).setConcurrency(3); + endpoints.get(1).setConcurrency(0); + + UnoServerPool pool = new UnoServerPool(endpoints); + assertEquals(4, pool.totalSlots(), "3 + a clamped 1"); + } + private List createEndpoints( int count) { List endpoints = new ArrayList<>(); diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index ecf7ea8538..ecfdce8fc5 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -343,6 +343,7 @@ processExecutor: # port: 2003 # hostLocation: "auto" # auto|local|remote (use "remote" for port-forwarded servers) # protocol: "http" # http|https + # concurrency: 1 # - host: "remote-server.local" # port: 8080 # hostLocation: "remote" diff --git a/docker/unoserver/deploy/compose-direct.yml b/docker/unoserver/deploy/compose-direct.yml new file mode 100644 index 0000000000..587cd6f408 --- /dev/null +++ b/docker/unoserver/deploy/compose-direct.yml @@ -0,0 +1,44 @@ +x-uno: &uno + image: ghcr.io/stirling-tools/stirling-unoserver:latest + environment: + UNOSERVER_PORT: "2003" + UNOSERVER_CONVERSION_TIMEOUT: "1800" + UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600" + healthcheck: + test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + networks: [stirling] + +services: + stirling-pdf: + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + ports: + - "8080:8080" + environment: + PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-1" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_HOST: "unoserver-2" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_1_HOST_LOCATION: "remote" + PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "2" + depends_on: + unoserver-1: + condition: service_healthy + unoserver-2: + condition: service_healthy + restart: unless-stopped + networks: [stirling] + + unoserver-1: + <<: *uno + unoserver-2: + <<: *uno + +networks: + stirling: diff --git a/docker/unoserver/deploy/compose-haproxy.yml b/docker/unoserver/deploy/compose-haproxy.yml new file mode 100644 index 0000000000..c8f7adf993 --- /dev/null +++ b/docker/unoserver/deploy/compose-haproxy.yml @@ -0,0 +1,43 @@ +services: + stirling-pdf: + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + ports: + - "8080:8080" + environment: + PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "4" + PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "4" + depends_on: + - unoserver-lb + restart: unless-stopped + networks: [stirling] + + unoserver-lb: + image: haproxy:3.0-alpine + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "8404:8404" + restart: unless-stopped + networks: [stirling] + + unoserver: + image: ghcr.io/stirling-tools/stirling-unoserver:latest + environment: + UNOSERVER_PORT: "2003" + UNOSERVER_CONVERSION_TIMEOUT: "1800" + UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600" + healthcheck: + test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + networks: [stirling] + +networks: + stirling: diff --git a/docker/unoserver/deploy/compose-nginx.yml b/docker/unoserver/deploy/compose-nginx.yml new file mode 100644 index 0000000000..f6b642a23a --- /dev/null +++ b/docker/unoserver/deploy/compose-nginx.yml @@ -0,0 +1,48 @@ +x-uno: &uno + image: ghcr.io/stirling-tools/stirling-unoserver:latest + environment: + UNOSERVER_PORT: "2003" + UNOSERVER_CONVERSION_TIMEOUT: "1800" + UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600" + healthcheck: + test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + networks: [stirling] + +services: + stirling-pdf: + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + ports: + - "8080:8080" + environment: + PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "3" + PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "3" + depends_on: + - unoserver-lb + restart: unless-stopped + networks: [stirling] + + unoserver-lb: + image: nginx:1.27-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + restart: unless-stopped + networks: [stirling] + + unoserver-1: + <<: *uno + unoserver-2: + <<: *uno + unoserver-3: + <<: *uno + +networks: + stirling: diff --git a/docker/unoserver/deploy/compose-sharedfs.yml b/docker/unoserver/deploy/compose-sharedfs.yml new file mode 100644 index 0000000000..5acaa96c93 --- /dev/null +++ b/docker/unoserver/deploy/compose-sharedfs.yml @@ -0,0 +1,72 @@ +x-uno: &uno + image: ghcr.io/stirling-tools/stirling-unoserver:latest + environment: + UNOSERVER_PORT: "2003" + UNOSERVER_CONVERSION_TIMEOUT: "1800" + UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600" + volumes: + - uno-shared:/shared/stirling-pdf + healthcheck: + test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s + depends_on: + uno-shared-init: + condition: service_completed_successfully + restart: unless-stopped + networks: [stirling] + +services: + uno-shared-init: + image: alpine:3.20 + user: "0:0" + command: ["sh", "-c", "mkdir -p /shared/stirling-pdf && chown -R 1001:1001 /shared/stirling-pdf && chmod 775 /shared/stirling-pdf"] + volumes: + - uno-shared:/shared/stirling-pdf + networks: [stirling] + + stirling-pdf: + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + ports: + - "8080:8080" + environment: + PUID: "1001" + PGID: "1001" + SYSTEM_TEMPFILEMANAGEMENT_BASETMPDIR: "/shared/stirling-pdf" + PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "local" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "3" + PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "3" + volumes: + - uno-shared:/shared/stirling-pdf + depends_on: + uno-shared-init: + condition: service_completed_successfully + unoserver-lb: + condition: service_started + restart: unless-stopped + networks: [stirling] + + unoserver-lb: + image: nginx:1.27-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + restart: unless-stopped + networks: [stirling] + + unoserver-1: + <<: *uno + unoserver-2: + <<: *uno + unoserver-3: + <<: *uno + +volumes: + uno-shared: + +networks: + stirling: diff --git a/docker/unoserver/deploy/haproxy.cfg b/docker/unoserver/deploy/haproxy.cfg new file mode 100644 index 0000000000..6b02f8c965 --- /dev/null +++ b/docker/unoserver/deploy/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 info + maxconn 4000 + +defaults + mode http + log global + option httplog + timeout connect 5s + timeout client 1830s + timeout server 1830s + timeout queue 300s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold other 5s + hold refused 5s + hold nx 5s + hold timeout 5s + +frontend uno_in + bind *:2003 + default_backend uno_pool + +frontend uno_stats + bind *:8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri / + stats refresh 5s + +backend uno_pool + balance leastconn + server-template uno 20 unoserver:2003 check resolvers docker init-addr none maxconn 1 diff --git a/docker/unoserver/deploy/kubernetes.yaml b/docker/unoserver/deploy/kubernetes.yaml new file mode 100644 index 0000000000..daf3f4ccd1 --- /dev/null +++ b/docker/unoserver/deploy/kubernetes.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: unoserver + labels: + app: unoserver +spec: + replicas: 5 + selector: + matchLabels: + app: unoserver + template: + metadata: + labels: + app: unoserver + spec: + terminationGracePeriodSeconds: 1800 + containers: + - name: unoserver + image: ghcr.io/stirling-tools/stirling-unoserver:latest + ports: + - containerPort: 2003 + name: rpc + env: + - name: UNOSERVER_PORT + value: "2003" + - name: UNOSERVER_CONVERSION_TIMEOUT + value: "1800" + - name: UNOSERVER_RECYCLE_INTERVAL_SECONDS + value: "3600" + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + memory: "2Gi" + readinessProbe: + exec: + command: ["sh", "-c", "unoping --host 127.0.0.1 --port 2003"] + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + livenessProbe: + exec: + command: ["sh", "-c", "unoping --host 127.0.0.1 --port 2003"] + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: unoserver +spec: + selector: + app: unoserver + ports: + - port: 2003 + targetPort: 2003 + name: rpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: stirling-pdf + labels: + app: stirling-pdf +spec: + replicas: 3 + selector: + matchLabels: + app: stirling-pdf + template: + metadata: + labels: + app: stirling-pdf + spec: + containers: + - name: stirling-pdf + image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest + ports: + - containerPort: 8080 + env: + - name: PROCESS_EXECUTOR_AUTO_UNO_SERVER + value: "false" + - name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST + value: "unoserver" + - name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT + value: "2003" + - name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION + value: "remote" + - name: PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY + value: "2" + - name: PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT + value: "2" + readinessProbe: + httpGet: + path: /api/v1/info/status + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 diff --git a/docker/unoserver/deploy/nginx.conf b/docker/unoserver/deploy/nginx.conf new file mode 100644 index 0000000000..d5f32d8e8d --- /dev/null +++ b/docker/unoserver/deploy/nginx.conf @@ -0,0 +1,29 @@ +worker_processes 1; +events { worker_connections 1024; } + +http { + upstream uno_pool { + least_conn; + server unoserver-1:2003 max_fails=1 fail_timeout=10s; + server unoserver-2:2003 max_fails=1 fail_timeout=10s; + server unoserver-3:2003 max_fails=1 fail_timeout=10s; + } + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + + server { + listen 2003; + + location / { + proxy_pass http://uno_pool; + + proxy_http_version 1.1; + proxy_connect_timeout 5s; + proxy_send_timeout 1830s; + proxy_read_timeout 1830s; + proxy_next_upstream off; + } + } +} diff --git a/testing/compose/docker-compose-multinode-unoserver.override.yml b/testing/compose/docker-compose-multinode-unoserver.override.yml new file mode 100644 index 0000000000..5550e50093 --- /dev/null +++ b/testing/compose/docker-compose-multinode-unoserver.override.yml @@ -0,0 +1,69 @@ +# Shared unoserver pool for the multi-node stack: every Stirling node converts through +# one HAProxy-fronted fleet instead of running its own LibreOffice. +# +# Bring up: docker compose -f docker-compose-multinode.yml -f docker-compose-multinode-unoserver.override.yml up -d --scale unoserver=3 +# Resize: docker compose ... up -d --scale unoserver=6 --no-recreate (nothing restarts) +# Watch: http://localhost:8404/ (queue depth, per-server sessions) +# +# Why a proxy rather than listing unoserver hosts on each node: unoserver converts +# strictly in series (measured: 1 job 6.0s, 2 jobs 10.5s, 4 jobs 20.8s against one +# instance) and queues without bound instead of rejecting. UnoServerPool's semaphore +# is per-JVM, so N nodes each believing they may run M conversions oversubscribes the +# fleet by N and the excess piles up invisibly inside unoserver. One shared queue in +# front is the only place that limit can actually be enforced. + +x-uno-endpoints: &uno-endpoints + PROCESS_EXECUTOR_AUTO_UNO_SERVER: "false" + # One entry for the whole fleet. concurrency is this node's in-flight cap, not a host + # count: real capacity is the replica count, and HAProxy queues the difference. + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST: "unoserver-lb" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_PORT: "2003" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_HOST_LOCATION: "remote" + PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_0_CONCURRENCY: "4" + # Matches the slot count so RuntimePathConfig does not log a mismatch warning. + PROCESS_EXECUTOR_SESSION_LIMIT_LIBRE_OFFICE_SESSION_LIMIT: "4" + +services: + stirling-1: + environment: *uno-endpoints + depends_on: + unoserver-lb: + condition: service_started + + stirling-2: + environment: *uno-endpoints + depends_on: + unoserver-lb: + condition: service_started + + # No container_name: that is what allows --scale. + unoserver: + image: ghcr.io/stirling-tools/stirling-unoserver:latest + environment: + UNOSERVER_PORT: "2003" + UNOSERVER_UNO_PORT: "2002" + UNOSERVER_CONVERSION_TIMEOUT: "1800" + # Hourly restart bounds LibreOffice memory growth. Safe to leave on here only + # because HAProxy health checks pull a recycling instance out of rotation. + UNOSERVER_RECYCLE_INTERVAL_SECONDS: "3600" + healthcheck: + # unoping is a real RPC round-trip: the TCP port stays bound while LibreOffice + # is wedged, so a port check would keep a dead instance in the pool. + test: ["CMD-SHELL", "unoping --host 127.0.0.1 --port 2003 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + networks: + - stirling-multinode + + unoserver-lb: + image: haproxy:3.0-alpine + volumes: + - ./multinode/haproxy-unoserver.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "8404:8404" # stats + /metrics + restart: unless-stopped + networks: + - stirling-multinode diff --git a/testing/compose/multinode/haproxy-unoserver.cfg b/testing/compose/multinode/haproxy-unoserver.cfg new file mode 100644 index 0000000000..1b5651a7a9 --- /dev/null +++ b/testing/compose/multinode/haproxy-unoserver.cfg @@ -0,0 +1,55 @@ +# Shared unoserver pool for a multi-node Stirling cluster. +# +# Every Stirling node points every one of its PROCESS_EXECUTOR_UNO_SERVER_ENDPOINTS_n +# entries at this one address, so the whole cluster draws from a single queue instead +# of each node guessing at capacity it cannot see. + +global + log stdout format raw local0 info + maxconn 4000 + +defaults + mode http + log global + option httplog + timeout connect 5s + # A conversion holds its connection open for the whole job, so these must exceed + # UNOSERVER_CONVERSION_TIMEOUT or HAProxy cuts jobs unoserver would have finished. + timeout client 1830s + timeout server 1830s + # Bounded wait for a free backend. Past this the caller gets 503 rather than + # queueing forever behind a fleet that is too small. + timeout queue 300s + +# Docker's embedded DNS. Re-resolved on the hold timers below, which is what lets +# `docker compose up --scale unoserver=N` change capacity with nothing restarted. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold other 5s + hold refused 5s + hold nx 5s + hold timeout 5s + +frontend uno_in + bind *:2003 + default_backend uno_pool + +# Stats page plus a Prometheus endpoint: queue depth and per-server session counts +# are the two numbers that say whether the pool is the bottleneck. +frontend uno_stats + bind *:8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri / + stats refresh 5s + +backend uno_pool + balance leastconn + # maxconn 1 because unoserver runs one LibreOffice and converts strictly in + # series: a second request does not run faster, it waits. Holding that queue + # here makes it bounded, measurable, and shared across every Stirling node. + server-template uno 20 unoserver:2003 check resolvers docker init-addr none maxconn 1 diff --git a/testing/compose/validate-unoserver-pool.sh b/testing/compose/validate-unoserver-pool.sh new file mode 100644 index 0000000000..a2c0e94f9e --- /dev/null +++ b/testing/compose/validate-unoserver-pool.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Proves the shared unoserver pool scales without restarting anything: capacity tracks +# the replica count, a killed instance leaves rotation, and the queue is observable. +# Run against a stack started with docker-compose-multinode-unoserver.override.yml. +# Non-destructive apart from killing one unoserver replica, which restarts on its own. +set -uo pipefail +cd "$(dirname "$0")" + +COMPOSE="docker compose -f docker-compose-multinode.yml -f docker-compose-multinode-unoserver.override.yml" +STATS="http://localhost:8404/metrics" +pass=0; fail=0 +ok() { echo " PASS - $*"; pass=$((pass+1)); } +bad() { echo " FAIL - $*"; fail=$((fail+1)); } + +# Servers HAProxy has resolved an address for and considers UP. +up_backends() { + curl -s "$STATS" \ + | awk -F'server="' '/haproxy_server_status\{proxy="uno_pool".*state="UP"\} 1$/ {split($2,a,"\""); print a[1]}' \ + | sort -u +} +up_count() { up_backends | grep -c . ; } + +wait_for_backends() { # $1 = expected count, $2 = seconds + for _ in $(seq 1 "$2"); do + [ "$(up_count)" -ge "$1" ] && return 0 + sleep 1 + done + return 1 +} + +echo "== 1. HAProxy discovered the running replicas ==" +replicas=$($COMPOSE ps -q unoserver | grep -c .) +if wait_for_backends "$replicas" 60; then + ok "$(up_count) backends UP for $replicas replicas: $(up_backends | paste -sd, -)" +else + bad "only $(up_count) backends UP, expected $replicas" +fi + +echo "== 2. Scaling up is picked up with nothing restarted ==" +lb_before=$($COMPOSE ps -q unoserver-lb) +target=$((replicas + 1)) +$COMPOSE up -d --scale unoserver=$target --no-recreate >/dev/null 2>&1 +if wait_for_backends "$target" 90; then + ok "scaled $replicas -> $target, HAProxy sees $(up_count)" +else + bad "HAProxy still sees $(up_count) after scaling to $target" +fi +[ "$lb_before" = "$($COMPOSE ps -q unoserver-lb)" ] \ + && ok "HAProxy container never restarted" \ + || bad "HAProxy was recreated (it should not need to be)" + +echo "== 3. A dead instance leaves rotation ==" +victim=$($COMPOSE ps -q unoserver | tail -1) +docker kill "$victim" >/dev/null 2>&1 +dropped=1 +for _ in $(seq 1 30); do + [ "$(up_count)" -lt "$target" ] && { dropped=0; break; } + sleep 1 +done +[ "$dropped" -eq 0 ] && ok "backend removed, $(up_count) left serving" \ + || bad "dead backend still marked UP after 30s" +docker start "$victim" >/dev/null 2>&1 + +echo "== 4. Queue depth is exported for alerting and autoscaling ==" +# Scraped into a variable rather than piped into grep -q: under pipefail an early +# grep exit makes curl fail on SIGPIPE and the check reports a false negative. +metrics=$(curl -s "$STATS") +q=$(printf '%s\n' "$metrics" | awk '/haproxy_backend_current_queue\{proxy="uno_pool"\}/ {print $2}') +if [ -n "$q" ]; then + ok "haproxy_backend_current_queue is exported (currently $q)" +else + bad "queue metric missing; is the prometheus-exporter line in haproxy-unoserver.cfg?" +fi + +echo +echo "passed: $pass failed: $fail" +[ "$fail" -eq 0 ]