fix(valkey): skip boot handshake backoff after the final attempt

.github/workflows/build-enterprise.yml

@@ -330,8 +330,8 @@ jobs:
           rm -f /tmp/helpers.sh /tmp/backend.log /tmp/backend.pid
         continue-on-error: true
 
-  # Multi-node regression: builds + seeds the clustered stack (testing/compose/docker-compose-multinode.yml)
-  # and runs behave features/multinode. Licence-gated, so it runs after the Playwright job (not in parallel).
+  # Multi-node regression: builds + seeds the clustered stack once per Valkey topology and runs behave
+  # features/multinode. Licence-gated, so it runs after the Playwright job (not in parallel).
   multinode-e2e:
     environment:
       name: ci-unsigned
@@ -341,14 +341,22 @@ jobs:
     if: >-
       always() && needs.pick.outputs.is_fork != 'true'
       && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
-    runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
+    # Depot is disabled repo-wide, so the depot-* class never gets a runner and the job dies unassigned.
+    runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'ubuntu-24.04-8core' }}
     timeout-minutes: 60
+    strategy:
+      # One leg per Valkey topology. fail-fast off so a sentinel break still reports cluster.
+      fail-fast: false
+      matrix:
+        valkey: [standalone, sentinel, cluster]
     env:
       PREMIUM_KEY: ${{ secrets.PREMIUM_KEY_ENTERPRISE }}
       PREMIUM_ENABLED: "true"
       SYSTEM_ENABLEANALYTICS: "false"
       DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
-      MN_COMPOSE: docker-compose-multinode.yml
+      # Unquoted at every use site so it word-splits into repeated -f flags. The topology overlay
+      # must come last: it overrides valkey.command and compose REPLACES command.
+      MN_FILES: -f docker-compose-multinode.yml ${{ matrix.valkey != 'standalone' && format('-f docker-compose-multinode.valkey-{0}.yml', matrix.valkey) || '' }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0
@@ -368,11 +376,11 @@ jobs:
           uv sync --project engine --locked --group cucumber
       - name: Build the multi-node image
         working-directory: testing/compose
-        run: docker compose -f "$MN_COMPOSE" build
+        run: docker compose $MN_FILES build
       - name: Bring up the cluster and wait for both nodes healthy
         working-directory: testing/compose
         run: |
-          docker compose -f "$MN_COMPOSE" up -d
+          docker compose $MN_FILES up -d
           for i in $(seq 1 90); do
             h1=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-1 2>/dev/null || echo starting)
             h2=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-2 2>/dev/null || echo starting)
@@ -380,11 +388,11 @@ jobs:
             sleep 5
           done
           echo "::error::nodes did not become healthy"
-          docker compose -f "$MN_COMPOSE" logs --tail=200 stirling-1 stirling-2
+          docker compose $MN_FILES logs --tail=200 stirling-1 stirling-2
           exit 1
       - name: Seed the cluster (teams, users, S3 connection, policy)
         working-directory: testing/compose
-        run: docker compose -f "$MN_COMPOSE" --profile seed run --rm seed
+        run: docker compose $MN_FILES --profile seed run --rm seed
       - name: Run multi-node regression (implemented guarantees)
         working-directory: testing/cucumber
         # -e overrides behave.ini's exclusion of features/multinode; ~@known_gap skips any tracked-gap scenarios.
@@ -395,8 +403,8 @@ jobs:
       - name: Dump node logs on failure
         if: failure()
         working-directory: testing/compose
-        run: docker compose -f "$MN_COMPOSE" logs --tail=400 stirling-1 stirling-2
+        run: docker compose $MN_FILES logs --tail=400 stirling-1 stirling-2
       - name: Tear down
         if: always()
         working-directory: testing/compose
-        run: docker compose -f "$MN_COMPOSE" --profile seed down -v --remove-orphans
+        run: docker compose $MN_FILES --profile seed down -v --remove-orphans

app/common/src/main/java/stirling/software/common/cluster/ClusterConfig.java

@@ -1,29 +1,87 @@
 package stirling.software.common.cluster;
 
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
 
 import jakarta.annotation.PostConstruct;
 
-import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.ApplicationProperties.Cluster;
+import stirling.software.common.util.GeneralUtils;
 
-/**
- * Validates that cluster mode is internally consistent.
- *
- * <p>Cluster settings are bound on the central {@link ApplicationProperties} under {@code
- * cluster.*}; this class reads {@link ApplicationProperties#getCluster()} and runs guards in {@link
- * PostConstruct}. When {@code cluster.enabled=false} (the default) all checks are skipped so a
- * single-instance install needs no new config.
- */
+/** Validates cluster config consistency. All guards are skipped when cluster.enabled=false. */
 @Slf4j
 @Configuration
-@RequiredArgsConstructor
 public class ClusterConfig {
 
+    private static final String MISSING_URL_MESSAGE =
+            "cluster.enabled=true with backplane=valkey requires"
+                    + " cluster.valkey.url to be set (e.g."
+                    + " redis://valkey:6379).";
+
+    private static final String SHARED_SECRET_MESSAGE_SUFFIX =
+            " must be set to the same UUID on every node when cluster.enabled=true (env"
+                    + " AUTOMATICALLYGENERATED_KEY and AUTOMATICALLYGENERATED_UUID). Otherwise every"
+                    + " node mints its own at first boot, so workflow metadata encrypted on one"
+                    + " node cannot be decrypted on another and licence seat signatures do not"
+                    + " verify across nodes. The value must be a UUID; anything else (the shipped"
+                    + " 'example' placeholder included) is replaced by a per-node random UUID.";
+
+    /** Default bean name of stirling.software.SPDF.config.InitialSetup, which lives in :core. */
+    private static final String INITIAL_SETUP_BEAN = "initialSetup";
+
     private final ApplicationProperties applicationProperties;
+    private final String automaticallyGeneratedKey;
+    private final String automaticallyGeneratedUuid;
+
+    // Read from config, not from ApplicationProperties: InitialSetup overwrites the bound values
+    // with per-node UUIDs in its own @PostConstruct, which would defeat the guard below.
+    public ClusterConfig(
+            ApplicationProperties applicationProperties,
+            @Value("${AutomaticallyGenerated.key:}") String automaticallyGeneratedKey,
+            @Value("${AutomaticallyGenerated.UUID:}") String automaticallyGeneratedUuid) {
+        this.applicationProperties = applicationProperties;
+        this.automaticallyGeneratedKey = automaticallyGeneratedKey;
+        this.automaticallyGeneratedUuid = automaticallyGeneratedUuid;
+    }
+
+    // InitialSetup's @PostConstruct usually wins the race against ours and would persist a
+    // per-node UUID before we can refuse; a BeanFactoryPostProcessor runs before either of them.
+    @Bean
+    static BeanFactoryPostProcessor clusterSharedSecretGuard(Environment environment) {
+        return beanFactory -> {
+            // InitialSetup is the only thing that mints per-node UUIDs; without it there is
+            // nothing to pre-empt, and slice tests that wire ClusterConfig alone stay usable.
+            if (!beanFactory.containsBeanDefinition(INITIAL_SETUP_BEAN)) {
+                return;
+            }
+            validateSharedCryptoMaterial(
+                    environment.getProperty("cluster.enabled", Boolean.class, false),
+                    environment.getProperty("AutomaticallyGenerated.key", ""),
+                    environment.getProperty("AutomaticallyGenerated.UUID", ""));
+        };
+    }
+
+    /** Cluster nodes derive metadata encryption and licence HMAC keys from these two values. */
+    static void validateSharedCryptoMaterial(boolean clusterEnabled, String key, String uuid) {
+        if (!clusterEnabled) {
+            return;
+        }
+        requireSharedUuid("AutomaticallyGenerated.key", key);
+        requireSharedUuid("AutomaticallyGenerated.UUID", uuid);
+    }
+
+    // InitialSetup replaces any non-UUID value, so only a valid UUID survives startup unchanged.
+    private static void requireSharedUuid(String property, String value) {
+        if (!GeneralUtils.isValidUUID(value)) {
+            throw new IllegalStateException(property + SHARED_SECRET_MESSAGE_SUFFIX);
+        }
+    }
 
     @PostConstruct
     void validate() {
@@ -31,15 +89,21 @@ public class ClusterConfig {
         if (!cluster.isEnabled()) {
             return;
         }
+        validateSharedCryptoMaterial(true, automaticallyGeneratedKey, automaticallyGeneratedUuid);
         String backplane = cluster.getBackplane();
         if ("valkey".equalsIgnoreCase(backplane)) {
-            String url = cluster.getValkey() == null ? null : cluster.getValkey().getUrl();
-            if (url == null || url.isBlank()) {
-                throw new IllegalStateException(
-                        "cluster.enabled=true with backplane=valkey requires"
-                                + " cluster.valkey.url to be set (e.g."
-                                + " redis://valkey:6379).");
+            // getValkey() re-seeds a null block, so an absent 'valkey:' reads as a missing url.
+            ApplicationProperties.Cluster.Valkey valkey = cluster.getValkey();
+            // resolvedMode() throws on an unknown/ambiguous mode; let it propagate so the
+            // operator sees the property name rather than a later missing-bean error.
+            ApplicationProperties.Cluster.Valkey.ValkeyMode mode = valkey.resolvedMode();
+            validateModeConsistency(valkey, mode);
+            switch (mode) {
+                case STANDALONE -> validateStandalone(valkey);
+                case SENTINEL -> validateSentinel(valkey);
+                case CLUSTER -> validateCluster(valkey);
             }
+            validateCommon(valkey, mode);
         } else if ("inprocess".equalsIgnoreCase(backplane)) {
             // enabled+inprocess only coordinates the local JVM; cross-node lookups will 410.
             log.warn(
@@ -55,9 +119,135 @@ public class ClusterConfig {
                             + "'. Valid values: inprocess | valkey.");
         }
         log.info(
-                "Cluster mode enabled (backplane={}, nodeRole={}, nodeId={}).",
+                "Cluster mode enabled (backplane={}, valkeyMode={}, nodeRole={}, nodeId={}).",
                 backplane,
+                "valkey".equalsIgnoreCase(backplane) ? cluster.getValkey().resolvedMode() : "n/a",
                 cluster.resolvedRole(),
                 cluster.resolvedNodeId());
     }
+
+    private static void validateStandalone(ApplicationProperties.Cluster.Valkey valkey) {
+        String url = valkey.getUrl();
+        if (url == null || url.isBlank()) {
+            throw new IllegalStateException(MISSING_URL_MESSAGE);
+        }
+    }
+
+    /**
+     * Endpoint lists are only read by their own mode. Without this an operator who sets the nodes
+     * but forgets the mode selector silently connects to whatever {@code cluster.valkey.url} holds.
+     */
+    private static void validateModeConsistency(
+            ApplicationProperties.Cluster.Valkey valkey,
+            ApplicationProperties.Cluster.Valkey.ValkeyMode mode) {
+        var sentinel = valkey.getSentinel();
+        boolean sentinelNodesSet = !sentinel.getNodes().isEmpty();
+        if (sentinelNodesSet && mode != ApplicationProperties.Cluster.Valkey.ValkeyMode.SENTINEL) {
+            throw new IllegalStateException(
+                    "cluster.valkey.sentinel.nodes is set but the resolved mode is "
+                            + mode
+                            + ", so the sentinel list is ignored and the client would connect to"
+                            + " cluster.valkey.url instead. Set cluster.valkey.sentinel.master (the"
+                            + " monitored primary name, e.g. mymaster) or"
+                            + " cluster.valkey.mode=sentinel.");
+        }
+        if (!valkey.getNodes().isEmpty()
+                && mode != ApplicationProperties.Cluster.Valkey.ValkeyMode.CLUSTER) {
+            throw new IllegalStateException(
+                    "cluster.valkey.nodes is set but the resolved mode is "
+                            + mode
+                            + ", so the seed node list is ignored. Set"
+                            + " cluster.valkey.mode=cluster, or remove cluster.valkey.nodes if this"
+                            + " deployment is not a Valkey Cluster.");
+        }
+    }
+
+    private static void validateSentinel(ApplicationProperties.Cluster.Valkey valkey) {
+        var sentinel = valkey.getSentinel();
+        if (sentinel.getMaster() == null || sentinel.getMaster().isBlank()) {
+            throw new IllegalStateException(
+                    "cluster.valkey.mode=sentinel requires cluster.valkey.sentinel.master to be"
+                            + " set (the monitored primary name, e.g. mymaster).");
+        }
+        if (sentinel.getNodes().isEmpty()) {
+            throw new IllegalStateException(
+                    "cluster.valkey.mode=sentinel requires cluster.valkey.sentinel.nodes to list"
+                            + " at least one sentinel (e.g."
+                            + " sentinel-1:26379,sentinel-2:26379,sentinel-3:26379).");
+        }
+        for (String entry : sentinel.getNodes()) {
+            HostPort.parse(entry, "cluster.valkey.sentinel.nodes", "sentinel-1:26379");
+        }
+        // Sentinel AUTH is separate from data-node AUTH; only warn, some sentinels are open.
+        if ((sentinel.getPassword() == null || sentinel.getPassword().isBlank())
+                && valkey.getPassword() != null
+                && !valkey.getPassword().isBlank()) {
+            log.warn(
+                    "cluster.valkey.password is set but cluster.valkey.sentinel.password is not."
+                            + " Sentinel connections authenticate separately; if your sentinels"
+                            + " require AUTH, set cluster.valkey.sentinel.password too.");
+        }
+    }
+
+    private static void validateCluster(ApplicationProperties.Cluster.Valkey valkey) {
+        if (valkey.getNodes().isEmpty()) {
+            throw new IllegalStateException(
+                    "cluster.valkey.mode=cluster requires cluster.valkey.nodes to list at least"
+                            + " one seed node (e.g. valkey-1:6379,valkey-2:6379,valkey-3:6379).");
+        }
+        for (String entry : valkey.getNodes()) {
+            HostPort.parse(entry, "cluster.valkey.nodes", "valkey-1:6379");
+        }
+        if (valkey.getMaxRedirects() < 1) {
+            throw new IllegalStateException(
+                    "cluster.valkey.maxRedirects must be >= 1 in cluster mode; got "
+                            + valkey.getMaxRedirects()
+                            + ".");
+        }
+        // Lettuce rejects a non-positive refresh period with an opaque assertion at boot.
+        if (valkey.getTopologyRefreshMs() <= 0) {
+            throw new IllegalStateException(
+                    "cluster.valkey.topologyRefreshMs must be > 0 in cluster mode; got "
+                            + valkey.getTopologyRefreshMs()
+                            + ".");
+        }
+    }
+
+    private static void validateCommon(
+            ApplicationProperties.Cluster.Valkey valkey,
+            ApplicationProperties.Cluster.Valkey.ValkeyMode mode) {
+        var pool = valkey.getPool();
+        if (pool.isEnabled() && pool.getMaxActive() < 2) {
+            throw new IllegalStateException(
+                    "cluster.valkey.pool.maxActive must be >= 2 when pooling is enabled (one"
+                            + " connection is permanently held by the shared native connection);"
+                            + " got "
+                            + pool.getMaxActive()
+                            + ".");
+        }
+        if (pool.isEnabled() && pool.getMaxWaitMillis() <= 0) {
+            throw new IllegalStateException(
+                    "cluster.valkey.pool.maxWaitMillis must be > 0 (a negative value blocks"
+                            + " forever, which defeats cluster.valkey.commandTimeoutMs, and 0"
+                            + " fails the borrow instantly once the pool is exhausted); got "
+                            + pool.getMaxWaitMillis()
+                            + ".");
+        }
+        if (valkey.getCommandTimeoutMs() <= 0) {
+            throw new IllegalStateException(
+                    "cluster.valkey.commandTimeoutMs must be > 0; got "
+                            + valkey.getCommandTimeoutMs()
+                            + ".");
+        }
+        if (mode != ApplicationProperties.Cluster.Valkey.ValkeyMode.STANDALONE
+                && valkey.getUrl() != null
+                && !valkey.getUrl().isBlank()) {
+            log.info(
+                    "cluster.valkey.url is ignored in mode={} (endpoints come from {}).",
+                    mode,
+                    mode == ApplicationProperties.Cluster.Valkey.ValkeyMode.SENTINEL
+                            ? "cluster.valkey.sentinel.nodes"
+                            : "cluster.valkey.nodes");
+        }
+    }
 }

app/common/src/main/java/stirling/software/common/cluster/HostPort.java

@@ -0,0 +1,84 @@
+package stirling.software.common.cluster;
+
+/**
+ * One {@code host:port} entry from a Valkey cluster or sentinel node list. Shared by config
+ * validation and by the connection builder so the two can never disagree on what a valid entry is.
+ */
+public record HostPort(String host, int port) {
+
+    /**
+     * Port is always explicit: a bare host would silently connect somewhere the operator never
+     * named. IPv6 literals must be bracketed ({@code [::1]:6379}).
+     *
+     * @throws IllegalStateException naming {@code propertyName} and echoing {@code entry}
+     */
+    public static HostPort parse(String entry, String propertyName, String example) {
+        String trimmed = entry == null ? "" : entry.trim();
+        if (trimmed.isEmpty()) {
+            throw new IllegalStateException(
+                    propertyName
+                            + " contains a blank entry (expected host:port, e.g. "
+                            + example
+                            + ").");
+        }
+        if (trimmed.charAt(0) == '[') {
+            return parseBracketed(trimmed, entry, propertyName, example);
+        }
+        int colon = trimmed.lastIndexOf(':');
+        if (colon < 0) {
+            throw reject(entry, propertyName, example, "is not host:port");
+        }
+        if (trimmed.indexOf(':') != colon) {
+            throw reject(
+                    entry,
+                    propertyName,
+                    example,
+                    "has more than one ':' - bracket IPv6 literals as [::1]:6379");
+        }
+        return build(
+                trimmed.substring(0, colon),
+                trimmed.substring(colon + 1),
+                entry,
+                propertyName,
+                example);
+    }
+
+    /** Handles {@code [::1]:6379}; the returned host keeps no brackets. */
+    private static HostPort parseBracketed(
+            String trimmed, String entry, String propertyName, String example) {
+        int close = trimmed.indexOf(']');
+        if (close < 0) {
+            throw reject(entry, propertyName, example, "has an unclosed '['");
+        }
+        String rest = trimmed.substring(close + 1);
+        if (rest.isEmpty() || rest.charAt(0) != ':') {
+            throw reject(entry, propertyName, example, "is not host:port");
+        }
+        return build(trimmed.substring(1, close), rest.substring(1), entry, propertyName, example);
+    }
+
+    private static HostPort build(
+            String host, String rawPort, String entry, String propertyName, String example) {
+        int port;
+        try {
+            port = Integer.parseInt(rawPort);
+        } catch (NumberFormatException ex) {
+            throw new IllegalStateException(
+                    message(entry, propertyName, example, "is not host:port"), ex);
+        }
+        if (host.isBlank() || port < 1 || port > 65535) {
+            throw reject(entry, propertyName, example, "is not host:port");
+        }
+        return new HostPort(host, port);
+    }
+
+    private static IllegalStateException reject(
+            String entry, String propertyName, String example, String problem) {
+        return new IllegalStateException(message(entry, propertyName, example, problem));
+    }
+
+    private static String message(
+            String entry, String propertyName, String example, String problem) {
+        return propertyName + " entry '" + entry + "' " + problem + " (e.g. " + example + ").";
+    }
+}

app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java

@@ -5,6 +5,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -537,11 +538,7 @@ public class ApplicationProperties {
         }
     }
 
-    /**
-     * Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
-     * (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
-     * off; when off the in-process backplane is wired and no other cluster keys are required.
-     */
+    /** Cluster backplane config, bound under the top-level {@code cluster.*} prefix. */
     @Data
     public static class Cluster {
 
@@ -552,20 +549,24 @@ public class ApplicationProperties {
         private String backplane = "inprocess";
 
         /**
-         * Transient cluster job-artifact store selector. Valid values: {@code local} | {@code s3}.
-         *
-         * <p>This is distinct from {@code storage.provider}, which selects the backend for
-         * persistent user-uploaded files. The two switches exist because the user-facing storage
-         * feature is optional ({@code storage.enabled=false} is common) but every multi-node
-         * cluster still needs a shared artifact store to serve cross-node downloads. Both
-         * implementations share credentials from {@code storage.s3.*} when set to {@code s3}.
+         * {@code local} | {@code s3}. Distinct from {@code storage.provider} (persistent uploads);
+         * shares the {@code storage.s3.*} credentials when set to {@code s3}.
          */
         private String artifactStore = "local";
 
         private Valkey valkey = new Valkey();
         private Node node = new Node();
 
+        // A bare 'valkey:' key in settings.yml binds null; re-seed rather than hand one back.
+        public Valkey getValkey() {
+            if (valkey == null) {
+                valkey = new Valkey();
+            }
+            return valkey;
+        }
+
         private transient String cachedNodeId;
+        private transient String cachedNodeName;
 
         public NodeRole resolvedRole() {
             if (node == null || node.getRole() == null) {
@@ -589,6 +590,37 @@ public class ApplicationProperties {
             return cachedNodeId;
         }
 
+        /**
+         * Stable per-node label for CLIENT SETNAME: {@code cluster.node.id}, else hostname, else
+         * {@link #resolvedNodeId()}. The first two survive a restart; the UUID fallback does not.
+         */
+        public synchronized String resolvedNodeName() {
+            if (node != null && node.getId() != null && !node.getId().isBlank()) {
+                return node.getId();
+            }
+            if (cachedNodeName == null) {
+                cachedNodeName = localHostname();
+            }
+            return cachedNodeName != null ? cachedNodeName : resolvedNodeId();
+        }
+
+        // Hostname resolution depends on DNS and can throw; a missing name must never fail startup.
+        private static String localHostname() {
+            String env = java.lang.System.getenv("HOSTNAME");
+            if (env == null || env.isBlank()) {
+                env = java.lang.System.getenv("COMPUTERNAME");
+            }
+            if (env != null && !env.isBlank()) {
+                return env.trim();
+            }
+            try {
+                String host = InetAddress.getLocalHost().getHostName();
+                return host != null && !host.isBlank() ? host.trim() : null;
+            } catch (Exception ex) {
+                return null;
+            }
+        }
+
         public enum NodeRole {
             WEB,
             WORKER,
@@ -598,21 +630,201 @@ public class ApplicationProperties {
         @Data
         public static class Valkey {
             /**
-             * {@code redis://host:6379} or {@code rediss://...} for TLS. Required when cluster mode
-             * is on and backplane is valkey.
+             * {@code redis://} or {@code rediss://} URL; read ONLY in standalone mode. Excluded
+             * from toString because it can carry userinfo credentials.
              */
-            private String url = "";
+            @ToString.Exclude private String url = "";
 
+            /**
+             * {@code standalone} | {@code sentinel} | {@code cluster}; blank auto-resolves (see
+             * {@link #resolvedMode()}). {@code url} is read only in standalone mode.
+             */
+            private String mode = "";
+
+            /** Data-node username; overrides any userinfo in {@link #url}. */
+            private String username = "";
+
+            /** Data-node password; overrides any userinfo in {@link #url}. */
+            @ToString.Exclude private String password = "";
+
+            /** Valkey Cluster seed nodes as {@code host:port}. Read only when mode is cluster. */
+            private List<String> nodes = new ArrayList<>();
+
+            /** Max MOVED/ASK redirects the cluster client follows before failing a command. */
+            private int maxRedirects = 3;
+
+            /**
+             * Periodic cluster topology refresh interval in milliseconds. Adaptive refresh on
+             * MOVED/ASK/reconnect is always on; this is the backstop when no redirect is seen.
+             */
+            private long topologyRefreshMs = 30000;
+
+            /**
+             * CLIENT SETNAME applied to every connection so Valkey monitoring can attribute load to
+             * a node. Blank (default) = {@code stirling-} + {@code Cluster.resolvedNodeName()}.
+             */
+            private String clientName = "";
+
+            /**
+             * Per-command timeout in milliseconds. Bounds every backplane call so a slow or
+             * partitioned Valkey cannot stall request threads.
+             */
+            private long commandTimeoutMs = 2000;
+
+            private Sentinel sentinel = new Sentinel();
             private Tls tls = new Tls();
+            private Pool pool = new Pool();
+
+            // A bare 'sentinel:'/'tls:'/'pool:'/'nodes:' key in settings.yml binds null. Re-seed
+            // the default here so no call site has to guard, and none can forget to.
+            public Sentinel getSentinel() {
+                if (sentinel == null) {
+                    sentinel = new Sentinel();
+                }
+                return sentinel;
+            }
+
+            public Tls getTls() {
+                if (tls == null) {
+                    tls = new Tls();
+                }
+                return tls;
+            }
+
+            public Pool getPool() {
+                if (pool == null) {
+                    pool = new Pool();
+                }
+                return pool;
+            }
+
+            public List<String> getNodes() {
+                if (nodes == null) {
+                    nodes = new ArrayList<>();
+                }
+                return nodes;
+            }
+
+            /**
+             * Explicit {@link #mode} wins; blank infers SENTINEL from sentinel.master, CLUSTER from
+             * nodes, else STANDALONE, and throws when both are set (ambiguous).
+             */
+            public ValkeyMode resolvedMode() {
+                if (mode != null && !mode.isBlank()) {
+                    try {
+                        return ValkeyMode.valueOf(mode.trim().toUpperCase(Locale.ROOT));
+                    } catch (IllegalArgumentException ex) {
+                        throw new IllegalStateException(
+                                "cluster.valkey.mode has unknown value '"
+                                        + mode
+                                        + "'. Valid values: standalone | sentinel | cluster.",
+                                ex);
+                    }
+                }
+                String master = getSentinel().getMaster();
+                boolean sentinelConfigured = master != null && !master.isBlank();
+                boolean clusterConfigured = !getNodes().isEmpty();
+                if (sentinelConfigured && clusterConfigured) {
+                    throw new IllegalStateException(
+                            "cluster.valkey.mode is not set but both"
+                                    + " cluster.valkey.sentinel.master and cluster.valkey.nodes are"
+                                    + " configured. Set cluster.valkey.mode explicitly to"
+                                    + " 'sentinel' or 'cluster'.");
+                }
+                if (sentinelConfigured) {
+                    return ValkeyMode.SENTINEL;
+                }
+                if (clusterConfigured) {
+                    return ValkeyMode.CLUSTER;
+                }
+                return ValkeyMode.STANDALONE;
+            }
+
+            public enum ValkeyMode {
+                STANDALONE,
+                SENTINEL,
+                CLUSTER
+            }
+
+            @Data
+            public static class Sentinel {
+                /** Monitored primary name, i.e. the name in {@code sentinel monitor <name> ...}. */
+                private String master = "";
+
+                /** Sentinel endpoints as {@code host:port}; sentinel's default port is 26379. */
+                private List<String> nodes = new ArrayList<>();
+
+                /** Username for the SENTINEL connections. Separate from the data-node username. */
+                private String username = "";
+
+                /**
+                 * Password for the SENTINEL connections. Separate from the data-node password -
+                 * setting only {@code cluster.valkey.password} does NOT authenticate to sentinels.
+                 */
+                @ToString.Exclude private String password = "";
+
+                // A bare 'nodes:' key binds null.
+                public List<String> getNodes() {
+                    if (nodes == null) {
+                        nodes = new ArrayList<>();
+                    }
+                    return nodes;
+                }
+            }
 
             @Data
             public static class Tls {
+                /**
+                 * Force TLS. Required in sentinel/cluster mode, which have no {@code rediss://} URL
+                 * to carry the scheme; in standalone it is OR-ed with the scheme, never overridden.
+                 */
+                private boolean enabled = false;
+
                 /**
                  * When {@code true}, skip Valkey/Redis TLS certificate verification (dev/test
                  * only). Leave {@code false} in production.
                  */
                 private boolean skipCertVerification = false;
             }
+
+            @Data
+            public static class Pool {
+                /**
+                 * Pooling for dedicated connections. Backplane traffic multiplexes over the shared
+                 * native connection, so the pool backs only that one connection today.
+                 */
+                private boolean enabled = true;
+
+                /**
+                 * Max pooled connections; at least 2, one is held by the shared native connection.
+                 * Headroom for a future dedicated path - raising it changes no current throughput.
+                 */
+                private int maxActive = 16;
+
+                /** Max idle connections kept in the pool. Keep equal to maxActive. */
+                private int maxIdle = 16;
+
+                /**
+                 * Connections kept warm. Default 0: backplane traffic runs on the shared native
+                 * connection, so warm pooled sockets would idle unused on every node.
+                 */
+                private int minIdle = 0;
+
+                /**
+                 * Max wait for a pooled connection. Never 0/negative: negative blocks forever and
+                 * defeats commandTimeoutMs, 0 fails the borrow instantly once the pool is drained.
+                 */
+                private long maxWaitMillis = 2000;
+
+                /** Idle-evictor interval. minIdle is only honoured while the evictor runs. */
+                private long timeBetweenEvictionRunsMillis = 30000;
+
+                /**
+                 * Validate on borrow. Cheap (no round trip - Lettuce checks isOpen) but it only
+                 * rejects explicitly closed connections; a disconnected, reconnecting one is open.
+                 */
+                private boolean testOnBorrow = true;
+            }
         }
 
         @Data

app/common/src/test/java/stirling/software/common/cluster/ClusterConfigValidationTest.java

@@ -1,21 +1,43 @@
 package stirling.software.common.cluster;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.lang.reflect.Method;
+import java.util.List;
 
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
 
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.ApplicationProperties.Cluster;
+import stirling.software.common.model.ApplicationProperties.Cluster.Valkey;
 
 class ClusterConfigValidationTest {
 
+    /** Stand-ins for the shared AutomaticallyGenerated values every node must be given. */
+    private static final String SHARED_KEY = "11111111-1111-1111-1111-111111111111";
+
+    private static final String SHARED_UUID = "22222222-2222-2222-2222-222222222222";
+
+    private static ClusterConfig config(ApplicationProperties props) {
+        return new ClusterConfig(props, SHARED_KEY, SHARED_UUID);
+    }
+
     @Test
     void validationPassesWhenDisabled() {
         ApplicationProperties props = new ApplicationProperties();
-        ClusterConfig config = new ClusterConfig(props);
+        ClusterConfig config = config(props);
         assertDoesNotThrow(() -> invokeValidate(config));
     }
 
@@ -25,18 +47,19 @@ class ClusterConfigValidationTest {
         Cluster cluster = props.getCluster();
         cluster.setEnabled(true);
         cluster.setBackplane("valkey");
-        ClusterConfig config = new ClusterConfig(props);
+        ClusterConfig config = config(props);
         assertThrows(IllegalStateException.class, () -> invokeValidate(config));
     }
 
     @Test
+    @DisplayName("backward compatibility: url only, no new keys, still validates")
     void validationPassesWhenValkeyEnabledWithUrl() {
         ApplicationProperties props = new ApplicationProperties();
         Cluster cluster = props.getCluster();
         cluster.setEnabled(true);
         cluster.setBackplane("valkey");
         cluster.getValkey().setUrl("redis://localhost:6379");
-        ClusterConfig config = new ClusterConfig(props);
+        ClusterConfig config = config(props);
         assertDoesNotThrow(() -> invokeValidate(config));
     }
 
@@ -46,10 +69,450 @@ class ClusterConfigValidationTest {
         Cluster cluster = props.getCluster();
         cluster.setEnabled(true);
         cluster.setBackplane("inprocess");
-        ClusterConfig config = new ClusterConfig(props);
+        ClusterConfig config = config(props);
         assertDoesNotThrow(() -> invokeValidate(config));
     }
 
+    /** Rules V1-V13 of the topology spec. Each asserts the exact operator-facing wording. */
+    @Nested
+    @DisplayName("valkey topology + pool validation")
+    class TopologyValidation {
+
+        private ApplicationProperties props;
+
+        @Test
+        @DisplayName("V1: unknown mode names the bad value and the valid set")
+        void unknownModeRejected() {
+            Valkey v = valkeyProps();
+            v.setMode("clustr");
+            assertMessage("clustr", "standalone | sentinel | cluster");
+        }
+
+        @Test
+        @DisplayName("V2: blank mode with both sentinel.master and nodes is ambiguous")
+        void ambiguousModeRejected() {
+            Valkey v = valkeyProps();
+            v.getSentinel().setMaster("mymaster");
+            v.setNodes(List.of("valkey-1:6379"));
+            assertMessage("Set cluster.valkey.mode explicitly");
+        }
+
+        @Test
+        @DisplayName("V3: standalone without a url keeps the original message (test contract)")
+        void standaloneWithoutUrlRejected() {
+            valkeyProps();
+            assertMessage(
+                    "cluster.enabled=true with backplane=valkey requires",
+                    "cluster.valkey.url to be set",
+                    "redis://valkey:6379");
+        }
+
+        @Test
+        @DisplayName("V4: sentinel mode without a master name is rejected")
+        void sentinelWithoutMasterRejected() {
+            Valkey v = valkeyProps();
+            v.setMode("sentinel");
+            v.getSentinel().setNodes(List.of("sentinel-1:26379"));
+            assertMessage("cluster.valkey.sentinel.master to be set", "mymaster");
+        }
+
+        @Test
+        @DisplayName("V5: sentinel mode without any sentinel endpoints is rejected")
+        void sentinelWithoutNodesRejected() {
+            Valkey v = valkeyProps();
+            v.setMode("sentinel");
+            v.getSentinel().setMaster("mymaster");
+            assertMessage("cluster.valkey.sentinel.nodes to list at least one sentinel");
+        }
+
+        @Test
+        @DisplayName("V6: cluster mode without any seed nodes is rejected")
+        void clusterWithoutNodesRejected() {
+            Valkey v = valkeyProps();
+            v.setMode("cluster");
+            assertMessage("cluster.valkey.nodes to list at least one seed node");
+        }
+
+        @Test
+        @DisplayName("V7: a nodes entry that is not host:port is rejected, echoing the entry")
+        void clusterNodeEntryMustBeHostPort() {
+            Valkey v = valkeyProps();
+            v.setMode("cluster");
+            v.setNodes(List.of("valkey-1:6379", "valkey-2"));
+            assertMessage("cluster.valkey.nodes entry", "valkey-2", "is not host:port");
+        }
+
+        @ParameterizedTest
+        @ValueSource(strings = {"valkey-1:notaport", "valkey-1:70000", ":6379", "valkey-1:"})
+        @DisplayName("V7: a non-numeric or out-of-range port is rejected")
+        void clusterNodePortMustBeNumericAndInRange(String entry) {
+            Valkey v = valkeyProps();
+            v.setMode("cluster");
+            v.setNodes(List.of(entry));
+            assertMessage(entry, "is not host:port");
+        }
+
+        @Test
+        @DisplayName("V7: a sentinel.nodes entry that is not host:port is rejected")
+        void sentinelNodeEntryMustBeHostPort() {
+            Valkey v = valkeyProps();
+            v.setMode("sentinel");
+            v.getSentinel().setMaster("mymaster");
+            v.getSentinel().setNodes(List.of("sentinel-1"));
+            assertMessage("cluster.valkey.sentinel.nodes entry", "sentinel-1", "is not host:port");
+        }
+
+        @Test
+        @DisplayName("V8: pool.maxActive < 2 is rejected (the shared connection holds one)")
+        void poolMaxActiveMustLeaveRoomForSharedConnection() {
+            Valkey v = validStandalone();
+            v.getPool().setMaxActive(1);
+            assertMessage("cluster.valkey.pool.maxActive must be >= 2", "got 1");
+        }
+
+        @Test
+        @DisplayName("V8: maxActive < 2 is allowed when pooling is off (the check is pool-scoped)")
+        void poolMaxActiveIgnoredWhenPoolingDisabled() {
+            Valkey v = validStandalone();
+            v.getPool().setEnabled(false);
+            v.getPool().setMaxActive(1);
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("V9: pool.maxWaitMillis <= 0 is rejected (0 fails borrows, negative blocks)")
+        void poolMaxWaitMustBePositive() {
+            Valkey v = validStandalone();
+            v.getPool().setMaxWaitMillis(0);
+            assertMessage("cluster.valkey.pool.maxWaitMillis must be > 0", "got 0");
+        }
+
+        @Test
+        @DisplayName("V10: commandTimeoutMs <= 0 is rejected")
+        void commandTimeoutMustBePositive() {
+            Valkey v = validStandalone();
+            v.setCommandTimeoutMs(0);
+            assertMessage("cluster.valkey.commandTimeoutMs must be > 0", "got 0");
+        }
+
+        @Test
+        @DisplayName("V11: maxRedirects < 1 is rejected in cluster mode")
+        void maxRedirectsMustBeAtLeastOneInClusterMode() {
+            Valkey v = valkeyProps();
+            v.setMode("cluster");
+            v.setNodes(List.of("valkey-1:6379"));
+            v.setMaxRedirects(0);
+            assertMessage("cluster.valkey.maxRedirects must be >= 1 in cluster mode", "got 0");
+        }
+
+        @Test
+        @DisplayName("V11: maxRedirects is not checked outside cluster mode")
+        void maxRedirectsIgnoredOutsideClusterMode() {
+            Valkey v = validStandalone();
+            v.setMaxRedirects(0);
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("V12: a url set alongside sentinel mode is accepted (ignored, not fatal)")
+        void urlAlongsideSentinelIsNotFatal() {
+            Valkey v = valkeyProps();
+            v.setUrl("redis://valkey:6379");
+            v.setMode("sentinel");
+            v.getSentinel().setMaster("mymaster");
+            v.getSentinel().setNodes(List.of("sentinel-1:26379", "sentinel-2:26379"));
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("V13: data password without a sentinel password warns but still boots")
+        void sentinelPasswordMismatchIsOnlyAWarning() {
+            Valkey v = valkeyProps();
+            v.setMode("sentinel");
+            v.setPassword("data-pw");
+            v.getSentinel().setMaster("mymaster");
+            v.getSentinel().setNodes(List.of("sentinel-1:26379"));
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("a fully configured sentinel topology validates")
+        void validSentinelTopologyPasses() {
+            Valkey v = valkeyProps();
+            v.setMode("sentinel");
+            v.getSentinel().setMaster("mymaster");
+            v.getSentinel()
+                    .setNodes(List.of("sentinel-1:26379", "sentinel-2:26379", "sentinel-3:26379"));
+            v.getSentinel().setPassword("sentinel-pw");
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("a fully configured cluster topology validates")
+        void validClusterTopologyPasses() {
+            Valkey v = valkeyProps();
+            v.setMode("cluster");
+            v.setNodes(List.of("valkey-1:6379", "valkey-2:6379", "valkey-3:6379"));
+            assertPasses();
+        }
+
+        @Test
+        @DisplayName("cluster.enabled=false skips every new check, garbage mode included")
+        void disabledClusterSkipsTopologyChecks() {
+            valkeyProps().setMode("not-a-mode");
+            props.getCluster().setEnabled(false);
+            assertPasses();
+        }
+
+        /** enabled + backplane=valkey with no topology keys set yet. */
+        private Valkey valkeyProps() {
+            props = new ApplicationProperties();
+            props.getCluster().setEnabled(true);
+            props.getCluster().setBackplane("valkey");
+            return props.getCluster().getValkey();
+        }
+
+        /** The minimal legacy configuration: standalone via url only. */
+        private Valkey validStandalone() {
+            Valkey v = valkeyProps();
+            v.setUrl("redis://valkey:6379");
+            return v;
+        }
+
+        private void assertPasses() {
+            ClusterConfig config = config(props);
+            assertDoesNotThrow(() -> invokeValidate(config));
+        }
+
+        private void assertMessage(String... expectedSubstrings) {
+            ClusterConfig config = config(props);
+            IllegalStateException ex =
+                    assertThrows(IllegalStateException.class, () -> invokeValidate(config));
+            for (String expected : expectedSubstrings) {
+                assertTrue(
+                        ex.getMessage().contains(expected),
+                        "message must contain '" + expected + "'; got: " + ex.getMessage());
+            }
+        }
+    }
+
+    /** A bare 'valkey:'/'sentinel:'/'pool:' key binds null; validation must not NPE on it. */
+    @Nested
+    @DisplayName("null config blocks give the operator message, never an NPE")
+    class BareYamlKeys {
+
+        private ApplicationProperties props;
+
+        @Test
+        void nullValkeyBlockReportsTheMissingUrl() {
+            enabled();
+            props.getCluster().setValkey(null);
+            assertMessage("cluster.valkey.url");
+        }
+
+        @Test
+        void nullSentinelBlockReportsTheMissingMaster() {
+            Valkey v = enabled();
+            v.setMode("sentinel");
+            v.setSentinel(null);
+            assertMessage("cluster.valkey.sentinel.master");
+        }
+
+        @Test
+        void nullSentinelNodesReportsTheMissingNodeList() {
+            Valkey v = enabled();
+            v.setMode("sentinel");
+            v.getSentinel().setMaster("mymaster");
+            v.getSentinel().setNodes(null);
+            assertMessage("cluster.valkey.sentinel.nodes");
+        }
+
+        @Test
+        void nullNodesBlockReportsTheMissingSeedList() {
+            Valkey v = enabled();
+            v.setMode("cluster");
+            v.setNodes(null);
+            assertMessage("cluster.valkey.nodes");
+        }
+
+        @Test
+        @DisplayName("null pool and tls blocks fall back to defaults and validate")
+        void nullPoolAndTlsBlocksValidate() {
+            Valkey v = enabled();
+            v.setUrl("redis://valkey:6379");
+            v.setPool(null);
+            v.setTls(null);
+            ClusterConfig config = config(props);
+            assertDoesNotThrow(() -> invokeValidate(config));
+        }
+
+        private Valkey enabled() {
+            props = new ApplicationProperties();
+            props.getCluster().setEnabled(true);
+            props.getCluster().setBackplane("valkey");
+            return props.getCluster().getValkey();
+        }
+
+        private void assertMessage(String expected) {
+            ClusterConfig config = config(props);
+            IllegalStateException ex =
+                    assertThrows(IllegalStateException.class, () -> invokeValidate(config));
+            assertTrue(
+                    ex.getMessage().contains(expected),
+                    "message must contain '" + expected + "'; got: " + ex.getMessage());
+        }
+    }
+
+    /** AutomaticallyGenerated.key/.UUID feed metadata encryption and licence seat HMACs. */
+    @Nested
+    @DisplayName("shared AutomaticallyGenerated key/UUID guard")
+    class SharedCryptoMaterial {
+
+        private ApplicationProperties props;
+
+        @Test
+        @DisplayName("an unset key is rejected and names the env var to set")
+        void missingKeyRejected() {
+            enabled("valkey");
+            assertMessage(
+                    new ClusterConfig(props, "", SHARED_UUID),
+                    "AutomaticallyGenerated.key",
+                    "same UUID on every node",
+                    "AUTOMATICALLYGENERATED_KEY");
+        }
+
+        @Test
+        @DisplayName("an unset UUID is rejected even when the key is set")
+        void missingUuidRejected() {
+            enabled("valkey");
+            assertMessage(
+                    new ClusterConfig(props, SHARED_KEY, null),
+                    "AutomaticallyGenerated.UUID",
+                    "AUTOMATICALLYGENERATED_UUID");
+        }
+
+        @Test
+        @DisplayName("the settings.yml.template placeholder is rejected: InitialSetup replaces it")
+        void nonUuidPlaceholderRejected() {
+            enabled("valkey");
+            assertMessage(
+                    new ClusterConfig(props, "example", "example"),
+                    "AutomaticallyGenerated.key",
+                    "must be a UUID");
+        }
+
+        @Test
+        @DisplayName("the guard also applies to backplane=inprocess")
+        void inProcessBackplaneAlsoGuarded() {
+            enabled("inprocess");
+            assertMessage(new ClusterConfig(props, "", ""), "AutomaticallyGenerated.key");
+        }
+
+        @Test
+        @DisplayName("two explicit UUIDs pass")
+        void explicitSharedValuesPass() {
+            enabled("valkey");
+            props.getCluster().getValkey().setUrl("redis://valkey:6379");
+            assertDoesNotThrow(() -> invokeValidate(config(props)));
+        }
+
+        @Test
+        @DisplayName("cluster.enabled=false never requires the shared values")
+        void disabledClusterSkipsGuard() {
+            enabled("valkey");
+            props.getCluster().setEnabled(false);
+            assertDoesNotThrow(() -> invokeValidate(new ClusterConfig(props, "", "")));
+        }
+
+        @Test
+        @DisplayName("the pre-bean guard reads the same rule as the @PostConstruct one")
+        void staticGuardMatchesPostConstructGuard() {
+            assertDoesNotThrow(() -> ClusterConfig.validateSharedCryptoMaterial(false, "", ""));
+            assertDoesNotThrow(
+                    () ->
+                            ClusterConfig.validateSharedCryptoMaterial(
+                                    true, SHARED_KEY, SHARED_UUID));
+            assertThrows(
+                    IllegalStateException.class,
+                    () -> ClusterConfig.validateSharedCryptoMaterial(true, SHARED_KEY, ""));
+        }
+
+        private void enabled(String backplane) {
+            props = new ApplicationProperties();
+            props.getCluster().setEnabled(true);
+            props.getCluster().setBackplane(backplane);
+        }
+
+        private void assertMessage(ClusterConfig config, String... expectedSubstrings) {
+            IllegalStateException ex =
+                    assertThrows(IllegalStateException.class, () -> invokeValidate(config));
+            for (String expected : expectedSubstrings) {
+                assertTrue(
+                        ex.getMessage().contains(expected),
+                        "message must contain '" + expected + "'; got: " + ex.getMessage());
+            }
+        }
+    }
+
+    /** The guard must abort the context before InitialSetup's @PostConstruct can generate one. */
+    @Nested
+    @DisplayName("pre-bean shared key/UUID guard")
+    class SharedCryptoMaterialBootGuard {
+
+        private final ApplicationContextRunner runner =
+                new ApplicationContextRunner()
+                        .withConfiguration(
+                                AutoConfigurations.of(PropertyPlaceholderAutoConfiguration.class))
+                        .withUserConfiguration(TestAppPropertiesConfig.class, ClusterConfig.class)
+                        .withPropertyValues("cluster.enabled=true", "cluster.backplane=inprocess");
+
+        @Test
+        @DisplayName("boot fails before any bean is created when the shared values are missing")
+        void bootFailsWhenSharedValuesMissing() {
+            runner.withUserConfiguration(StubInitialSetupConfig.class)
+                    .run(
+                            context ->
+                                    assertThat(context)
+                                            .getFailure()
+                                            .hasMessageContaining("AutomaticallyGenerated.key")
+                                            .hasMessageContaining("AUTOMATICALLYGENERATED_KEY"));
+        }
+
+        @Test
+        @DisplayName("boot succeeds once both values are configured")
+        void bootSucceedsWhenSharedValuesConfigured() {
+            runner.withUserConfiguration(StubInitialSetupConfig.class)
+                    .withPropertyValues(
+                            "AutomaticallyGenerated.key=" + SHARED_KEY,
+                            "AutomaticallyGenerated.UUID=" + SHARED_UUID)
+                    .run(context -> assertThat(context).hasNotFailed());
+        }
+
+        @Test
+        @DisplayName("no InitialSetup in the context means nothing mints a per-node UUID")
+        void guardSkippedWithoutInitialSetup() {
+            runner.run(context -> assertThat(context).hasNotFailed());
+        }
+    }
+
+    /** Defaults-only bean: the production class loads YAML in {@code @PostConstruct}. */
+    @Configuration
+    static class TestAppPropertiesConfig {
+        @Bean
+        ApplicationProperties applicationProperties() {
+            return new ApplicationProperties();
+        }
+    }
+
+    /** Stands in for :core's InitialSetup, which the guard keys off by bean name. */
+    @Configuration
+    static class StubInitialSetupConfig {
+        @Bean(name = "initialSetup")
+        Object initialSetup() {
+            return new Object();
+        }
+    }
+
     private void invokeValidate(ClusterConfig config) throws Exception {
         Method m = ClusterConfig.class.getDeclaredMethod("validate");
         m.setAccessible(true);

app/common/src/test/java/stirling/software/common/cluster/ClusterPropertiesTest.java

@@ -3,11 +3,19 @@ package stirling.software.common.cluster;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.util.List;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.ApplicationProperties.Cluster;
+import stirling.software.common.model.ApplicationProperties.Cluster.Valkey;
+import stirling.software.common.model.ApplicationProperties.Cluster.Valkey.ValkeyMode;
 
 class ClusterPropertiesTest {
 
@@ -25,6 +33,39 @@ class ClusterPropertiesTest {
         assertEquals(5000L, props.getNode().getHeartbeatIntervalMs());
     }
 
+    // A drifted default here is a silent breaking change for existing url-only installs.
+    @Test
+    @DisplayName("new valkey topology + pool keys default to the backward-compatible values")
+    void valkeyTopologyAndPoolDefaults() {
+        Valkey valkey = new ApplicationProperties().getCluster().getValkey();
+
+        assertEquals("", valkey.getMode(), "blank mode must auto-resolve, not force a topology");
+        assertEquals("", valkey.getUsername());
+        assertEquals("", valkey.getPassword());
+        assertTrue(valkey.getNodes().isEmpty());
+        assertEquals(3, valkey.getMaxRedirects());
+        assertEquals(30000L, valkey.getTopologyRefreshMs());
+        assertEquals("", valkey.getClientName());
+        assertEquals(2000L, valkey.getCommandTimeoutMs());
+
+        assertEquals("", valkey.getSentinel().getMaster());
+        assertTrue(valkey.getSentinel().getNodes().isEmpty());
+        assertEquals("", valkey.getSentinel().getUsername());
+        assertEquals("", valkey.getSentinel().getPassword());
+
+        assertFalse(valkey.getTls().isEnabled());
+        assertFalse(valkey.getTls().isSkipCertVerification());
+
+        assertTrue(valkey.getPool().isEnabled(), "pooling is on by default");
+        assertEquals(16, valkey.getPool().getMaxActive());
+        assertEquals(16, valkey.getPool().getMaxIdle());
+        // 0, not a warm floor: backplane traffic runs on the shared native connection.
+        assertEquals(0, valkey.getPool().getMinIdle());
+        assertEquals(2000L, valkey.getPool().getMaxWaitMillis());
+        assertEquals(30000L, valkey.getPool().getTimeBetweenEvictionRunsMillis());
+        assertTrue(valkey.getPool().isTestOnBorrow());
+    }
+
     @Test
     void resolvedRoleParsesCaseInsensitively() {
         Cluster props = new ApplicationProperties().getCluster();
@@ -59,4 +100,121 @@ class ClusterPropertiesTest {
         props.getNode().setId("abc");
         assertEquals("abc", props.resolvedNodeId());
     }
+
+    @Nested
+    @DisplayName("Valkey.resolvedMode()")
+    class ResolvedMode {
+
+        private Valkey valkey() {
+            return new ApplicationProperties().getCluster().getValkey();
+        }
+
+        @Test
+        @DisplayName("blank mode with nothing else configured is STANDALONE (url-only upgrade)")
+        void blankDefaultsToStandalone() {
+            Valkey v = valkey();
+            v.setUrl("redis://valkey:6379");
+            assertEquals(ValkeyMode.STANDALONE, v.resolvedMode());
+        }
+
+        @Test
+        @DisplayName("blank mode + sentinel.master infers SENTINEL")
+        void blankWithSentinelMasterInfersSentinel() {
+            Valkey v = valkey();
+            v.getSentinel().setMaster("mymaster");
+            assertEquals(ValkeyMode.SENTINEL, v.resolvedMode());
+        }
+
+        @Test
+        @DisplayName("blank mode + nodes infers CLUSTER")
+        void blankWithNodesInfersCluster() {
+            Valkey v = valkey();
+            v.setNodes(List.of("valkey-1:6379"));
+            assertEquals(ValkeyMode.CLUSTER, v.resolvedMode());
+        }
+
+        @Test
+        @DisplayName("blank mode + BOTH sentinel.master and nodes is ambiguous and throws")
+        void blankWithBothIsAmbiguous() {
+            Valkey v = valkey();
+            v.getSentinel().setMaster("mymaster");
+            v.setNodes(List.of("valkey-1:6379"));
+            IllegalStateException ex = assertThrows(IllegalStateException.class, v::resolvedMode);
+            assertTrue(
+                    ex.getMessage().contains("Set cluster.valkey.mode explicitly"),
+                    "message must tell the operator how to disambiguate; got: " + ex.getMessage());
+        }
+
+        @Test
+        @DisplayName("explicit mode parses case-insensitively and trims surrounding whitespace")
+        void explicitModeParsingIsLenient() {
+            Valkey v = valkey();
+            v.setMode("SENTINEL");
+            assertEquals(ValkeyMode.SENTINEL, v.resolvedMode());
+            v.setMode(" sentinel ");
+            assertEquals(ValkeyMode.SENTINEL, v.resolvedMode());
+            v.setMode("Cluster");
+            assertEquals(ValkeyMode.CLUSTER, v.resolvedMode());
+            v.setMode("standalone");
+            assertEquals(ValkeyMode.STANDALONE, v.resolvedMode());
+        }
+
+        @Test
+        @DisplayName("unknown mode names the bad value and lists the valid ones")
+        void unknownModeThrows() {
+            Valkey v = valkey();
+            v.setMode("clustr");
+            IllegalStateException ex = assertThrows(IllegalStateException.class, v::resolvedMode);
+            assertTrue(ex.getMessage().contains("clustr"), "must echo the bad value");
+            assertTrue(
+                    ex.getMessage().contains("standalone | sentinel | cluster"),
+                    "must list valid values; got: " + ex.getMessage());
+        }
+
+        @Test
+        @DisplayName("explicit mode wins over inference (standalone even when nodes are set)")
+        void explicitModeWinsOverInference() {
+            Valkey v = valkey();
+            v.setNodes(List.of("valkey-1:6379"));
+            v.setMode("standalone");
+            assertEquals(ValkeyMode.STANDALONE, v.resolvedMode());
+        }
+    }
+
+    /** A bare 'valkey:'/'sentinel:'/'tls:'/'pool:'/'nodes:' key in settings.yml binds null. */
+    @Nested
+    @DisplayName("bare yaml keys bind null")
+    class BareYamlKeys {
+
+        @Test
+        @DisplayName("a null nested block is re-seeded with its defaults, never handed back")
+        void nullNestedBlocksAreReSeeded() {
+            Cluster cluster = new ApplicationProperties().getCluster();
+            cluster.setValkey(null);
+            Valkey valkey = cluster.getValkey();
+            assertNotNull(valkey);
+
+            valkey.setSentinel(null);
+            valkey.setTls(null);
+            valkey.setPool(null);
+            valkey.setNodes(null);
+            assertNotNull(valkey.getSentinel());
+            assertNotNull(valkey.getTls());
+            assertNotNull(valkey.getPool());
+            assertEquals(16, valkey.getPool().getMaxActive());
+            assertTrue(valkey.getNodes().isEmpty());
+
+            valkey.getSentinel().setNodes(null);
+            assertTrue(valkey.getSentinel().getNodes().isEmpty());
+        }
+
+        @Test
+        @DisplayName("resolvedMode() survives null sentinel and nodes blocks")
+        void resolvedModeSurvivesNullBlocks() {
+            Valkey valkey = new ApplicationProperties().getCluster().getValkey();
+            valkey.setSentinel(null);
+            valkey.setNodes(null);
+            assertEquals(ValkeyMode.STANDALONE, valkey.resolvedMode());
+        }
+    }
 }

app/common/src/test/java/stirling/software/common/cluster/HostPortTest.java

@@ -0,0 +1,96 @@
+package stirling.software.common.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+@DisplayName("HostPort.parse()")
+class HostPortTest {
+
+    private static final String PROP = "cluster.valkey.nodes";
+    private static final String EXAMPLE = "valkey-1:6379";
+
+    @Test
+    @DisplayName("host:port splits into host and port")
+    void hostAndPort() {
+        HostPort e = HostPort.parse("valkey-1:6379", PROP, EXAMPLE);
+        assertEquals("valkey-1", e.host());
+        assertEquals(6379, e.port());
+    }
+
+    @Test
+    @DisplayName("surrounding whitespace is trimmed (comma-separated env vars keep spaces)")
+    void trimsWhitespace() {
+        HostPort e = HostPort.parse("  valkey-2:6380  ", PROP, EXAMPLE);
+        assertEquals("valkey-2", e.host());
+        assertEquals(6380, e.port());
+    }
+
+    @Test
+    @DisplayName("a bracketed IPv6 literal keeps no brackets in the host")
+    void bracketedIpv6() {
+        HostPort e = HostPort.parse("[::1]:6379", PROP, EXAMPLE);
+        assertEquals("::1", e.host());
+        assertEquals(6379, e.port());
+    }
+
+    @ParameterizedTest
+    @ValueSource(
+            strings = {
+                "valkey-1:abc",
+                "valkey-1:0",
+                "valkey-1:70000",
+                "valkey-1:",
+                ":6379",
+                "[::1",
+                "[::1]",
+                "[::1]6379"
+            })
+    @DisplayName("a bad port, a missing host or a malformed bracket throws")
+    void badEntriesThrow(String entry) {
+        assertRejected(entry);
+    }
+
+    @Test
+    @DisplayName("a bare host is rejected rather than silently taking a default port")
+    void bareHostIsRejected() {
+        assertRejected("valkey-1");
+    }
+
+    @Test
+    @DisplayName("an unbracketed IPv6 literal is rejected, not read as host ':' port")
+    void unbracketedIpv6IsRejected() {
+        IllegalStateException ex = assertRejected("::1");
+        assertTrue(
+                ex.getMessage().contains("[::1]:6379"),
+                "message must show the bracketed form; got: " + ex.getMessage());
+    }
+
+    @Test
+    @DisplayName("blank entry throws with a host:port example")
+    void blankEntryThrows() {
+        IllegalStateException ex =
+                assertThrows(
+                        IllegalStateException.class, () -> HostPort.parse("  ", PROP, EXAMPLE));
+        assertTrue(ex.getMessage().contains(PROP));
+        assertTrue(ex.getMessage().contains("host:port"));
+    }
+
+    private IllegalStateException assertRejected(String entry) {
+        IllegalStateException ex =
+                assertThrows(
+                        IllegalStateException.class, () -> HostPort.parse(entry, PROP, EXAMPLE));
+        assertTrue(
+                ex.getMessage().contains(PROP),
+                "message must name the property; got: " + ex.getMessage());
+        assertTrue(
+                ex.getMessage().contains(entry),
+                "message must echo the offending entry; got: " + ex.getMessage());
+        return ex;
+    }
+}

app/common/src/test/java/stirling/software/common/cluster/InProcessConfigurationConditionalTest.java

@@ -11,11 +11,6 @@ import org.springframework.context.annotation.Configuration;
 import stirling.software.common.cluster.inprocess.InProcessClusterConfiguration;
 import stirling.software.common.model.ApplicationProperties;
 
-/**
- * Verifies the {@link InProcessClusterConfiguration} conditional wiring: in-process beans wire when
- * cluster mode is off or {@code backplane=inprocess}, and are skipped when {@code
- * backplane=valkey}.
- */
 class InProcessConfigurationConditionalTest {
 
     private final ApplicationContextRunner runner =
@@ -76,9 +71,8 @@ class InProcessConfigurationConditionalTest {
     }
 
     /**
-     * Hand-rolled {@link ApplicationProperties} bean: the production class loads YAML at startup
-     * via a {@code @PostConstruct} hook that isn't appropriate for the slice runner, so we wire a
-     * defaults-only instance here.
+     * Defaults-only bean: the production class loads YAML in {@code @PostConstruct}, which the
+     * slice runner cannot do.
      */
     @Configuration
     static class TestAppPropertiesConfig {

app/core/src/main/resources/settings.yml.template

@@ -329,9 +329,11 @@ metrics:
   enabled: true # 'true' to enable Info APIs (`/api/*`) endpoints, 'false' to disable
 
 # Automatically Generated Settings (Do Not Edit Directly)
+# Generated on first boot and written back here. In a cluster EVERY node must carry the SAME key and
+# UUID (env: AUTOMATICALLYGENERATED_KEY / AUTOMATICALLYGENERATED_UUID) or nodes cannot read each other's data.
 AutomaticallyGenerated:
-  key: example
-  UUID: example
+  key: example # AES key for stored metadata. Differs per node = one node cannot decrypt another's rows.
+  UUID: example # Part of the licence-seat HMAC. Differs per node = peers reject each other's seat count.
   appVersion: 0.35.0
 
 processExecutor:
@@ -443,23 +445,60 @@ mcp:
     requireExistingAccount: true # Reject tokens whose subject has no enabled Stirling account (recommended)
   engineCapabilityRefreshMinutes: 5 # How often to refresh the AI capabilities manifest from the engine
 
-# Cluster configuration. NOT YET ENABLED - scaffolding for later work. Leave at defaults.
+# Multi-node Stirling PDF. Two different things are called "nodes" here: cluster.node.* is THIS
+# Stirling PDF instance, cluster.valkey.* is the shared Valkey - so 'mode: cluster' is Valkey's own.
 cluster:
-  enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here.
-  backplane: inprocess # Backplane implementation: 'inprocess' (single JVM only) or 'valkey' (multi-node via Valkey/Redis)
-  artifactStore: local # Transient cluster job-artifact backend: 'local' (per-node disk; single-node only) or 's3' (shared object store; required for multi-node). Distinct from 'storage.provider' which controls persistent user uploads - when both are 's3' they share the storage.s3.* credentials block. Multi-node deployments MUST set this to 's3'.
+  enabled: false # Run more than one Stirling PDF node. 'false' (default) = single instance; nothing below applies.
+  backplane: inprocess # How Stirling PDF nodes share state: 'inprocess' (one JVM) or 'valkey'.
+  artifactStore: local # Job artifacts: 'local' (per-node disk) or 's3'. Multi-node MUST use 's3'.
   s3:
-    keyPrefix: transient/ # Bucket key prefix used by the cluster artifact store when artifactStore=s3. Trailing slash recommended. Lets a single bucket host both persistent uploads (storage.s3.*) and transient job artifacts under separate prefixes.
-  valkey:
-    url: "" # Valkey/Redis URL, e.g. 'redis://valkey:6379' or 'rediss://...' for TLS. Required when enabled=true and backplane=valkey.
-    tls:
-      skipCertVerification: false # set to 'true' to skip TLS certificate verification on Valkey connections (dev/test only)
+    keyPrefix: transient/ # Prefix for job artifacts, so one bucket can also hold storage.s3.* uploads.
+
+  # This Stirling PDF instance's own identity.
   node:
-    id: "" # Optional explicit node id. Blank = auto-generated UUID at startup.
-    role: both # 'web' (serves HTTP), 'worker' (runs jobs), or 'both' (default)
-    internalAddress: "" # host:port advertised in the instance registry for peer-to-peer cluster traffic. Blank = derived at startup.
-    scheme: http # 'http' or 'https' - scheme peers use to call this node's /internal/cluster/** endpoints
-    heartbeatIntervalMs: 5000 # Heartbeat publish interval for the instance registry (ms)
+    id: "" # Blank = fresh UUID each boot. Set it for a stable name in monitoring.
+    role: both # 'web' (serves HTTP), 'worker' (runs jobs), or 'both'.
+    internalAddress: "" # host:port other Stirling PDF nodes reach this one on. Blank = derived at startup.
+    scheme: http # Scheme peers use for this node's /internal/cluster/** endpoints.
+    heartbeatIntervalMs: 5000 # How often this node publishes its heartbeat.
+
+  # The Valkey/Redis server every Stirling PDF node shares.
+  valkey:
+    mode: "" # 'standalone' | 'sentinel' | 'cluster'. Blank = inferred from whichever block below is filled in.
+    url: "" # e.g. 'redis://valkey:6379', optionally '/2' to pick a database. STANDALONE ONLY - sentinel and cluster ignore it, so a 'rediss://' url there refuses to boot unless tls.enabled is true.
+    # Two Stirling deployments sharing one Valkey MUST use different database indexes: there is no key
+    # prefix, so on the same index they merge each other's node registries and cross-route job traffic.
+    username: "" # Valkey ACL user. Overrides any user in 'url'.
+    password: "" # Valkey password. Overrides any password in 'url'. Never logged.
+    commandTimeoutMs: 2000 # Per-command timeout, so a slow Valkey cannot stall request threads.
+    clientName: "" # CLIENT SETNAME, so monitoring can tell the Stirling PDF nodes apart. Blank = 'stirling-' + node id; 'off' sends none.
+
+    # mode=sentinel: one Valkey primary plus replicas, with sentinels failing over between them.
+    sentinel:
+      master: "" # The name from 'sentinel monitor <name> ...'. Required for mode=sentinel.
+      nodes: [] # e.g. ['sentinel-1:26379','sentinel-2:26379']. Required for mode=sentinel.
+      username: "" # Sentinel ACL user - separate from the Valkey user above.
+      password: "" # Sentinel password - separate from the Valkey password; setting only that one does NOT authenticate here.
+
+    # mode=cluster: several Valkey nodes sharding the keyspace between them.
+    nodes: [] # e.g. ['valkey-1:6379','valkey-2:6379','valkey-3:6379']. Required for mode=cluster.
+    maxRedirects: 3 # Max MOVED/ASK redirects before a command fails.
+    topologyRefreshMs: 30000 # Periodic topology re-read; refresh on MOVED/ASK/reconnect is always on.
+
+    tls:
+      enabled: false # The only way to get TLS in sentinel/cluster mode. Standalone also accepts a 'rediss://' url.
+      skipCertVerification: false # Skip TLS chain and hostname checks. Dev only.
+
+    # Backplane traffic rides one shared connection, so the pool holds only that today.
+    # These are headroom for a future dedicated-connection path.
+    pool:
+      enabled: true
+      maxActive: 16 # Must be >= 2. Raising it does not change current throughput.
+      maxIdle: 16 # Keep equal to maxActive so idle connections are not churned.
+      minIdle: 0 # Connections kept warm. 0 because nothing borrows from the pool today.
+      maxWaitMillis: 2000 # Must be > 0: negative blocks forever, 0 fails the borrow instantly.
+      timeBetweenEvictionRunsMillis: 30000 # Idle-evictor interval; minIdle only applies while it runs.
+      testOnBorrow: true # Local isOpen() check only, no round trip.
 
 pdfEditor:
   fallback-font: classpath:/static/fonts/NotoSans-Regular.ttf # Override to point at a custom fallback font

app/proprietary/build.gradle

@@ -27,6 +27,9 @@ dependencies {
     api 'org.springframework.boot:spring-boot-starter-cache'
     api 'com.github.ben-manes.caffeine:caffeine'
     implementation 'org.springframework.boot:spring-boot-starter-data-redis'
+    // Supplies GenericObjectPoolConfig for the Lettuce pool, which today backs only the single
+    // shared native connection; headroom for a future dedicated-connection path.
+    implementation 'org.apache.commons:commons-pool2'
     api 'io.swagger.core.v3:swagger-core-jakarta:2.2.53'
     implementation "com.bucket4j:bucket4j_jdk17-core:${bucket4jVersion}"
     // Lettuce-backed Bucket4j ProxyManager used by ValkeyRateLimitStore for cluster-wide

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java

@@ -8,9 +8,8 @@ import java.lang.annotation.Target;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 
 /**
- * Composite condition: matches only when cluster.enabled=true AND cluster.backplane=valkey. Both
- * checks are required (enabled alone may select the in-process backplane, which must not load
- * Valkey beans); a single {@code @ConditionalOnExpression} keeps the guard in one place.
+ * Both checks are required: enabled alone may still select the in-process backplane, which must not
+ * load Valkey beans.
  */
 @Target({ElementType.TYPE, ElementType.METHOD})
 @Retention(RetentionPolicy.RUNTIME)

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyBackplaneHealthIndicator.java

@@ -0,0 +1,33 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.boot.health.contributor.HealthIndicator;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+
+/**
+ * Publishes backplane reachability as the {@code valkeyBackplane} contributor of {@code
+ * /actuator/health}. Without it a node whose Valkey is dead still reports UP, so a load balancer
+ * keeps sending it traffic it cannot serve.
+ */
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+public class ValkeyBackplaneHealthIndicator implements HealthIndicator {
+
+    private final ValkeyClusterBackplane backplane;
+
+    @Override
+    public Health health() {
+        try {
+            Health.Builder builder = backplane.isHealthy() ? Health.up() : Health.down();
+            return builder.withDetail("backplane", backplane.backplaneType())
+                    .withDetail("nodeId", backplane.localNodeId())
+                    .build();
+        } catch (RuntimeException ex) {
+            // isHealthy() swallows its own probe errors; this covers node-id resolution failing.
+            return Health.down(ex).withDetail("backplane", backplane.backplaneType()).build();
+        }
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java

@@ -1,6 +1,5 @@
 package stirling.software.proprietary.cluster.valkey;
 
-import org.springframework.data.redis.core.RedisCallback;
 import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.stereotype.Component;
 
@@ -22,12 +21,9 @@ public class ValkeyClusterBackplane implements ClusterBackplane {
     @Override
     public boolean isHealthy() {
         try {
-            // template.execute() borrows from the pool and returns the connection in a finally
-            // block - critical because isHealthy() is hit on every k8s liveness/readiness probe
-            // tick. Calling getConnectionFactory().getConnection() directly leaks the connection
-            // and exhausts the pool under monitoring load.
-            String pong = template.execute((RedisCallback<String>) connection -> connection.ping());
-            return "PONG".equalsIgnoreCase(pong);
+            // Single-key EXISTS, not PING: on Cluster spring-data fans PING to every node and
+            // fails if any is down. The key need not exist; a completed round trip is the signal.
+            return template.hasKey("stirling:health:" + localNodeId()) != null;
         } catch (RuntimeException ex) {
             log.warn("Valkey backplane health check failed: {}", ex.getMessage());
             return false;

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java

@@ -2,27 +2,40 @@ package stirling.software.proprietary.cluster.valkey;
 
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
+import org.springframework.data.redis.connection.ClusterInfo;
+import org.springframework.data.redis.connection.RedisClusterCommandsProvider;
+import org.springframework.data.redis.connection.RedisClusterConfiguration;
+import org.springframework.data.redis.connection.RedisConfiguration;
 import org.springframework.data.redis.connection.RedisConnection;
 import org.springframework.data.redis.connection.RedisPassword;
+import org.springframework.data.redis.connection.RedisSentinelConfiguration;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
 import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
 import org.springframework.data.redis.core.StringRedisTemplate;
 
 import io.lettuce.core.RedisCommandExecutionException;
 import io.lettuce.core.SslVerifyMode;
+import io.lettuce.core.api.StatefulConnection;
+import io.lettuce.core.cluster.ClusterClientOptions;
+import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.HostPort;
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.ApplicationProperties.Cluster;
+import stirling.software.common.model.ApplicationProperties.Cluster.Valkey;
 
 @Slf4j
 @Configuration
@@ -31,76 +44,201 @@ import stirling.software.common.model.ApplicationProperties.Cluster;
 @DependsOn("clusterLicenseGate")
 public class ValkeyConnectionConfiguration {
 
+    /** Command name in a NOPERM reply; the key-permission variant quotes nothing. */
+    private static final java.util.regex.Pattern NOPERM_COMMAND =
+            java.util.regex.Pattern.compile("run the '([^']+)'");
+
+    /** Valkey rejects a client name outside printable ASCII, space included, during HELLO. */
+    private static final java.util.regex.Pattern UNSAFE_CLIENT_NAME =
+            java.util.regex.Pattern.compile("[^\\x21-\\x7e]");
+
+    private static final int BOOT_PROBE_ATTEMPTS = 10;
+
     private final ApplicationProperties applicationProperties;
 
     @Bean(destroyMethod = "destroy")
     @ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
     public LettuceConnectionFactory valkeyConnectionFactory() {
         Cluster cluster = applicationProperties.getCluster();
-        Endpoint endpoint = parseUrl(cluster.getValkey().getUrl());
-        RedisStandaloneConfiguration cfg =
-                new RedisStandaloneConfiguration(endpoint.host(), endpoint.port());
-        if (endpoint.username() != null) {
-            cfg.setUsername(endpoint.username());
-        }
-        if (endpoint.password() != null) {
-            cfg.setPassword(RedisPassword.of(endpoint.password()));
-        }
-        boolean skipCertVerification =
-                cluster.getValkey().getTls() != null
-                        && cluster.getValkey().getTls().isSkipCertVerification();
+        Valkey valkey = cluster.getValkey();
+        Valkey.ValkeyMode mode = valkey.resolvedMode();
+        // Only standalone reads the URL; sentinel/cluster take endpoints and credentials from
+        // their own properties so there is one obvious source of truth per mode.
+        Endpoint endpoint = mode == Valkey.ValkeyMode.STANDALONE ? parseUrl(valkey.getUrl()) : null;
+        Valkey.Tls tlsProps = valkey.getTls();
+        // tls.enabled is OR-ed with the url scheme, never overridden by it.
+        boolean tls = tlsProps.isEnabled() || (endpoint != null && endpoint.tls());
+        guardIgnoredUrl(valkey, mode, tls);
+        String username =
+                firstNonBlank(valkey.getUsername(), endpoint == null ? null : endpoint.username());
+        String password =
+                firstNonBlank(valkey.getPassword(), endpoint == null ? null : endpoint.password());
+        String clientName = resolveClientName(cluster);
+
         LettuceClientConfiguration clientConfig =
-                buildClientConfiguration(endpoint.tls(), skipCertVerification);
-        LettuceConnectionFactory factory = new LettuceConnectionFactory(cfg, clientConfig);
+                buildClientConfiguration(
+                        tls,
+                        tlsProps.isSkipCertVerification(),
+                        clientName,
+                        Duration.ofMillis(valkey.getCommandTimeoutMs()),
+                        valkey.getPool(),
+                        mode,
+                        Duration.ofMillis(valkey.getTopologyRefreshMs()));
+
+        LettuceConnectionFactory factory =
+                switch (mode) {
+                    case STANDALONE ->
+                            new LettuceConnectionFactory(
+                                    standaloneConfiguration(endpoint, username, password),
+                                    clientConfig);
+                    case SENTINEL ->
+                            new LettuceConnectionFactory(
+                                    sentinelConfiguration(valkey, username, password),
+                                    clientConfig);
+                    case CLUSTER ->
+                            new LettuceConnectionFactory(
+                                    clusterConfiguration(valkey, username, password), clientConfig);
+                };
         factory.afterPropertiesSet();
         // Eager handshake with retry tolerates docker-compose DNS races; fails boot loudly
         // if Valkey is genuinely unreachable.
-        eagerHandshake(factory, endpoint.host(), endpoint.port(), endpoint.tls());
+        String target = describeTarget(mode, endpoint, valkey);
+        eagerHandshake(factory, target, tls, mode == Valkey.ValkeyMode.CLUSTER, clientName);
         log.info(
-                "Valkey connection configured: {}:{} tls={} verifyPeer={}",
-                endpoint.host(),
-                endpoint.port(),
-                endpoint.tls(),
-                endpoint.tls() ? clientConfig.getVerifyMode() : "n/a");
+                "Valkey connection configured: mode={} endpoints={} tls={} verifyPeer={} pooled={}"
+                        + " clientName={}",
+                mode,
+                target,
+                tls,
+                tls ? clientConfig.getVerifyMode() : "n/a",
+                valkey.getPool().isEnabled(),
+                clientName == null ? "disabled (no CLIENT SETNAME)" : clientName);
         return factory;
     }
 
+    /**
+     * Sentinel/cluster ignore {@code cluster.valkey.url}; a dropped {@code rediss://} would
+     * silently downgrade TLS to plaintext, so that combination refuses boot (userinfo only warns).
+     */
+    static void guardIgnoredUrl(Valkey valkey, Valkey.ValkeyMode mode, boolean tls) {
+        if (mode == Valkey.ValkeyMode.STANDALONE || !isSet(valkey.getUrl())) {
+            return;
+        }
+        URI uri;
+        try {
+            uri = new URI(valkey.getUrl().trim());
+        } catch (URISyntaxException ex) {
+            log.warn("cluster.valkey.url is ignored in {} mode and is not a valid URI", mode);
+            return;
+        }
+        if (!tls && "rediss".equalsIgnoreCase(uri.getScheme())) {
+            throw new IllegalStateException(
+                    "cluster.valkey.url uses rediss:// (TLS) but cluster.valkey.mode="
+                            + mode.name().toLowerCase(java.util.Locale.ROOT)
+                            + " ignores the url and cluster.valkey.tls.enabled is false. Refusing"
+                            + " to connect in plaintext. Set cluster.valkey.tls.enabled=true, or"
+                            + " clear cluster.valkey.url if plaintext is intended.");
+        }
+        if (isSet(uri.getUserInfo())) {
+            log.warn(
+                    "cluster.valkey.url carries credentials but mode={} ignores the url - set"
+                            + " cluster.valkey.username/password (and sentinel.password) instead",
+                    mode);
+        }
+        int database = databaseOrZero(uri.getPath());
+        if (database != 0) {
+            log.warn(
+                    "cluster.valkey.url selects database {} but mode={} ignores the url - this"
+                            + " deployment will use database 0",
+                    database,
+                    mode);
+        }
+    }
+
+    static RedisStandaloneConfiguration standaloneConfiguration(
+            Endpoint endpoint, String username, String password) {
+        RedisStandaloneConfiguration cfg =
+                new RedisStandaloneConfiguration(endpoint.host(), endpoint.port());
+        // The database is the only isolation between two deployments sharing one Valkey.
+        cfg.setDatabase(endpoint.database());
+        applyAuth(cfg, username, password);
+        return cfg;
+    }
+
+    static RedisSentinelConfiguration sentinelConfiguration(
+            Valkey valkey, String username, String password) {
+        RedisSentinelConfiguration cfg = new RedisSentinelConfiguration();
+        cfg.master(valkey.getSentinel().getMaster().trim());
+        for (String node : valkey.getSentinel().getNodes()) {
+            HostPort e = HostPort.parse(node, "cluster.valkey.sentinel.nodes", "sentinel-1:26379");
+            cfg.sentinel(e.host(), e.port());
+        }
+        applyAuth(cfg, username, password);
+        // Sentinel AUTH is separate from data-node AUTH - the commonest sentinel misconfiguration.
+        if (isSet(valkey.getSentinel().getUsername())) {
+            cfg.setSentinelUsername(valkey.getSentinel().getUsername());
+        }
+        if (isSet(valkey.getSentinel().getPassword())) {
+            cfg.setSentinelPassword(RedisPassword.of(valkey.getSentinel().getPassword()));
+        }
+        return cfg;
+    }
+
+    static RedisClusterConfiguration clusterConfiguration(
+            Valkey valkey, String username, String password) {
+        RedisClusterConfiguration cfg = new RedisClusterConfiguration();
+        for (String node : valkey.getNodes()) {
+            HostPort e = HostPort.parse(node, "cluster.valkey.nodes", "valkey-1:6379");
+            cfg.clusterNode(e.host(), e.port());
+        }
+        cfg.setMaxRedirects(valkey.getMaxRedirects());
+        applyAuth(cfg, username, password);
+        return cfg;
+    }
+
+    private static void applyAuth(
+            RedisConfiguration.WithAuthentication cfg, String username, String password) {
+        if (username != null) {
+            cfg.setUsername(username);
+        }
+        if (password != null) {
+            cfg.setPassword(RedisPassword.of(password));
+        }
+    }
+
     /** Parsed connection endpoint; username/password are null when absent. */
-    record Endpoint(String host, int port, boolean tls, String username, String password) {}
+    record Endpoint(
+            String host, int port, boolean tls, String username, String password, int database) {}
 
     /**
-     * Parses {@code redis://[user:password@]host[:port]} (or {@code rediss://} for TLS) into an
-     * {@link Endpoint}. Package-private and side-effect-free so URL handling is unit-testable.
-     *
-     * <ul>
-     *   <li>Missing port defaults to 6379.
-     *   <li>{@code rediss} scheme selects TLS.
-     *   <li>Userinfo {@code :password@} (empty user) is treated as password-only auth against the
-     *       default user, not a login with an empty username.
-     *   <li>Reserved characters in the password ({@code @ : / # ?}) must be percent-encoded; {@link
-     *       URI} parses them structurally otherwise (e.g. {@code #} starts the fragment).
-     * </ul>
-     *
-     * @throws IllegalStateException if the URL is blank, syntactically invalid, or has no host
+     * Reserved chars in the password ({@code @ : / # ?}) must be percent-encoded - {@link URI}
+     * otherwise parses them structurally (e.g. {@code #} starts the fragment).
      */
-    static Endpoint parseUrl(String url) {
-        if (url == null || url.isBlank()) {
+    static Endpoint parseUrl(String rawUrl) {
+        if (rawUrl == null || rawUrl.isBlank()) {
             throw new IllegalStateException("cluster.valkey.url must be set when backplane=valkey");
         }
+        // A .env line or YAML block scalar leaves a trailing newline that URI rejects.
+        String url = rawUrl.trim();
         URI uri;
         try {
             uri = new URI(url);
         } catch (URISyntaxException ex) {
+            // Never attach ex: its own message echoes the url, credentials included.
             throw new IllegalStateException(
-                    "cluster.valkey.url is not a valid URI: " + url + " (" + ex.getMessage() + ")",
-                    ex);
+                    "cluster.valkey.url is not a valid URI: "
+                            + redactUserInfo(url)
+                            + " ("
+                            + ex.getReason()
+                            + (ex.getIndex() >= 0 ? " at index " + ex.getIndex() : "")
+                            + ")");
         }
         String host = uri.getHost();
         if (host == null || host.isBlank()) {
             throw new IllegalStateException(
                     "cluster.valkey.url has no host: "
-                            + url
-                            + " (expected redis://[user:password@]host[:port])");
+                            + redactUserInfo(url)
+                            + " (expected redis://[user:password@]host[:port][/database])");
         }
         boolean tls = "rediss".equalsIgnoreCase(uri.getScheme());
         int port = uri.getPort() <= 0 ? 6379 : uri.getPort();
@@ -116,7 +254,69 @@ public class ValkeyConnectionConfiguration {
                 password = parts[0];
             }
         }
-        return new Endpoint(host, port, tls, username, password);
+        int database;
+        try {
+            database = parseDatabaseSegment(uri.getPath());
+        } catch (NumberFormatException ex) {
+            throw new IllegalStateException(
+                    "cluster.valkey.url has an invalid database index: "
+                            + redactUserInfo(url)
+                            + " (the path must be a non-negative integer, e.g."
+                            + " redis://valkey:6379/2)");
+        }
+        return new Endpoint(host, port, tls, username, password, database);
+    }
+
+    /**
+     * Spring Boot's {@code spring.data.redis.url} reads the path as the database index, so an
+     * operator copying that syntax must not be silently downgraded to database 0.
+     */
+    private static int parseDatabaseSegment(String path) {
+        if (path == null || path.isBlank() || "/".equals(path)) {
+            return 0;
+        }
+        int database = Integer.parseInt(path.startsWith("/") ? path.substring(1) : path);
+        if (database < 0) {
+            throw new NumberFormatException("negative database index " + database);
+        }
+        return database;
+    }
+
+    /** Lenient variant: the ignored-url guard must never fail boot over an unused database. */
+    private static int databaseOrZero(String path) {
+        try {
+            return parseDatabaseSegment(path);
+        } catch (NumberFormatException ex) {
+            return 0;
+        }
+    }
+
+    /**
+     * Masks {@code user:password@} so a url can be named in an error. Bounded to the authority: an
+     * unbounded lastIndexOf('@') would mangle {@code redis://valkey:6379/0?tag=a@b}.
+     */
+    static String redactUserInfo(String url) {
+        if (url == null) {
+            return null;
+        }
+        int slashes = url.indexOf("//");
+        if (slashes < 0) {
+            return url;
+        }
+        int start = slashes + 2;
+        int end = url.length();
+        for (int i = start; i < url.length(); i++) {
+            char c = url.charAt(i);
+            if (c == '/' || c == '?' || c == '#') {
+                end = i;
+                break;
+            }
+        }
+        int at = end == start ? -1 : url.lastIndexOf('@', end - 1);
+        if (at < start) {
+            return url;
+        }
+        return url.substring(0, start) + "****" + url.substring(at);
     }
 
     /**
@@ -124,14 +324,34 @@ public class ValkeyConnectionConfiguration {
      * default change cannot silently weaken our TLS handshake. skipCertVerification is dev-only.
      */
     static LettuceClientConfiguration buildClientConfiguration(
-            boolean tls, boolean skipCertVerification) {
-        LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder =
-                LettuceClientConfiguration.builder();
-        // Bound every backplane command. Lettuce defaults to 60s; without this a partitioned or
-        // slow Valkey would stall hot-path calls (e.g. JobController.guardNonOwner -> jobStore.get
-        // on each request) for up to a minute, exhausting request threads. All backplane ops are
-        // non-blocking single commands, so a short timeout is safe.
-        clientBuilder.commandTimeout(Duration.ofSeconds(2));
+            boolean tls,
+            boolean skipCertVerification,
+            String clientName,
+            Duration commandTimeout,
+            Valkey.Pool pool,
+            Valkey.ValkeyMode mode,
+            Duration topologyRefresh) {
+        LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder;
+        if (pool.isEnabled()) {
+            // poolConfig() MUST be called before useSsl(): the SSL sub-builder's static type is the
+            // non-pooling one, so chaining it after would not compile.
+            clientBuilder =
+                    LettucePoolingClientConfiguration.builder().poolConfig(toPoolConfig(pool));
+        } else {
+            clientBuilder = LettuceClientConfiguration.builder();
+        }
+        // Lettuce defaults to 60s; unbounded, a slow Valkey stalls hot-path calls and exhausts
+        // request threads. All backplane ops are single non-blocking commands, so short is safe.
+        clientBuilder.commandTimeout(commandTimeout);
+        // CLIENT SETNAME attributes load per node; null = opted out (see resolveClientName).
+        if (clientName != null) {
+            clientBuilder.clientName(clientName);
+        }
+        if (mode == Valkey.ValkeyMode.CLUSTER) {
+            // Must be ClusterClientOptions: spring-data filters on that type, and anything else
+            // is dropped along with our topology refresh settings.
+            clientBuilder.clientOptions(clusterClientOptions(topologyRefresh));
+        }
         if (tls) {
             clientBuilder
                     .useSsl()
@@ -146,26 +366,131 @@ public class ValkeyConnectionConfiguration {
         return clientBuilder.build();
     }
 
+    static GenericObjectPoolConfig<StatefulConnection<?, ?>> toPoolConfig(Valkey.Pool pool) {
+        GenericObjectPoolConfig<StatefulConnection<?, ?>> cfg = new GenericObjectPoolConfig<>();
+        cfg.setMaxTotal(pool.getMaxActive());
+        cfg.setMaxIdle(pool.getMaxIdle());
+        cfg.setMinIdle(pool.getMinIdle());
+        cfg.setMaxWait(Duration.ofMillis(pool.getMaxWaitMillis()));
+        // Local isOpen() check, no round trip: it only rejects already-closed connections. Lettuce
+        // auto-reconnect means a stale post-failover connection can still report open and be lent.
+        cfg.setTestOnBorrow(pool.isTestOnBorrow());
+        cfg.setTestWhileIdle(false);
+        // minIdle and idle eviction are inert unless the evictor thread actually runs.
+        cfg.setTimeBetweenEvictionRuns(Duration.ofMillis(pool.getTimeBetweenEvictionRunsMillis()));
+        cfg.setJmxEnabled(false);
+        return cfg;
+    }
+
+    static ClusterClientOptions clusterClientOptions(Duration topologyRefresh) {
+        // Without adaptive triggers the client keeps hammering a demoted master for up to a
+        // full refresh period after a failover; periodic refresh alone is not enough.
+        ClusterTopologyRefreshOptions topology =
+                ClusterTopologyRefreshOptions.builder()
+                        .enablePeriodicRefresh(topologyRefresh)
+                        .enableAllAdaptiveRefreshTriggers()
+                        .adaptiveRefreshTriggersTimeout(topologyRefresh)
+                        .dynamicRefreshSources(true)
+                        .closeStaleConnections(true)
+                        .build();
+        return ClusterClientOptions.builder()
+                .topologyRefreshOptions(topology)
+                .validateClusterNodeMembership(true)
+                .build();
+    }
+
+    /**
+     * Blank = {@code stirling-} + node name; off/none/disabled = null. A missing SETNAME ACL
+     * refuses nothing (RESP3 folds it into HELLO, RESP2 swallows it); a name Valkey rejects does.
+     */
+    static String resolveClientName(Cluster cluster) {
+        String configured = cluster.getValkey().getClientName();
+        if (!isSet(configured)) {
+            return sanitiseClientName("stirling-" + cluster.resolvedNodeName(), "cluster.node.id");
+        }
+        String trimmed = configured.trim();
+        return isClientNameOptOut(trimmed)
+                ? null
+                : sanitiseClientName(trimmed, "cluster.valkey.clientName");
+    }
+
+    private static String sanitiseClientName(String name, String source) {
+        String safe = UNSAFE_CLIENT_NAME.matcher(name).replaceAll("-");
+        if (!safe.equals(name)) {
+            log.warn(
+                    "Valkey client name from {} contains characters Valkey refuses in the"
+                            + " handshake; using '{}' instead of '{}'",
+                    source,
+                    safe,
+                    name);
+        }
+        return safe;
+    }
+
+    private static boolean isClientNameOptOut(String value) {
+        String lower = value.toLowerCase(java.util.Locale.ROOT);
+        return "off".equals(lower) || "none".equals(lower) || "disabled".equals(lower);
+    }
+
+    private static boolean isSet(String v) {
+        return v != null && !v.isBlank();
+    }
+
+    private static String firstNonBlank(String preferred, String fallback) {
+        if (isSet(preferred)) {
+            return preferred;
+        }
+        return isSet(fallback) ? fallback : null;
+    }
+
+    static String describeTarget(Valkey.ValkeyMode mode, Endpoint endpoint, Valkey valkey) {
+        return switch (mode) {
+            // Database only when non-zero: the boot log stays host:port for the common case.
+            case STANDALONE ->
+                    endpoint.host()
+                            + ":"
+                            + endpoint.port()
+                            + (endpoint.database() == 0 ? "" : "/" + endpoint.database());
+            case SENTINEL ->
+                    "sentinels="
+                            + String.join(",", valkey.getSentinel().getNodes())
+                            + " master="
+                            + valkey.getSentinel().getMaster();
+            case CLUSTER -> "nodes=" + String.join(",", valkey.getNodes());
+        };
+    }
+
     /**
      * 10 x 3s = 30s boot-time retry. Auth failures (WRONGPASS/NOAUTH/NOPERM) short-circuit
      * immediately; only transport errors get the loop. Package-private for testing.
      */
     static void eagerHandshake(
-            LettuceConnectionFactory factory, String host, int port, boolean tls) {
+            LettuceConnectionFactory factory,
+            String target,
+            boolean tls,
+            boolean clusterMode,
+            String clientName) {
+        // Single-key EXISTS, not PING: on Valkey Cluster spring-data fans PING out to EVERY node
+        // and reports failure if any one is down, which would refuse boot on a healthy cluster.
+        byte[] probeKey =
+                ("stirling:health:boot:" + (clientName == null ? "unnamed" : clientName))
+                        .getBytes(StandardCharsets.UTF_8);
         RuntimeException last = null;
-        for (int attempt = 1; attempt <= 10; attempt++) {
+        int attempt = 0;
+        while (attempt < BOOT_PROBE_ATTEMPTS) {
+            attempt++;
             try {
-                String pong;
                 RedisConnection conn = factory.getConnection();
                 try {
-                    pong = conn.ping();
+                    if (conn.keyCommands().exists(probeKey) == null) {
+                        throw new IllegalStateException("Valkey EXISTS probe returned no reply");
+                    }
+                    if (clusterMode) {
+                        assertClusterServesSlots(conn);
+                    }
                 } finally {
                     conn.close();
                 }
-                if (!"PONG".equalsIgnoreCase(pong)) {
-                    throw new IllegalStateException(
-                            "Valkey PING returned '" + pong + "' (expected PONG)");
-                }
                 if (attempt > 1) {
                     log.info("Valkey reachable after {} attempts", attempt);
                 }
@@ -173,52 +498,114 @@ public class ValkeyConnectionConfiguration {
             } catch (RuntimeException ex) {
                 if (isAuthFailure(ex)) {
                     factory.destroy();
-                    throw new IllegalStateException(
-                            "Valkey authentication failed for "
-                                    + host
-                                    + ":"
-                                    + port
-                                    + " (tls="
-                                    + tls
-                                    + "): "
-                                    + rootAuthMessage(ex)
-                                    + ". Check cluster.valkey.url credentials"
-                                    + " (user/password and ACL permissions).",
-                            ex);
+                    throw new IllegalStateException(authFailureMessage(ex, target, tls), ex);
                 }
                 last = ex;
                 log.warn(
-                        "Valkey PING attempt {}/10 failed ({}:{}, tls={}): {}",
+                        "Valkey probe attempt {}/{} failed ({}, tls={}): {}",
                         attempt,
-                        host,
-                        port,
+                        BOOT_PROBE_ATTEMPTS,
+                        target,
                         tls,
                         ex.getMessage());
-                try {
-                    Thread.sleep(3000);
-                } catch (InterruptedException ie) {
-                    Thread.currentThread().interrupt();
-                    break;
+                // No backoff after the final attempt: it would only delay the throw below.
+                if (attempt < BOOT_PROBE_ATTEMPTS) {
+                    try {
+                        Thread.sleep(3000);
+                    } catch (InterruptedException ie) {
+                        // Destroy first: an armed interrupt aborts Lettuce's shutdown await and
+                        // leaks the client resources.
+                        factory.destroy();
+                        Thread.currentThread().interrupt();
+                        throw new IllegalStateException(
+                                unreachableMessage(attempt, target, tls, last), last);
+                    }
                 }
             }
         }
         factory.destroy();
-        throw new IllegalStateException(
-                "Valkey unreachable at boot after 10 attempts ("
-                        + host
-                        + ":"
-                        + port
-                        + ", tls="
-                        + tls
-                        + "): "
-                        + (last == null ? "no detail" : last.getMessage()),
-                last);
+        throw new IllegalStateException(unreachableMessage(attempt, target, tls, last), last);
+    }
+
+    private static String unreachableMessage(
+            int attempts, String target, boolean tls, RuntimeException last) {
+        return "Valkey unreachable at boot after "
+                + attempts
+                + " attempts ("
+                + target
+                + ", tls="
+                + tls
+                + "): "
+                + (last == null ? "no detail" : last.getMessage());
     }
 
     /**
-     * Walks the cause chain for WRONGPASS/NOAUTH/NOPERM replies. Spring Data Redis wraps Lettuce's
-     * RedisCommandExecutionException in RedisSystemException, so the auth signal may be one level
-     * down. No typed auth exception exists in spring-data-redis 4.0.5 / Lettuce 6.8.2.
+     * A cluster answers commands long before it covers all 16384 slots; boot must wait for both.
+     */
+    private static void assertClusterServesSlots(RedisConnection conn) {
+        if (!(conn instanceof RedisClusterCommandsProvider provider)) {
+            throw new IllegalStateException(
+                    "Expected a cluster connection but got " + conn.getClass().getName());
+        }
+        ClusterInfo info = provider.clusterCommands().clusterGetClusterInfo();
+        if (info == null || !"ok".equalsIgnoreCase(String.valueOf(info.getState()))) {
+            throw new IllegalStateException(
+                    "Valkey cluster_state is not ok (cluster is not serving all slots yet)");
+        }
+        Long slotsOk = info.getSlotsOk();
+        if (slotsOk != null && slotsOk < 16384L) {
+            throw new IllegalStateException(
+                    "Valkey cluster covers only " + slotsOk + "/16384 slots");
+        }
+    }
+
+    /**
+     * NOPERM means the credentials were accepted and the ACL user lacks a command or key
+     * permission, so the message must not send operators hunting a password problem.
+     */
+    static String authFailureMessage(Throwable ex, String target, boolean tls) {
+        String reply = rootAuthMessage(ex);
+        String where = target + " (tls=" + tls + "): " + reply;
+        if (!isPermissionFailure(ex)) {
+            return "Valkey authentication failed for "
+                    + where
+                    + ". Check cluster.valkey credentials (username/password,"
+                    + " sentinel.password).";
+        }
+        String command = refusedCommand(reply);
+        return "Valkey ACL refused a command for "
+                + where
+                + ". The credentials were accepted; this ACL user is missing a command or key"
+                + " permission"
+                + (command == null ? "" : " - grant '+" + command + "'")
+                + ". The backplane needs the boot probe (EXISTS) and read/write access to the"
+                + " 'stirling:*' keyspace.";
+    }
+
+    /** NOPERM only - a permitted-command/key problem, distinct from bad credentials. */
+    static boolean isPermissionFailure(Throwable t) {
+        for (Throwable cur = t; cur != null; cur = cur.getCause()) {
+            if (startsWithToken(cur.getMessage(), "NOPERM")) {
+                return true;
+            }
+            if (cur.getCause() == cur) {
+                break;
+            }
+        }
+        return false;
+    }
+
+    private static String refusedCommand(String reply) {
+        if (reply == null) {
+            return null;
+        }
+        java.util.regex.Matcher m = NOPERM_COMMAND.matcher(reply);
+        return m.find() ? m.group(1) : null;
+    }
+
+    /**
+     * WRONGPASS/NOAUTH/NOPERM are all unrecoverable at boot, so they share this fast-fail path. No
+     * typed auth exception in spring-data-redis 4.0.5 / Lettuce 6.8.2, hence the text match.
      */
     static boolean isAuthFailure(Throwable t) {
         for (Throwable cur = t; cur != null; cur = cur.getCause()) {
@@ -236,13 +623,14 @@ public class ValkeyConnectionConfiguration {
     }
 
     private static boolean hasAuthPrefix(String message) {
-        if (message == null) {
-            return false;
-        }
-        String upper = message.toUpperCase(java.util.Locale.ROOT).stripLeading();
-        return upper.startsWith("WRONGPASS")
-                || upper.startsWith("NOAUTH")
-                || upper.startsWith("NOPERM");
+        return startsWithToken(message, "WRONGPASS")
+                || startsWithToken(message, "NOAUTH")
+                || startsWithToken(message, "NOPERM");
+    }
+
+    private static boolean startsWithToken(String message, String token) {
+        return message != null
+                && message.toUpperCase(java.util.Locale.ROOT).stripLeading().startsWith(token);
     }
 
     private static String rootAuthMessage(Throwable t) {

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java

@@ -15,6 +15,10 @@ import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.common.cluster.DistributedLock;
 
+/**
+ * Valkey-backed {@link DistributedLock}; single-key throughout, so cluster-safe. NOT Redlock: a
+ * failover can grant the same lock twice, so treat {@code renew() == false} as lost and abort.
+ */
 @Component
 @RequiredArgsConstructor
 @ConditionalOnValkeyBackplane
@@ -64,9 +68,8 @@ public class ValkeyDistributedLock implements DistributedLock {
                 return;
             }
             released = true;
-            // Swallow + log: LockHandle is AutoCloseable, so release() runs from close() inside
-            // try-with-resources. An uncaught Valkey error here would mask the body's exception.
-            // The lease TTL-expires anyway, so a failed explicit release is safe.
+            // Swallow: release() runs from close(), so throwing would mask the body's exception.
+            // The lease TTL-expires anyway.
             try {
                 template.execute(RELEASE_SCRIPT, Collections.singletonList(key), value);
             } catch (RuntimeException ex) {

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java

@@ -1,6 +1,5 @@
 package stirling.software.proprietary.cluster.valkey;
 
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -11,9 +10,10 @@ import java.util.Map;
 import java.util.Optional;
 
 import org.springframework.data.redis.core.Cursor;
-import org.springframework.data.redis.core.RedisCallback;
 import org.springframework.data.redis.core.ScanOptions;
 import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.script.DefaultRedisScript;
+import org.springframework.data.redis.core.script.RedisScript;
 import org.springframework.stereotype.Component;
 
 import lombok.RequiredArgsConstructor;
@@ -21,10 +21,7 @@ import lombok.RequiredArgsConstructor;
 import stirling.software.common.cluster.ClusterNode;
 import stirling.software.common.cluster.InstanceRegistry;
 
-/**
- * Valkey-backed {@link InstanceRegistry}. Each node is stored as a hash with a TTL equal to the
- * configured heartbeat TTL; the heartbeat re-arms the TTL.
- */
+/** Every operation is single-key, so it is correct on standalone, sentinel and cluster alike. */
 @Component
 @RequiredArgsConstructor
 @ConditionalOnValkeyBackplane
@@ -32,6 +29,14 @@ public class ValkeyInstanceRegistry implements InstanceRegistry {
 
     private static final String PREFIX = "stirling:nodes:";
 
+    // Single-key HSET+PEXPIRE. Atomic server-side, so a crash can never leave the node hash
+    // without a TTL, which would mask a dead node as alive forever.
+    private static final RedisScript<Long> HSET_WITH_TTL =
+            new DefaultRedisScript<>(
+                    "redis.call('HSET', KEYS[1], unpack(ARGV, 2));"
+                            + " redis.call('PEXPIRE', KEYS[1], ARGV[1]); return 1",
+                    Long.class);
+
     private final StringRedisTemplate template;
 
     @Override
@@ -44,25 +49,13 @@ public class ValkeyInstanceRegistry implements InstanceRegistry {
         fields.put("role", node.role());
         fields.put("lastHeartbeat", node.lastHeartbeat().toString());
 
-        // MULTI/EXEC so the hash fields and the TTL commit together. Without this, a crash
-        // between HSET and EXPIRE leaves the hash with no TTL: it never expires, masks the
-        // dead node as alive, and only a subsequent successful register() would re-arm it.
-        template.execute(
-                (RedisCallback<Object>)
-                        connection -> {
-                            connection.multi();
-                            byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-                            Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
-                            for (Map.Entry<String, String> f : fields.entrySet()) {
-                                hashBytes.put(
-                                        f.getKey().getBytes(StandardCharsets.UTF_8),
-                                        f.getValue().getBytes(StandardCharsets.UTF_8));
-                            }
-                            connection.hashCommands().hMSet(keyBytes, hashBytes);
-                            connection.keyCommands().pExpire(keyBytes, ttlMs);
-                            connection.exec();
-                            return null;
-                        });
+        List<String> args = new ArrayList<>(1 + fields.size() * 2);
+        args.add(Long.toString(ttlMs));
+        for (Map.Entry<String, String> f : fields.entrySet()) {
+            args.add(f.getKey());
+            args.add(f.getValue());
+        }
+        template.execute(HSET_WITH_TTL, List.of(key), args.toArray());
     }
 
     @Override

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java

@@ -1,6 +1,5 @@
 package stirling.software.proprietary.cluster.valkey;
 
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -11,10 +10,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
 import org.springframework.data.redis.core.Cursor;
-import org.springframework.data.redis.core.RedisCallback;
 import org.springframework.data.redis.core.ScanOptions;
 import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.script.DefaultRedisScript;
+import org.springframework.data.redis.core.script.RedisScript;
 import org.springframework.stereotype.Component;
 
 import lombok.RequiredArgsConstructor;
@@ -28,11 +30,8 @@ import tools.jackson.core.type.TypeReference;
 import tools.jackson.databind.ObjectMapper;
 
 /**
- * Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId.
- *
- * <p><b>put() atomicity:</b> the hash fields, the per-job TTL, and the reverse-index entries are
- * issued inside a single pipelined Redis transaction (MULTI/EXEC). A partial failure cannot leave
- * the hash without a TTL or with half the file→job index entries written.
+ * One hash per job plus a fileId to jobId index. Atomic on standalone/sentinel via one Lua script;
+ * on cluster those keys are cross-slot, so writes are separate and hash-first to keep tears benign.
  */
 @Component
 @RequiredArgsConstructor
@@ -49,12 +48,74 @@ public class ValkeyJobStore implements JobStore {
     private static final TypeReference<Map<String, String>> MAP_STRING =
             new TypeReference<Map<String, String>>() {};
 
+    // Atomic HSET+PEXPIRE: the hash must never exist without a TTL.
+    private static final RedisScript<Long> HSET_WITH_TTL =
+            new DefaultRedisScript<>(
+                    "redis.call('HSET', KEYS[1], unpack(ARGV, 2));"
+                            + " redis.call('PEXPIRE', KEYS[1], ARGV[1]); return 1",
+                    Long.class);
+
+    // Value-guarded delete: never removes an index row a newer job already owns.
+    private static final RedisScript<Long> DEL_INDEX_IF_OWNER =
+            new DefaultRedisScript<>(
+                    "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1])"
+                            + " else return 0 end",
+                    Long.class);
+
+    // Non-cluster put(): hash, TTL and index rows in one round trip, so no torn write.
+    // KEYS[1]=hash, KEYS[2..]=index rows; ARGV[1]=ttlMs, ARGV[2]=jobId, ARGV[3..]=fields.
+    private static final RedisScript<Long> PUT_ATOMIC =
+            new DefaultRedisScript<>(
+                    "redis.call('HSET', KEYS[1], unpack(ARGV, 3));"
+                            + " redis.call('PEXPIRE', KEYS[1], ARGV[1]);"
+                            + " for i = 2, #KEYS do"
+                            + " redis.call('SET', KEYS[i], ARGV[2], 'PX', ARGV[1]) end;"
+                            + " return 1",
+                    Long.class);
+
+    // Non-cluster delete(): hash read, DEL and the value-guarded index deletes in one round trip.
+    // ARGV[1]=jobId, ARGV[2]=index prefix; malformed fileIds JSON must not error the script.
+    private static final RedisScript<Long> DELETE_JOB_AND_INDEX =
+            new DefaultRedisScript<>(
+                    "local ids = redis.call('HGET', KEYS[1], 'fileIds');"
+                            + " redis.call('DEL', KEYS[1]);"
+                            + " if not ids then return 0 end;"
+                            + " local ok, decoded = pcall(cjson.decode, ids);"
+                            + " if not ok or type(decoded) ~= 'table' then return 0 end;"
+                            + " local n = 0;"
+                            + " for i = 1, #decoded do"
+                            + " if type(decoded[i]) == 'string' then"
+                            + " local k = ARGV[2] .. decoded[i];"
+                            + " if redis.call('GET', k) == ARGV[1] then"
+                            + " n = n + redis.call('DEL', k) end end end;"
+                            + " return n",
+                    Long.class);
+
     private final StringRedisTemplate template;
 
+    // Cluster rejects a script spanning the job key and its index keys: they hash to other slots.
+    private boolean isClusterAware() {
+        RedisConnectionFactory factory = template.getConnectionFactory();
+        return factory instanceof LettuceConnectionFactory lettuce && lettuce.isClusterAware();
+    }
+
     @Override
     public void put(JobStoreEntry entry, Duration ttl) {
         String key = JOB_PREFIX + entry.jobId();
         long ttlMs = ttl.toMillis();
+        // SET..PX rejects a non-positive TTL, so an already-expired entry deletes instead.
+        // Reachable via stirling.jobResultExpiryMinutes=0.
+        if (ttlMs <= 0) {
+            if (entry.fileIds() != null) {
+                for (String fileId : entry.fileIds()) {
+                    // Value-guarded like delete(): never drop a row a newer job already owns.
+                    template.execute(
+                            DEL_INDEX_IF_OWNER, List.of(FILE_INDEX_PREFIX + fileId), entry.jobId());
+                }
+            }
+            template.delete(key);
+            return;
+        }
         Map<String, String> fields = new LinkedHashMap<>();
         fields.put("jobId", entry.jobId());
         fields.put("state", entry.state().name());
@@ -73,37 +134,37 @@ public class ValkeyJobStore implements JobStore {
                 "resultMeta",
                 writeJson(entry.resultMeta() == null ? Map.of() : entry.resultMeta()));
 
-        // Build pipelined MULTI/EXEC so the hash, its TTL, and every reverse-index entry
-        // commit atomically.
-        template.execute(
-                (RedisCallback<Object>)
-                        connection -> {
-                            connection.multi();
-                            byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
-                            Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
-                            for (Map.Entry<String, String> f : fields.entrySet()) {
-                                hashBytes.put(
-                                        f.getKey().getBytes(StandardCharsets.UTF_8),
-                                        f.getValue().getBytes(StandardCharsets.UTF_8));
-                            }
-                            connection.hashCommands().hMSet(keyBytes, hashBytes);
-                            connection.keyCommands().pExpire(keyBytes, ttlMs);
-                            if (entry.fileIds() != null) {
-                                for (String fileId : entry.fileIds()) {
-                                    byte[] idxKey =
-                                            (FILE_INDEX_PREFIX + fileId)
-                                                    .getBytes(StandardCharsets.UTF_8);
-                                    connection
-                                            .stringCommands()
-                                            .set(
-                                                    idxKey,
-                                                    entry.jobId().getBytes(StandardCharsets.UTF_8));
-                                    connection.keyCommands().pExpire(idxKey, ttlMs);
-                                }
-                            }
-                            connection.exec();
-                            return null;
-                        });
+        List<String> indexKeys = new ArrayList<>();
+        if (entry.fileIds() != null) {
+            for (String fileId : entry.fileIds()) {
+                indexKeys.add(FILE_INDEX_PREFIX + fileId);
+            }
+        }
+        List<String> fieldArgs = new ArrayList<>(fields.size() * 2);
+        for (Map.Entry<String, String> f : fields.entrySet()) {
+            fieldArgs.add(f.getKey());
+            fieldArgs.add(f.getValue());
+        }
+        if (!isClusterAware()) {
+            List<String> keys = new ArrayList<>(1 + indexKeys.size());
+            keys.add(key);
+            keys.addAll(indexKeys);
+            List<String> args = new ArrayList<>(2 + fieldArgs.size());
+            args.add(Long.toString(ttlMs));
+            args.add(entry.jobId());
+            args.addAll(fieldArgs);
+            template.execute(PUT_ATOMIC, keys, args.toArray());
+            return;
+        }
+        // Cluster only: hash first, so a torn write leaves an unindexed job rather than an index
+        // row pointing at a hash that does not exist.
+        List<String> args = new ArrayList<>(1 + fieldArgs.size());
+        args.add(Long.toString(ttlMs));
+        args.addAll(fieldArgs);
+        template.execute(HSET_WITH_TTL, List.of(key), args.toArray());
+        for (String indexKey : indexKeys) {
+            template.opsForValue().set(indexKey, entry.jobId(), ttl);
+        }
     }
 
     @Override
@@ -111,70 +172,25 @@ public class ValkeyJobStore implements JobStore {
         return readEntry(JOB_PREFIX + jobId);
     }
 
+    /**
+     * Live path: /api/v1/general/jobs/cleanup and /api/v1/admin/job/cleanup?force=true, neither
+     * gated by shouldRunLocalCleanup(). One script except on cluster, where a put() can interleave.
+     */
     @Override
     public void delete(String jobId) {
-        // WATCH/MULTI/EXEC: read fileIds INSIDE the watched scope so a concurrent put() that
-        // adds new fileIds between our read and EXEC aborts the transaction. Without this guard,
-        // an interleaved put() that grows fileIds would leave orphaned reverse-index entries
-        // pointing at the deleted jobId until their TTL expires. One retry handles the common
-        // case; further contention falls through to lazy TTL cleanup (acceptable - this is an
-        // eviction path, not a correctness primitive).
         String jobKey = JOB_PREFIX + jobId;
-        byte[] jobKeyBytes = jobKey.getBytes(StandardCharsets.UTF_8);
-        for (int attempt = 0; attempt < 2; attempt++) {
-            Boolean committed =
-                    template.execute(
-                            (RedisCallback<Boolean>)
-                                    connection -> {
-                                        connection.watch(jobKeyBytes);
-                                        // Read the single fileIds field with hGet rather than
-                                        // hGetAll + map.get: hGetAll returns a Map<byte[],byte[]>
-                                        // whose keys compare by identity, so a fresh
-                                        // "fileIds".getBytes() lookup never matches and the reverse
-                                        // index would be left orphaned. hGet resolves the field
-                                        // server-side.
-                                        byte[] fileIdsBytes =
-                                                connection
-                                                        .hashCommands()
-                                                        .hGet(
-                                                                jobKeyBytes,
-                                                                "fileIds"
-                                                                        .getBytes(
-                                                                                StandardCharsets
-                                                                                        .UTF_8));
-                                        List<byte[]> keysToDelete = new ArrayList<>();
-                                        keysToDelete.add(jobKeyBytes);
-                                        if (fileIdsBytes != null) {
-                                            List<String> fileIds =
-                                                    readJsonList(
-                                                            new String(
-                                                                    fileIdsBytes,
-                                                                    StandardCharsets.UTF_8),
-                                                            jobKey);
-                                            for (String fileId : fileIds) {
-                                                keysToDelete.add(
-                                                        (FILE_INDEX_PREFIX + fileId)
-                                                                .getBytes(StandardCharsets.UTF_8));
-                                            }
-                                        }
-                                        connection.multi();
-                                        for (byte[] key : keysToDelete) {
-                                            connection.keyCommands().del(key);
-                                        }
-                                        List<Object> results = connection.exec();
-                                        // exec() returns null when WATCH detected a concurrent
-                                        // write; spring-data-redis surfaces this as either null
-                                        // or empty depending on the driver path.
-                                        return results != null && !results.isEmpty();
-                                    });
-            if (Boolean.TRUE.equals(committed)) {
-                return;
-            }
+        if (!isClusterAware()) {
+            template.execute(DELETE_JOB_AND_INDEX, List.of(jobKey), jobId, FILE_INDEX_PREFIX);
+            return;
+        }
+        String fileIdsJson = (String) template.opsForHash().get(jobKey, "fileIds");
+        template.delete(jobKey);
+        if (fileIdsJson == null) {
+            return;
+        }
+        for (String fileId : readJsonList(fileIdsJson, jobKey)) {
+            template.execute(DEL_INDEX_IF_OWNER, List.of(FILE_INDEX_PREFIX + fileId), jobId);
         }
-        log.warn(
-                "JobStore.delete({}) lost two WATCH races to concurrent put(); reverse-index"
-                        + " entries may linger until TTL expiry",
-                jobId);
     }
 
     @Override
@@ -190,7 +206,8 @@ public class ValkeyJobStore implements JobStore {
 
     @Override
     public Collection<JobStoreEntry> all() {
-        // SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk.
+        // SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk. On Cluster
+        // Lettuce walks every master, so this is a best-effort snapshot, not a point-in-time one.
         ScanOptions options = ScanOptions.scanOptions().match(JOB_PREFIX + "*").count(256).build();
         List<JobStoreEntry> result = new ArrayList<>();
         try (Cursor<String> cursor = template.scan(options)) {

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java

@@ -4,7 +4,6 @@ import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.TimeUnit;
 
 import org.springframework.data.redis.core.Cursor;
 import org.springframework.data.redis.core.ScanOptions;
@@ -26,8 +25,7 @@ public class ValkeyKeyValueCache implements KeyValueCache {
 
     @Override
     public void put(String namespace, String key, String value, Duration ttl) {
-        template.opsForValue()
-                .set(buildKey(namespace, key), value, ttl.toMillis(), TimeUnit.MILLISECONDS);
+        template.opsForValue().set(buildKey(namespace, key), value, ttl);
     }
 
     @Override
@@ -50,8 +48,10 @@ public class ValkeyKeyValueCache implements KeyValueCache {
                 keys.add(cursor.next());
             }
         }
-        if (!keys.isEmpty()) {
-            template.delete(keys);
+        // Batched UNLINK, 500 keys a call. On Cluster spring-data splits a cross-slot batch
+        // into one command per key (fanned out per node), so expect n commands there, not n/500.
+        for (int i = 0; i < keys.size(); i += 500) {
+            template.unlink(keys.subList(i, Math.min(i + 500, keys.size())));
         }
     }
 

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java

@@ -14,6 +14,11 @@ import io.github.bucket4j.distributed.proxy.ProxyManager;
 import io.github.bucket4j.redis.lettuce.Bucket4jLettuce;
 import io.lettuce.core.AbstractRedisClient;
 import io.lettuce.core.RedisClient;
+import io.lettuce.core.api.StatefulConnection;
+import io.lettuce.core.api.StatefulRedisConnection;
+import io.lettuce.core.cluster.RedisClusterClient;
+import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
+import io.lettuce.core.codec.ByteArrayCodec;
 
 import jakarta.annotation.PostConstruct;
 import jakarta.annotation.PreDestroy;
@@ -21,9 +26,8 @@ import jakarta.annotation.PreDestroy;
 import stirling.software.common.cluster.RateLimitStore;
 
 /**
- * Valkey-backed token-bucket rate limiting via Bucket4j's Lettuce ProxyManager. The token bucket
- * refills continuously and enforces one global limit across nodes, with the same semantics as the
- * in-process {@code InProcessRateLimitStore} (which also uses Bucket4j).
+ * Bucket4j CAS scripts are {@code KEYS[1]}-only, so each bucket stays linearizable on one slot with
+ * no hash tag needed - correct on standalone, sentinel and cluster.
  */
 @Component
 @ConditionalOnValkeyBackplane
@@ -32,6 +36,7 @@ public class ValkeyRateLimitStore implements RateLimitStore {
     private static final String PREFIX = "stirling:rl:";
 
     private final LettuceConnectionFactory connectionFactory;
+    private StatefulConnection<byte[], byte[]> connection;
     private ProxyManager<byte[]> proxyManager;
 
     public ValkeyRateLimitStore(LettuceConnectionFactory connectionFactory) {
@@ -41,18 +46,30 @@ public class ValkeyRateLimitStore implements RateLimitStore {
     @PostConstruct
     void initProxyManager() {
         AbstractRedisClient client = connectionFactory.getNativeClient();
-        if (!(client instanceof RedisClient redisClient)) {
+        // Own the connection (rather than the client overloads) so it is closed at shutdown; it
+        // inherits the RedisURI client name, so CLIENT LIST still attributes it to this node.
+        Bucket4jLettuce.LettuceBasedProxyManagerBuilder<byte[]> builder;
+        if (client instanceof RedisClusterClient clusterClient) {
+            StatefulRedisClusterConnection<byte[], byte[]> conn =
+                    clusterClient.connect(ByteArrayCodec.INSTANCE);
+            this.connection = conn;
+            builder = Bucket4jLettuce.casBasedBuilder(conn);
+        } else if (client instanceof RedisClient redisClient) {
+            // Sentinel also yields a plain RedisClient, so this arm covers standalone + sentinel.
+            StatefulRedisConnection<byte[], byte[]> conn =
+                    redisClient.connect(ByteArrayCodec.INSTANCE);
+            this.connection = conn;
+            builder = Bucket4jLettuce.casBasedBuilder(conn);
+        } else {
             throw new IllegalStateException(
-                    "ValkeyRateLimitStore requires a standalone Lettuce RedisClient; got "
+                    "ValkeyRateLimitStore needs a Lettuce RedisClient or RedisClusterClient; got "
                             + (client == null ? "null" : client.getClass().getName())
-                            + " (cluster client not supported by this rate limit impl)");
+                            + ". This is a Stirling bug - please report it.");
         }
-        // Expire idle bucket keys so they do not accumulate forever in Valkey (one key per
-        // user / API-key / IP). TTL tracks the time to refill the bucket from empty, capped at
-        // 25h to cover the longest (daily) rate-limit window; an idle bucket evicts after that.
+        // One key per user/API-key/IP, so idle buckets must expire. 25h cap covers the longest
+        // (daily) rate-limit window.
         this.proxyManager =
-                Bucket4jLettuce.casBasedBuilder(redisClient)
-                        .expirationAfterWrite(
+                builder.expirationAfterWrite(
                                 ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax(
                                         Duration.ofHours(25)))
                         .build();
@@ -61,6 +78,10 @@ public class ValkeyRateLimitStore implements RateLimitStore {
     @PreDestroy
     void shutdown() {
         proxyManager = null;
+        if (connection != null) {
+            connection.close();
+            connection = null;
+        }
     }
 
     @Override

app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java

@@ -84,6 +84,12 @@ import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
 import stirling.software.proprietary.policy.trigger.TriggerInfo;
 import stirling.software.proprietary.util.SecretMasker;
 
+import tools.jackson.core.JsonParser;
+import tools.jackson.databind.DeserializationContext;
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.ValueDeserializer;
+import tools.jackson.databind.annotation.JsonDeserialize;
+
 /**
  * Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
  * GET /run/{runId}} for status, download outputs via {@code GET /api/v1/general/files/{fileId}}.
@@ -271,8 +277,19 @@ public class PolicyController {
             summary = "Create or update a policy",
             description =
                     "Stores a policy (trigger config + steps + output + metadata). A blank id is"
-                            + " assigned; returns the stored policy with its id.")
-    public ResponseEntity<Policy> savePolicy(@RequestBody Policy policy) {
+                            + " assigned; returns the stored policy with its id. Sources and their"
+                            + " triggers live in 'inputs'; the pre-inputs body shape is rejected"
+                            + " rather than silently bound to nothing.")
+    public ResponseEntity<Policy> savePolicy(@RequestBody PolicySaveRequest request) {
+        requireInputsShape(request.legacyKeys());
+        return savePolicy(request.policy());
+    }
+
+    /**
+     * The save itself, once the body's shape is known good. Also the entry point for callers that
+     * already hold a typed {@link Policy}.
+     */
+    public ResponseEntity<Policy> savePolicy(Policy policy) {
         requirePolicyEditingAllowed();
         Policy owned = withStoredOutputSecrets(resolveOwnership(policy));
         requireAccessibleSources(owned);
@@ -310,6 +327,51 @@ public class PolicyController {
         return ResponseEntity.noContent().build();
     }
 
+    /**
+     * Reject the pre-inputs body shape rather than binding it to nothing: {@code sourceIds} and a
+     * populated {@code trigger} have no field on {@link Policy}, so Jackson drops them and the save
+     * would store a policy that references and watches nothing. Stored blobs in that shape are
+     * migrated on read by the policy store; an inbound request has no such path.
+     */
+    private static void requireInputsShape(List<String> legacyKeys) {
+        if (legacyKeys.isEmpty()) {
+            return;
+        }
+        throw new ResponseStatusException(
+                HttpStatus.BAD_REQUEST,
+                "Unsupported policy field(s): "
+                        + String.join(", ", legacyKeys)
+                        + ". Use 'inputs': sourceIds and trigger moved onto each input.");
+    }
+
+    /**
+     * The save body: the bound policy plus whichever dead pre-inputs keys it still carries. Binding
+     * straight to {@link Policy} would drop them silently, since unknown properties are ignored.
+     */
+    @JsonDeserialize(using = PolicySaveRequest.Deserializer.class)
+    public record PolicySaveRequest(Policy policy, List<String> legacyKeys) {
+
+        /** Reads the policy off the raw body, noting the dead top-level keys it carries. */
+        static final class Deserializer extends ValueDeserializer<PolicySaveRequest> {
+
+            @Override
+            public PolicySaveRequest deserialize(JsonParser parser, DeserializationContext ctxt) {
+                JsonNode root = ctxt.readTree(parser);
+                List<String> legacyKeys = new ArrayList<>();
+                // A null trigger drops nothing, and the editor and portal still send one on
+                // every save.
+                if (root.hasNonNull("trigger")) {
+                    legacyKeys.add("trigger");
+                }
+                if (root.has("sourceIds")) {
+                    legacyKeys.add("sourceIds");
+                }
+                return new PolicySaveRequest(
+                        ctxt.readTreeAsValue(root, Policy.class), List.copyOf(legacyKeys));
+            }
+        }
+    }
+
     /**
      * Every {@code sourceId} a policy references must resolve to a source in the caller's team, so
      * a client can neither reference a non-existent source nor reach across teams to use another

app/proprietary/src/main/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeeder.java

@@ -47,7 +47,7 @@ public class DefaultClassificationPolicySeeder {
     @EventListener(ApplicationReadyEvent.class)
     public void seedDefaultTeamOnStartup() {
         teamRepository
-                .findByName(TeamService.DEFAULT_TEAM_NAME)
+                .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
                 .ifPresent(team -> seedIfMissing(team.getId(), team.getName()));
     }
 

app/proprietary/src/main/java/stirling/software/proprietary/security/InitialSecuritySetup.java

@@ -8,6 +8,7 @@ import java.util.UUID;
 
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.env.Environment;
+import org.springframework.dao.DataAccessException;
 import org.springframework.stereotype.Component;
 
 import jakarta.annotation.PostConstruct;
@@ -44,43 +45,74 @@ public class InitialSecuritySetup {
     private final TeamMembershipService teamMembershipService;
 
     /**
-     * SaaS manages identity in Supabase and billing via PAYG, so the self-host bootstrap steps that
-     * scan/rewrite the whole user table (default-team backfill, seat-license grandfathering) don't
-     * apply - and against a large SaaS user table they stall startup with full-table loads +
-     * per-row saveAll. Per-user team assignment happens in SupabaseAuthenticationFilter instead.
+     * SaaS skips the self-host bootstrap: full-table backfills stall startup at SaaS scale, and
+     * team assignment happens per-user in SupabaseAuthenticationFilter instead.
      */
     private boolean isSaas() {
         return Arrays.asList(environment.getActiveProfiles()).contains("saas");
     }
 
+    // Peers racing a cold shared DB collide on a different row each pass, and each collision means
+    // a peer committed that row - so a bounded loop converges; one retry is not enough.
+    private static final int BOOTSTRAP_RACE_ATTEMPTS = 6;
+
     @PostConstruct
     public void init() {
         try {
-
-            if (!userService.hasUsers()) {
-                if (databaseService.hasBackup()) {
-                    databaseService.importDatabase();
-                } else {
-                    initializeAdminUser();
+            boolean restoredFromBackup = importBackupIfNeeded();
+            for (int attempt = 1; ; attempt++) {
+                try {
+                    runBootstrap(restoredFromBackup);
+                    return;
+                } catch (DataAccessException e) {
+                    if (attempt >= BOOTSTRAP_RACE_ATTEMPTS) {
+                        throw e;
+                    }
+                    log.info(
+                            "Security bootstrap lost a race to a peer node (attempt {}/{});"
+                                    + " re-running against its committed rows.",
+                            attempt,
+                            BOOTSTRAP_RACE_ATTEMPTS);
                 }
             }
-
-            configureJWTSettings();
-            initializeInternalApiUser();
-            if (isSaas()) {
-                log.info(
-                        "SaaS profile active - skipping self-host user-table bootstrap"
-                                + " (default-team backfill, seat-license grandfathering).");
-            } else {
-                assignUsersToDefaultTeamIfMissing();
-                initializeUserLicenseSettings();
-            }
-        } catch (IllegalArgumentException | SQLException | UnsupportedProviderException e) {
+        } catch (IllegalArgumentException
+                | SQLException
+                | UnsupportedProviderException
+                | DataAccessException e) {
+            // Widened for diagnosis, not recovery: unrecoverable cases such as duplicate team rows
+            // (tracked separately) still just exhaust the attempts and exit here.
             log.error("Failed to initialize security setup.", e);
             System.exit(1);
         }
     }
 
+    // Restoring a backup replays the whole schema, so it must run outside the retry loop.
+    private boolean importBackupIfNeeded() {
+        if (userService.hasUsers() || !databaseService.hasBackup()) {
+            return false;
+        }
+        databaseService.importDatabase();
+        return true;
+    }
+
+    private void runBootstrap(boolean restoredFromBackup)
+            throws IllegalArgumentException, SQLException, UnsupportedProviderException {
+        if (!restoredFromBackup && !userService.hasUsers()) {
+            initializeAdminUser();
+        }
+
+        configureJWTSettings();
+        initializeInternalApiUser();
+        if (isSaas()) {
+            log.info(
+                    "SaaS profile active - skipping self-host user-table bootstrap"
+                            + " (default-team backfill, seat-license grandfathering).");
+        } else {
+            assignUsersToDefaultTeamIfMissing();
+            initializeUserLicenseSettings();
+        }
+    }
+
     private void initializeUserLicenseSettings() {
         licenseSettingsService.initializeGrandfatheredCount();
         licenseSettingsService.updateLicenseMaxUsers();
@@ -115,8 +147,10 @@ public class InitialSecuritySetup {
             }
         }
 
-        userService.saveAll(usersWithoutTeam); // batch save
+        // A null team_id is the retry guard, so commit it last: syncMembership is idempotent and
+        // only needs the already-persisted team id, so a half-done pass is re-found and finished.
         usersWithoutTeam.forEach(teamMembershipService::syncMembership);
+        userService.saveAll(usersWithoutTeam); // batch save
         if (usersWithoutTeam != null && !usersWithoutTeam.isEmpty()) {
             log.info(
                     "Assigned {} user(s) without a team to the default team.",

app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/InviteLinkController.java

@@ -144,7 +144,9 @@ public class InviteLinkController {
             Long effectiveTeamId = teamId;
             if (effectiveTeamId == null) {
                 Team defaultTeam =
-                        teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME).orElse(null);
+                        teamRepository
+                                .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
+                                .orElse(null);
                 if (defaultTeam != null) {
                     effectiveTeamId = defaultTeam.getId();
                 }

app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/UserController.java

@@ -110,7 +110,10 @@ public class UserController {
                                                 + ", Available slots: "
                                                 + availableSlots));
             }
-            Team team = teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME).orElse(null);
+            Team team =
+                    teamRepository
+                            .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
+                            .orElse(null);
             SaveUserRequest.Builder builder =
                     SaveUserRequest.builder()
                             .username(username)
@@ -425,7 +428,9 @@ public class UserController {
         Long effectiveTeamId = teamId;
         if (effectiveTeamId == null) {
             Team defaultTeam =
-                    teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME).orElse(null);
+                    teamRepository
+                            .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
+                            .orElse(null);
             if (defaultTeam != null) {
                 effectiveTeamId = defaultTeam.getId();
             }
@@ -534,7 +539,9 @@ public class UserController {
         Long effectiveTeamId = teamId;
         if (effectiveTeamId == null) {
             Team defaultTeam =
-                    teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME).orElse(null);
+                    teamRepository
+                            .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
+                            .orElse(null);
             if (defaultTeam != null) {
                 effectiveTeamId = defaultTeam.getId();
             }

app/proprietary/src/main/java/stirling/software/proprietary/security/repository/TeamRepository.java

@@ -14,6 +14,10 @@ import stirling.software.proprietary.model.dto.TeamWithUserCountDTO;
 public interface TeamRepository extends JpaRepository<Team, Long> {
     Optional<Team> findByName(String name);
 
+    // teams.name is not unique, so peers cold-booting a shared DB can commit two "Default" rows.
+    // Converging on the lowest id keeps every node agreeing instead of throwing NonUniqueResult.
+    Optional<Team> findFirstByNameOrderByIdAsc(String name);
+
     @Query(
             "SELECT new stirling.software.proprietary.model.dto.TeamWithUserCountDTO(t.id, t.name, COUNT(u)) "
                     + "FROM Team t LEFT JOIN t.users u GROUP BY t.id, t.name")

app/proprietary/src/main/java/stirling/software/proprietary/security/service/TeamService.java

@@ -17,24 +17,25 @@ public class TeamService {
     public static final String INTERNAL_TEAM_NAME = "Internal";
 
     public Team getOrCreateDefaultTeam() {
-        return teamRepository
-                .findByName(DEFAULT_TEAM_NAME)
-                .orElseGet(
-                        () -> {
-                            Team defaultTeam = new Team();
-                            defaultTeam.setName(DEFAULT_TEAM_NAME);
-                            return teamRepository.save(defaultTeam);
-                        });
+        return getOrCreate(DEFAULT_TEAM_NAME);
     }
 
     public Team getOrCreateInternalTeam() {
+        return getOrCreate(INTERNAL_TEAM_NAME);
+    }
+
+    /**
+     * Lowest id wins, so peers that raced a cold shared DB into two same-named rows still agree.
+     * Duplicates cannot be prevented here: teams.name has no unique constraint, by design.
+     */
+    private Team getOrCreate(String name) {
         return teamRepository
-                .findByName(INTERNAL_TEAM_NAME)
+                .findFirstByNameOrderByIdAsc(name)
                 .orElseGet(
                         () -> {
-                            Team internalTeam = new Team();
-                            internalTeam.setName(INTERNAL_TEAM_NAME);
-                            return teamRepository.save(internalTeam);
+                            Team team = new Team();
+                            team.setName(name);
+                            return teamRepository.save(team);
                         });
     }
 }

app/proprietary/src/main/java/stirling/software/proprietary/security/service/UserService.java

@@ -474,11 +474,11 @@ public class UserService implements UserServiceInterface {
      */
     private Team getDefaultTeam() {
         return teamRepository
-                .findByName("Default")
+                .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
                 .orElseGet(
                         () -> {
                             Team team = new Team();
-                            team.setName("Default");
+                            team.setName(TeamService.DEFAULT_TEAM_NAME);
                             return teamRepository.save(team);
                         });
     }

app/proprietary/src/main/java/stirling/software/proprietary/workflow/service/MetadataEncryptionService.java

@@ -109,7 +109,7 @@ public class MetadataEncryptionService {
 
     /**
      * Reverses {@link #encryptBytes}. Also accepts legacy values stored as plain Base64 before
-     * encryption was introduced — {@link #decrypt} returns those unchanged, so they still decode.
+     * encryption was introduced - {@link #decrypt} returns those unchanged, so they still decode.
      */
     public byte[] decryptBytes(String stored) {
         if (stored == null) {
@@ -124,7 +124,7 @@ public class MetadataEncryptionService {
         String rawKey = applicationProperties.getAutomaticallyGenerated().getKey();
         if (rawKey == null || rawKey.isBlank()) {
             throw new IllegalStateException(
-                    "AutomaticallyGenerated.key is not initialised — cannot derive encryption key");
+                    "AutomaticallyGenerated.key is not initialised - cannot derive encryption key");
         }
         // SHA-256 of the raw key gives a stable 32-byte AES-256 key
         byte[] hash =

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java

@@ -28,14 +28,8 @@ import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
 import stirling.software.common.model.ApplicationProperties;
 
 /**
- * Opt-in live cluster test against an EXTERNAL Valkey/Redis given by {@code
- * STIRLING_TEST_VALKEY_URL} (e.g. a managed {@code rediss://} endpoint). Unlike {@link
- * LiveValkeyIntegrationTest} (no-auth local container) this drives three independent node stacks
- * through the production {@link ValkeyConnectionConfiguration#valkeyConnectionFactory()} bean - so
- * a {@code rediss://} URL exercises the real TLS handshake (verifyPeer=FULL) and credential path
- * end to end.
- *
- * <p>Skips unless the env var is set, so it never runs in normal CI. No secrets are committed.
+ * Opt-in: skipped unless {@code STIRLING_TEST_VALKEY_URL} is set, so it never runs in CI. A {@code
+ * rediss://} URL exercises the real TLS handshake (verifyPeer=FULL).
  */
 @EnabledIfEnvironmentVariable(named = "STIRLING_TEST_VALKEY_URL", matches = "rediss?://.+")
 class LiveExternalClusterTest {

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalSentinelTest.java

@@ -0,0 +1,127 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Opt-in, never runs in CI: needs STIRLING_TEST_VALKEY_SENTINEL_NODES (comma-separated host:port).
+ * Optional STIRLING_TEST_VALKEY_: SENTINEL_MASTER (default mymaster), SENTINEL_PASSWORD, PASSWORD.
+ */
+@EnabledIfEnvironmentVariable(named = "STIRLING_TEST_VALKEY_SENTINEL_NODES", matches = ".+")
+class LiveExternalSentinelTest {
+
+    private static final String NODE_ID = "sentinel-node";
+    private static final String RUN = UUID.randomUUID().toString().substring(0, 8);
+
+    private static LettuceConnectionFactory factory;
+    private static StringRedisTemplate template;
+
+    @BeforeAll
+    static void connect() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getNode().setId(NODE_ID);
+        var valkey = p.getCluster().getValkey();
+        valkey.setMode("sentinel");
+        valkey.getSentinel().setMaster(env("STIRLING_TEST_VALKEY_SENTINEL_MASTER", "mymaster"));
+        valkey.getSentinel()
+                .setNodes(
+                        Arrays.stream(
+                                        System.getenv("STIRLING_TEST_VALKEY_SENTINEL_NODES")
+                                                .split(","))
+                                .map(String::trim)
+                                .filter(s -> !s.isEmpty())
+                                .toList());
+        valkey.getSentinel().setPassword(env("STIRLING_TEST_VALKEY_SENTINEL_PASSWORD", ""));
+        valkey.setPassword(env("STIRLING_TEST_VALKEY_PASSWORD", ""));
+        // Production bean: sentinel discovery, credentials, pooling and the boot handshake.
+        factory = new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+        template = new StringRedisTemplate(factory);
+    }
+
+    @AfterAll
+    static void disconnect() {
+        if (factory != null) {
+            factory.destroy();
+        }
+    }
+
+    private static String env(String name, String fallback) {
+        String value = System.getenv(name);
+        return value == null || value.isBlank() ? fallback : value;
+    }
+
+    @Test
+    @DisplayName("sentinel-resolved primary is reachable and the backplane reports healthy")
+    void backplaneHealthyThroughSentinel() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().getNode().setId(NODE_ID);
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(p, template);
+        assertEquals("valkey", bp.backplaneType());
+        assertTrue(bp.isHealthy(), "sentinel must resolve a writable primary");
+    }
+
+    @Test
+    @DisplayName("JobStore round-trips through the sentinel-resolved primary")
+    void jobStoreRoundTripsThroughSentinel() {
+        ValkeyJobStore store = new ValkeyJobStore(template);
+        String jobId = "sent-job-" + RUN;
+        String fileId = "sent-file-" + RUN;
+
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.RUNNING,
+                        NODE_ID,
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(fileId),
+                        Map.of("k", "v")),
+                Duration.ofSeconds(60));
+
+        Optional<JobStoreEntry> seen = store.get(jobId);
+        assertTrue(seen.isPresent(), "the write must land on the primary, not a read-only replica");
+        assertEquals(NODE_ID, seen.get().owningNodeId());
+        assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
+
+        store.delete(jobId);
+        assertFalse(store.exists(jobId));
+        assertFalse(store.findJobIdByFileId(fileId).isPresent());
+    }
+
+    @Test
+    @DisplayName("InstanceRegistry register/deregister works through sentinel")
+    void instanceRegistryThroughSentinel() {
+        ValkeyInstanceRegistry registry = new ValkeyInstanceRegistry(template);
+        String nodeId = "sent-node-" + RUN;
+        registry.register(
+                new ClusterNode(nodeId, "10.0.0.9:8080", Instant.now(), "BOTH"),
+                Duration.ofSeconds(60));
+        assertTrue(registry.lookup(nodeId).isPresent());
+        registry.deregister(nodeId);
+        assertFalse(registry.lookup(nodeId).isPresent());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java

@@ -18,11 +18,8 @@ import org.testcontainers.utility.DockerImageName;
 import stirling.software.common.model.ApplicationProperties;
 
 /**
- * Live AUTH coverage against a password-protected Valkey. The main {@link
- * LiveValkeyIntegrationTest} runs against a no-auth instance, so the credential-bearing URL path
- * (parse -> RedisStandaloneConfiguration -> real AUTH handshake) was otherwise unexercised. Drives
- * the full production bean method {@code valkeyConnectionFactory()} so the parse, credential
- * wiring, and eager-handshake all run exactly as at boot.
+ * Live AUTH coverage: {@link LiveValkeyIntegrationTest} runs no-auth, so the credential-bearing URL
+ * path is otherwise unexercised. Drives the real {@code valkeyConnectionFactory()} bean.
  */
 @Testcontainers
 @EnabledIf("isDockerAvailable")

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java

@@ -20,13 +20,8 @@ import org.testcontainers.utility.DockerImageName;
 import stirling.software.common.model.ApplicationProperties;
 
 /**
- * Live failure-injection: a frozen (network-black-holed) Valkey must NOT stall hot-path commands
- * for Lettuce's 60s default. {@link ValkeyConnectionConfiguration} pins a 2s command timeout, so a
- * paused server must surface an error in seconds, and the connection must recover when it returns.
- *
- * <p>Uses {@code docker pause}/{@code unpause} (TCP stays ESTABLISHED but the server never replies)
- * to reproduce a partition rather than {@code stop} (which would fail fast with
- * connection-refused).
+ * Uses {@code docker pause}, not {@code stop}: TCP stays ESTABLISHED and the server never replies,
+ * which is a partition. {@code stop} would fail fast with connection-refused instead.
  */
 @Testcontainers
 @EnabledIf("isDockerAvailable")

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyClusterModeTest.java

@@ -0,0 +1,251 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import io.lettuce.core.cluster.RedisClusterClient;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.model.ApplicationProperties;
+
+// One node owning all 16384 slots exercises the RedisClusterClient path (rejects MULTI/WATCH).
+// It must announce 127.0.0.1:<mappedPort>: Lettuce follows the topology, not the seed URI.
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeyClusterModeTest {
+
+    private static final String NODE_ID = "cluster-node";
+    private static final String RUN = UUID.randomUUID().toString().substring(0, 8);
+
+    // Lifecycle is manual: the cluster must be formed and its announced address fixed up before
+    // any client connects, which has to happen after the mapped port is known.
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(6379)
+                    .withCommand(
+                            "valkey-server",
+                            "--cluster-enabled",
+                            "yes",
+                            "--cluster-config-file",
+                            "/tmp/nodes.conf",
+                            "--dir",
+                            "/tmp",
+                            "--appendonly",
+                            "no");
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    private static LettuceConnectionFactory factory;
+    private static StringRedisTemplate template;
+
+    @BeforeAll
+    static void formClusterAndConnect() throws Exception {
+        VALKEY.start();
+        String host = VALKEY.getHost();
+        int port = VALKEY.getMappedPort(6379);
+        String announceIp = "localhost".equalsIgnoreCase(host) ? "127.0.0.1" : host;
+
+        cli("config", "set", "cluster-announce-ip", announceIp);
+        cli("config", "set", "cluster-announce-port", String.valueOf(port));
+        cli("cluster", "addslotsrange", "0", "16383");
+        awaitAllSlotsServed();
+
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getNode().setId(NODE_ID);
+        p.getCluster().getValkey().setMode("cluster");
+        p.getCluster().getValkey().setNodes(List.of(announceIp + ":" + port));
+        factory = new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+        template = new StringRedisTemplate(factory);
+    }
+
+    @AfterAll
+    static void disconnect() {
+        if (factory != null) {
+            factory.destroy();
+        }
+        VALKEY.stop();
+    }
+
+    private static Container.ExecResult cli(String... args) throws Exception {
+        String[] cmd = new String[args.length + 1];
+        cmd[0] = "valkey-cli";
+        System.arraycopy(args, 0, cmd, 1, args.length);
+        Container.ExecResult res = VALKEY.execInContainer(cmd);
+        assertEquals(
+                0,
+                res.getExitCode(),
+                "valkey-cli " + String.join(" ", args) + " failed: " + res.getStderr());
+        return res;
+    }
+
+    /** cluster_state flips to ok before every slot is served, so both have to be waited on. */
+    private static void awaitAllSlotsServed() throws Exception {
+        long deadline = System.currentTimeMillis() + 60_000;
+        String last = "";
+        while (System.currentTimeMillis() < deadline) {
+            last = cli("cluster", "info").getStdout();
+            if (last.contains("cluster_state:ok") && last.contains("cluster_slots_ok:16384")) {
+                return;
+            }
+            Thread.sleep(250);
+        }
+        throw new IllegalStateException("cluster never became ready; last CLUSTER INFO:\n" + last);
+    }
+
+    @Test
+    @DisplayName("cluster mode wires a RedisClusterClient, not a standalone client")
+    void clusterModeUsesClusterClient() {
+        assertInstanceOf(RedisClusterClient.class, factory.getNativeClient());
+    }
+
+    @Test
+    @DisplayName("backplane health probe succeeds on Cluster (PING would fan out and fail)")
+    void backplaneHealthyOnCluster() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().getNode().setId(NODE_ID);
+        assertTrue(new ValkeyClusterBackplane(p, template).isHealthy());
+    }
+
+    @Test
+    @DisplayName("JobStore round-trips on Cluster: put, get, reverse index, all, delete")
+    void jobStoreRoundTripsOnCluster() {
+        ValkeyJobStore store = new ValkeyJobStore(template);
+        String jobId = "cl-job-" + RUN;
+        String fileId = "cl-file-" + RUN;
+
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.RUNNING,
+                        NODE_ID,
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(fileId),
+                        Map.of("k", "v")),
+                Duration.ofSeconds(60));
+
+        Optional<JobStoreEntry> seen = store.get(jobId);
+        assertTrue(seen.isPresent(), "MULTI/EXEC would have been rejected by the cluster client");
+        assertEquals(NODE_ID, seen.get().owningNodeId());
+        assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
+        // Both keys hash to different slots, so this also proves nothing needs a shared key tag.
+        assertTrue(
+                template.getExpire("stirling:job:" + jobId, java.util.concurrent.TimeUnit.SECONDS)
+                        > 0,
+                "the single-key script must arm the TTL on Cluster too");
+        assertTrue(store.all().stream().anyMatch(e -> jobId.equals(e.jobId())), "SCAN must work");
+
+        store.delete(jobId);
+        assertFalse(store.exists(jobId));
+        assertFalse(
+                store.findJobIdByFileId(fileId).isPresent(),
+                "the value-guarded index delete must run on Cluster");
+    }
+
+    @Test
+    @DisplayName("InstanceRegistry round-trips on Cluster: register, activeNodes, deregister")
+    void instanceRegistryRoundTripsOnCluster() {
+        ValkeyInstanceRegistry registry = new ValkeyInstanceRegistry(template);
+        String nodeId = "cl-node-" + RUN;
+        registry.register(
+                new ClusterNode(nodeId, "10.0.0.5:8080", Instant.now(), "BOTH"),
+                Duration.ofSeconds(60));
+
+        Optional<ClusterNode> looked = registry.lookup(nodeId);
+        assertTrue(looked.isPresent(), "register must not need MULTI/EXEC");
+        assertEquals("10.0.0.5:8080", looked.get().internalAddress());
+        assertTrue(registry.activeNodes().stream().anyMatch(n -> nodeId.equals(n.nodeId())));
+        assertTrue(
+                template.getExpire(
+                                "stirling:nodes:" + nodeId, java.util.concurrent.TimeUnit.SECONDS)
+                        > 0,
+                "a node hash without a TTL would mask a dead node as alive forever");
+
+        registry.deregister(nodeId);
+        assertFalse(registry.lookup(nodeId).isPresent());
+    }
+
+    @Test
+    @DisplayName("RateLimitStore boots on Cluster and enforces capacity (used to throw at startup)")
+    void rateLimitStoreWorksOnCluster() {
+        ValkeyRateLimitStore store = new ValkeyRateLimitStore(factory);
+        store.initProxyManager();
+        try {
+            String key = "cl-rl-" + RUN;
+            int allowed = 0;
+            for (int i = 0; i < 8; i++) {
+                if (store.tryConsume(key, 4, Duration.ofSeconds(60)).allowed()) {
+                    allowed++;
+                }
+            }
+            assertEquals(4, allowed, "the CAS bucket must stay single-slot and enforce capacity");
+        } finally {
+            store.shutdown();
+        }
+    }
+
+    @Test
+    @DisplayName("KeyValueCache evictNamespace clears every namespace key on Cluster")
+    void keyValueCacheEvictNamespaceOnCluster() {
+        ValkeyKeyValueCache cache = new ValkeyKeyValueCache(template);
+        String namespace = "cl-ns-" + RUN;
+        for (int i = 0; i < 25; i++) {
+            cache.put(namespace, "k" + i, "v" + i, Duration.ofSeconds(60));
+        }
+        assertEquals("v7", cache.get(namespace, "k7").orElse(null));
+
+        cache.evictNamespace(namespace);
+
+        for (int i = 0; i < 25; i++) {
+            assertFalse(
+                    cache.get(namespace, "k" + i).isPresent(),
+                    "bulk UNLINK must fan out per slot on Cluster; k" + i + " survived");
+        }
+    }
+
+    @Test
+    @DisplayName("DistributedLock is single-key and therefore correct on Cluster unchanged")
+    void distributedLockWorksOnCluster() {
+        ValkeyDistributedLock lock = new ValkeyDistributedLock(template);
+        String key = "cl-lock-" + RUN;
+
+        Optional<DistributedLock.LockHandle> held = lock.tryAcquire(key, Duration.ofSeconds(30));
+        assertTrue(held.isPresent());
+        assertFalse(
+                lock.tryAcquire(key, Duration.ofSeconds(30)).isPresent(),
+                "a second acquirer must be excluded");
+        assertTrue(held.get().renew(Duration.ofSeconds(60)));
+        held.get().release();
+        assertTrue(
+                lock.tryAcquire(key, Duration.ofSeconds(5)).isPresent(), "released lock reusable");
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java

@@ -33,11 +33,8 @@ import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
 import stirling.software.common.model.ApplicationProperties;
 
 /**
- * Live integration tests against a real Valkey instance, started by Testcontainers. The
- * {@code @EnabledIf} guard probes the Docker daemon via {@link
- * DockerClientFactory#isDockerAvailable()} (non-throwing) so the suite skips cleanly when Docker is
- * unavailable - without that guard, {@code @Testcontainers} would throw {@code initializationError}
- * (test FAILURE, not skip) on CI runners without Docker.
+ * The {@code @EnabledIf} Docker probe must stay: without it {@code @Testcontainers} throws {@code
+ * initializationError} - a FAILURE, not a skip - on runners without Docker.
  */
 @Testcontainers
 @EnabledIf("isDockerAvailable")
@@ -77,7 +74,7 @@ class LiveValkeyIntegrationTest {
     }
 
     @Test
-    @DisplayName("Valkey reachable and isHealthy() = true after PING round-trip")
+    @DisplayName("Valkey reachable and isHealthy() = true after the single-key probe")
     void backplaneHealthy() {
         ApplicationProperties propsA = newProps("node-A");
         ValkeyClusterBackplane bp = new ValkeyClusterBackplane(propsA, templateA);
@@ -254,7 +251,8 @@ class LiveValkeyIntegrationTest {
                         "BOTH");
         reg.register(node, Duration.ofSeconds(30));
 
-        // TTL must be positive; -1 would mean EXPIRE did not commit inside MULTI/EXEC.
+        // TTL must be positive; -1 would mean the PEXPIRE half of the script never ran, which
+        // would mask a dead node as alive forever.
         Long ttlMs =
                 templateA.getExpire(
                         "stirling:nodes:" + node.nodeId(),
@@ -265,7 +263,7 @@ class LiveValkeyIntegrationTest {
                 "register() must atomically arm TTL; expected (0, 30000] ms, got " + ttlMs);
 
         Optional<ClusterNode> seen = reg.lookup(node.nodeId());
-        assertTrue(seen.isPresent(), "hash fields must be visible after atomic register()");
+        assertTrue(seen.isPresent(), "hash fields must be visible after register()");
         assertEquals("10.0.0.99:8080", seen.get().internalAddress());
 
         reg.deregister(node.nodeId());
@@ -298,9 +296,8 @@ class LiveValkeyIntegrationTest {
         ValkeyRateLimitStore store = newRateLimitStore(factoryA);
         String key = "boundary-" + java.util.UUID.randomUUID();
         long capacity = 5;
-        // refillGreedy tops the bucket up continuously, one token every window/capacity. A 500ms
-        // window left the drain loop only 100ms before a 6th token appeared, so a slow Valkey
-        // round-trip broke the count; 4s spaces refills 800ms apart, clear of any burst.
+        // refillGreedy adds a token every window/capacity. A short window re-flakes this on a slow
+        // round-trip; 4s spaces refills 800ms apart. Do not shrink.
         Duration window = Duration.ofSeconds(4);
         long refillIntervalMs = window.toMillis() / capacity;
 
@@ -346,11 +343,12 @@ class LiveValkeyIntegrationTest {
     }
 
     @Test
-    @DisplayName("JobStore put is atomic (hash + TTL + reverse index visible together)")
-    void jobStorePutIsAtomic() {
+    @DisplayName("JobStore put arms a TTL on the hash AND on every reverse-index entry")
+    void jobStorePutArmsTtlOnEveryKey() {
         ValkeyJobStore store = new ValkeyJobStore(templateA);
         String jobId = "atomic-job-" + java.util.UUID.randomUUID();
-        String fileId = "atomic-file-" + java.util.UUID.randomUUID();
+        String fileA = "atomic-fileA-" + java.util.UUID.randomUUID();
+        String fileB = "atomic-fileB-" + java.util.UUID.randomUUID();
         store.put(
                 new JobStoreEntry(
                         jobId,
@@ -359,96 +357,99 @@ class LiveValkeyIntegrationTest {
                         Instant.now(),
                         null,
                         null,
-                        List.of(fileId),
+                        List.of(fileA, fileB),
                         Map.of("k", "v")),
                 Duration.ofSeconds(30));
 
         assertTrue(store.exists(jobId), "hash must be visible after put");
-        Long jobTtl =
-                templateA.getExpire(
-                        "stirling:job:" + jobId, java.util.concurrent.TimeUnit.MILLISECONDS);
-        assertNotNull(jobTtl);
-        assertTrue(jobTtl > 0, "hash must have TTL armed inside the same transaction");
-        assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
-        Long indexTtl =
-                templateA.getExpire(
-                        "stirling:file2job:" + fileId, java.util.concurrent.TimeUnit.MILLISECONDS);
-        assertNotNull(indexTtl);
-        assertTrue(indexTtl > 0, "reverse index must also have TTL armed");
+        // A TTL-less key would never evict and would leak for the life of the deployment; the
+        // single-key HSET+PEXPIRE script is what makes "hash without TTL" unreachable.
+        assertPositiveTtl("stirling:job:" + jobId, "job hash");
+        assertEquals(jobId, store.findJobIdByFileId(fileA).orElse(null));
+        assertEquals(jobId, store.findJobIdByFileId(fileB).orElse(null));
+        assertPositiveTtl("stirling:file2job:" + fileA, "reverse index fileA");
+        assertPositiveTtl("stirling:file2job:" + fileB, "reverse index fileB");
+    }
+
+    private void assertPositiveTtl(String key, String what) {
+        Long ttlMs = templateA.getExpire(key, java.util.concurrent.TimeUnit.MILLISECONDS);
+        assertNotNull(ttlMs, what + " must exist");
+        assertTrue(
+                ttlMs > 0 && ttlMs <= 30_000,
+                what + " must have a TTL in (0, 30000] ms, got " + ttlMs);
     }
 
     @Test
-    @DisplayName(
-            "JobStore.delete(): WATCH aborts when put() races between read and EXEC, no orphaned"
-                    + " reverse-index entries")
-    void jobStoreDeleteWatchRaceRetriesAndCleansUp() {
+    @DisplayName("JobStore.delete() never removes a reverse-index entry a NEWER job now owns")
+    void jobStoreDeleteIsValueGuarded() {
         ValkeyJobStore store = new ValkeyJobStore(templateA);
-        String jobId = "watch-race-job-" + java.util.UUID.randomUUID();
-        String originalFile = "orig-file-" + java.util.UUID.randomUUID();
-        String newFile = "new-file-" + java.util.UUID.randomUUID();
+        String oldJob = "guard-old-" + java.util.UUID.randomUUID();
+        String newJob = "guard-new-" + java.util.UUID.randomUUID();
+        String sharedFile = "guard-file-" + java.util.UUID.randomUUID();
 
         store.put(
                 new JobStoreEntry(
-                        jobId,
-                        JobStoreEntry.JobState.RUNNING,
+                        oldJob,
+                        JobStoreEntry.JobState.COMPLETE,
                         "node-A",
                         Instant.now(),
+                        Instant.now(),
+                        null,
+                        List.of(sharedFile),
+                        Map.of()),
+                Duration.ofSeconds(30));
+        // A later job takes ownership of the same fileId; the index row now points at newJob.
+        store.put(
+                new JobStoreEntry(
+                        newJob,
+                        JobStoreEntry.JobState.RUNNING,
+                        "node-B",
+                        Instant.now(),
                         null,
                         null,
-                        List.of(originalFile),
+                        List.of(sharedFile),
                         Map.of()),
                 Duration.ofSeconds(30));
+        assertEquals(newJob, store.findJobIdByFileId(sharedFile).orElse(null));
 
-        // Simulate the race: between delete()'s WATCH read and EXEC, add a new fileId.
-        // The first EXEC aborts; the retry catches the new fileId and deletes both entries.
-        Thread mutator =
-                new Thread(
-                        () -> {
-                            try {
-                                Thread.sleep(20);
-                            } catch (InterruptedException ignored) {
-                                Thread.currentThread().interrupt();
-                            }
-                            store.put(
-                                    new JobStoreEntry(
-                                            jobId,
-                                            JobStoreEntry.JobState.RUNNING,
-                                            "node-A",
-                                            Instant.now(),
-                                            null,
-                                            null,
-                                            List.of(originalFile, newFile),
-                                            Map.of()),
-                                    Duration.ofSeconds(30));
-                        });
-        mutator.start();
+        store.delete(oldJob);
 
-        store.delete(jobId);
-        try {
-            mutator.join(2000);
-        } catch (InterruptedException ignored) {
-            Thread.currentThread().interrupt();
-        }
-
-        boolean hashGone = !store.exists(jobId);
-        boolean origIndexGone = !store.findJobIdByFileId(originalFile).isPresent();
-        boolean newIndexGone = !store.findJobIdByFileId(newFile).isPresent();
-        if (hashGone) {
-            assertTrue(
-                    origIndexGone,
-                    "if hash is deleted, original reverse-index entry must also be gone");
-            assertTrue(
-                    newIndexGone,
-                    "if hash is deleted after the racing put(), the WATCH retry must catch the"
-                            + " new fileId and delete its reverse-index entry too");
-        } else {
-            assertEquals(jobId, store.findJobIdByFileId(originalFile).orElse(null));
-            assertEquals(jobId, store.findJobIdByFileId(newFile).orElse(null));
-        }
+        assertFalse(store.exists(oldJob), "the deleted job's hash must be gone");
+        assertEquals(
+                newJob,
+                store.findJobIdByFileId(sharedFile).orElse(null),
+                "deleting the older job must not strip the index row the newer job owns");
+        assertTrue(store.exists(newJob), "the newer job itself must be untouched");
     }
 
     @Test
-    @DisplayName("JobStore.delete() removes hash AND every reverse-index entry atomically")
+    @DisplayName("JobStore.delete() is idempotent and safe on a job that no longer exists")
+    void jobStoreDeleteIsIdempotent() {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        String jobId = "idem-job-" + java.util.UUID.randomUUID();
+        String fileId = "idem-file-" + java.util.UUID.randomUUID();
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.COMPLETE,
+                        "node-A",
+                        Instant.now(),
+                        Instant.now(),
+                        null,
+                        List.of(fileId),
+                        Map.of()),
+                Duration.ofSeconds(30));
+
+        store.delete(jobId);
+        store.delete(jobId);
+        store.delete("never-existed-" + java.util.UUID.randomUUID());
+
+        assertFalse(store.exists(jobId));
+        assertFalse(store.findJobIdByFileId(fileId).isPresent());
+    }
+
+    @Test
+    @DisplayName("JobStore.delete() removes the hash AND every reverse-index entry it owns")
     void jobStoreDeleteRemovesReverseIndexEntries() {
         ValkeyJobStore store = new ValkeyJobStore(templateA);
         String jobId = "del-atomic-job-" + java.util.UUID.randomUUID();
@@ -471,8 +472,7 @@ class LiveValkeyIntegrationTest {
 
         store.delete(jobId);
 
-        // Both the main hash AND every reverse-index entry must be gone; dangling reverse-index
-        // entries would cause findJobIdByFileId() to return a deleted jobId.
+        // Dangling reverse-index entries would make findJobIdByFileId() return a deleted jobId.
         assertFalse(store.exists(jobId), "main hash must be deleted");
         assertFalse(
                 store.findJobIdByFileId(fileA).isPresent(),

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyPoolingTest.java

@@ -0,0 +1,151 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.types.RedisClientInfo;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.model.ApplicationProperties;
+
+// Real server. Backplane commands multiplex over one shared native connection, and
+// connections without CLIENT SETNAME show as name="" in every Valkey monitor.
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeyPoolingTest {
+
+    private static final String NODE_ID = "pool-node";
+    private static final String EXPECTED_CLIENT_NAME = "stirling-" + NODE_ID;
+    private static final int WRITES = 200;
+
+    @Container
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(6379);
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    private static LettuceConnectionFactory factory;
+    private static StringRedisTemplate template;
+
+    @BeforeAll
+    static void connect() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getNode().setId(NODE_ID);
+        var valkey = p.getCluster().getValkey();
+        valkey.setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379));
+        valkey.getPool().setEnabled(true);
+        valkey.getPool().setMaxActive(4);
+        // Production bean: pooling, CLIENT SETNAME and the boot handshake all as they run at boot.
+        factory = new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+        template = new StringRedisTemplate(factory);
+    }
+
+    @AfterAll
+    static void disconnect() {
+        if (factory != null) {
+            factory.destroy();
+        }
+    }
+
+    @Test
+    @DisplayName("the production bean actually wires a pooling client configuration")
+    void factoryIsPooled() {
+        assertTrue(
+                factory.getClientConfiguration() instanceof LettucePoolingClientConfiguration,
+                "pool.enabled=true must reach the factory; nothing else pins the pool wiring");
+    }
+
+    @Test
+    @DisplayName(WRITES + " job writes multiplex over the single shared native connection")
+    void writesMultiplexOverTheSharedNativeConnection() {
+        // Not a pooling proof: shareNativeConnection defaults true and no command queues, so
+        // nothing borrows from the pool. This pins the multiplexing, which pooling cannot change.
+        long before = statLong("stats", "total_connections_received");
+
+        ValkeyJobStore store = new ValkeyJobStore(template);
+        for (int i = 0; i < WRITES; i++) {
+            store.put(
+                    new JobStoreEntry(
+                            "pool-job-" + i,
+                            JobStoreEntry.JobState.RUNNING,
+                            NODE_ID,
+                            Instant.now(),
+                            null,
+                            null,
+                            List.of("pool-file-" + i),
+                            Map.of("k", "v")),
+                    Duration.ofSeconds(60));
+        }
+
+        long delta = statLong("stats", "total_connections_received") - before;
+        assertTrue(
+                delta < 20,
+                "backplane writes must reuse the shared connection; "
+                        + WRITES
+                        + " writes opened "
+                        + delta
+                        + " new connections");
+        long connected = statLong("clients", "connected_clients");
+        assertTrue(
+                connected <= 6,
+                "connection count must stay flat under write load; connected_clients=" + connected);
+    }
+
+    @Test
+    @DisplayName(
+            "every connection is attributable: CLIENT LIST shows stirling-<nodeId>, never \"\"")
+    void everyConnectionIsNamed() {
+        // Force at least one write so the connection is live before the CLIENT LIST snapshot.
+        template.opsForValue().set("pool:probe", "v", Duration.ofSeconds(30));
+
+        List<RedisClientInfo> clients =
+                template.execute(
+                        (RedisCallback<List<RedisClientInfo>>)
+                                c -> c.serverCommands().getClientList());
+        assertNotNull(clients);
+        assertFalse(clients.isEmpty(), "CLIENT LIST must report at least our own connection");
+        for (RedisClientInfo client : clients) {
+            assertEquals(
+                    EXPECTED_CLIENT_NAME,
+                    client.getName(),
+                    "an unnamed connection is unattributable in any Valkey monitor; row: "
+                            + client);
+        }
+    }
+
+    private static long statLong(String section, String field) {
+        Properties info =
+                template.execute((RedisCallback<Properties>) c -> c.serverCommands().info(section));
+        assertNotNull(info, "INFO " + section + " must reply");
+        String raw = info.getProperty(field);
+        assertNotNull(raw, "INFO " + section + " must expose " + field);
+        return Long.parseLong(raw.trim());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeySentinelModeTest.java

@@ -0,0 +1,241 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import io.lettuce.core.RedisClient;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.model.ApplicationProperties;
+
+// Lettuce takes the primary's address from the SENTINEL'S REPLY, not from the seed URI, so the
+// announced address must resolve identically inside the container and on the host.
+// Primary and sentinel therefore share one container (localhost = the primary) and both ports are
+// published host:container identically, making "127.0.0.1:<port>" true on both sides.
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeySentinelModeTest {
+
+    private static final String NODE_ID = "sentinel-node";
+    private static final String MASTER = "mymaster";
+    private static final String RUN = UUID.randomUUID().toString().substring(0, 8);
+
+    private static final int PRIMARY_PORT;
+    private static final int SENTINEL_PORT;
+
+    static {
+        int[] ports = reserveTwoPorts();
+        PRIMARY_PORT = ports[0];
+        SENTINEL_PORT = ports[1];
+    }
+
+    // Lifecycle is manual: SENTINEL MONITOR must name a host-reachable address, and the sentinel
+    // must have seen the primary, before any client connects.
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(PRIMARY_PORT, SENTINEL_PORT)
+                    .withCommand("sh", "-c", bootstrapScript());
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    private static LettuceConnectionFactory factory;
+    private static StringRedisTemplate template;
+
+    @BeforeAll
+    static void monitorPrimaryAndConnect() throws Exception {
+        VALKEY.setPortBindings(
+                List.of(PRIMARY_PORT + ":" + PRIMARY_PORT, SENTINEL_PORT + ":" + SENTINEL_PORT));
+        VALKEY.start();
+        String host = VALKEY.getHost();
+        String announceIp = "localhost".equalsIgnoreCase(host) ? "127.0.0.1" : host;
+
+        // resolve-hostnames stays off (set in the config): a hostname whose record disappears
+        // stalls sentinel's event loop, and Lettuce would then get an address it cannot use.
+        cli("sentinel", "monitor", MASTER, announceIp, String.valueOf(PRIMARY_PORT), "1");
+        awaitPrimaryMonitored();
+
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getNode().setId(NODE_ID);
+        var valkey = p.getCluster().getValkey();
+        valkey.setMode("sentinel");
+        valkey.getSentinel().setMaster(MASTER);
+        valkey.getSentinel().setNodes(List.of(announceIp + ":" + SENTINEL_PORT));
+        factory = new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+        template = new StringRedisTemplate(factory);
+    }
+
+    @AfterAll
+    static void disconnect() {
+        if (factory != null) {
+            factory.destroy();
+        }
+        VALKEY.stop();
+    }
+
+    /** Primary in the background, sentinel in the foreground: same netns, so localhost is both. */
+    private static String bootstrapScript() {
+        return "valkey-server --port "
+                + PRIMARY_PORT
+                + " --save '' --appendonly no --daemonize yes"
+                + " && printf 'port "
+                + SENTINEL_PORT
+                + "\\ndir /tmp\\nsentinel resolve-hostnames no\\n' > /tmp/sentinel.conf"
+                + " && exec valkey-sentinel /tmp/sentinel.conf";
+    }
+
+    /** Both sockets are held open at once so the two ports cannot collide. */
+    private static int[] reserveTwoPorts() {
+        try (ServerSocket first = new ServerSocket(0);
+                ServerSocket second = new ServerSocket(0)) {
+            return new int[] {first.getLocalPort(), second.getLocalPort()};
+        } catch (IOException ex) {
+            throw new IllegalStateException("could not reserve local ports for valkey", ex);
+        }
+    }
+
+    private static Container.ExecResult cli(String... args) throws Exception {
+        String[] cmd = new String[args.length + 3];
+        cmd[0] = "valkey-cli";
+        cmd[1] = "-p";
+        cmd[2] = String.valueOf(SENTINEL_PORT);
+        System.arraycopy(args, 0, cmd, 3, args.length);
+        Container.ExecResult res = VALKEY.execInContainer(cmd);
+        assertEquals(
+                0,
+                res.getExitCode(),
+                "valkey-cli " + String.join(" ", args) + " failed: " + res.getStderr());
+        return res;
+    }
+
+    /** SENTINEL MONITOR returns OK before sentinel has reached the primary; wait for both. */
+    private static void awaitPrimaryMonitored() throws Exception {
+        long deadline = System.currentTimeMillis() + 60_000;
+        String last = "";
+        while (System.currentTimeMillis() < deadline) {
+            last = cli("sentinel", "master", MASTER).getStdout();
+            boolean quorum = cli("sentinel", "ckquorum", MASTER).getStdout().startsWith("OK");
+            if (quorum && !last.contains("_down") && last.contains(String.valueOf(PRIMARY_PORT))) {
+                return;
+            }
+            Thread.sleep(250);
+        }
+        throw new IllegalStateException(
+                "sentinel never saw the primary; last SENTINEL MASTER:\n" + last);
+    }
+
+    @Test
+    @DisplayName("sentinel mode wires a standalone client against the sentinel-resolved primary")
+    void sentinelModeUsesSentinelAwareFactory() {
+        assertTrue(factory.isRedisSentinelAware(), "the factory must hold a sentinel config");
+        assertInstanceOf(RedisClient.class, factory.getNativeClient());
+    }
+
+    @Test
+    @DisplayName("backplane health probe succeeds through sentinel")
+    void backplaneHealthyThroughSentinel() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().getNode().setId(NODE_ID);
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(p, template);
+        assertEquals("valkey", bp.backplaneType());
+        assertTrue(bp.isHealthy(), "sentinel must resolve a writable primary");
+    }
+
+    @Test
+    @DisplayName("JobStore round-trips through the sentinel-resolved primary")
+    void jobStoreRoundTripsThroughSentinel() throws Exception {
+        ValkeyJobStore store = new ValkeyJobStore(template);
+        String jobId = "sent-job-" + RUN;
+        String fileId = "sent-file-" + RUN;
+
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.RUNNING,
+                        NODE_ID,
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(fileId),
+                        Map.of("k", "v")),
+                Duration.ofSeconds(60));
+
+        Optional<JobStoreEntry> seen = store.get(jobId);
+        assertTrue(seen.isPresent(), "the write must land on the primary, not a read-only replica");
+        assertEquals(NODE_ID, seen.get().owningNodeId());
+        assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
+        assertTrue(
+                template.getExpire("stirling:job:" + jobId, java.util.concurrent.TimeUnit.SECONDS)
+                        > 0,
+                "the single-key script must arm the TTL through sentinel too");
+        // Proves the sentinel-announced address is the container's primary, not some other server.
+        assertEquals(
+                "1",
+                onPrimary("exists", "stirling:job:" + jobId).getStdout().trim(),
+                "the key must exist on the primary sentinel monitors");
+
+        store.delete(jobId);
+        assertFalse(store.exists(jobId));
+        assertFalse(store.findJobIdByFileId(fileId).isPresent());
+    }
+
+    @Test
+    @DisplayName("InstanceRegistry register/lookup/deregister works through sentinel")
+    void instanceRegistryThroughSentinel() {
+        ValkeyInstanceRegistry registry = new ValkeyInstanceRegistry(template);
+        String nodeId = "sent-node-" + RUN;
+        registry.register(
+                new ClusterNode(nodeId, "10.0.0.9:8080", Instant.now(), "BOTH"),
+                Duration.ofSeconds(60));
+
+        Optional<ClusterNode> looked = registry.lookup(nodeId);
+        assertTrue(looked.isPresent());
+        assertEquals("10.0.0.9:8080", looked.get().internalAddress());
+        assertTrue(registry.activeNodes().stream().anyMatch(n -> nodeId.equals(n.nodeId())));
+        assertTrue(
+                template.getExpire(
+                                "stirling:nodes:" + nodeId, java.util.concurrent.TimeUnit.SECONDS)
+                        > 0,
+                "a node hash without a TTL would mask a dead node as alive forever");
+
+        registry.deregister(nodeId);
+        assertFalse(registry.lookup(nodeId).isPresent());
+    }
+
+    private static Container.ExecResult onPrimary(String... args) throws Exception {
+        String[] cmd = new String[args.length + 3];
+        cmd[0] = "valkey-cli";
+        cmd[1] = "-p";
+        cmd[2] = String.valueOf(PRIMARY_PORT);
+        System.arraycopy(args, 0, cmd, 3, args.length);
+        return VALKEY.execInContainer(cmd);
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java

@@ -1,61 +1,174 @@
 package stirling.software.proprietary.cluster.valkey;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.boot.health.contributor.Status;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.data.redis.RedisConnectionFailureException;
 import org.springframework.data.redis.core.RedisCallback;
 import org.springframework.data.redis.core.StringRedisTemplate;
 
 import stirling.software.common.model.ApplicationProperties;
 
-/**
- * S3 regression: {@link ValkeyClusterBackplane#isHealthy()} must route through {@code
- * template.execute(...)} so the borrowed connection is always returned to the pool. Calling {@code
- * getConnectionFactory().getConnection()} directly would leak the connection on every k8s liveness
- * probe tick and exhaust the pool under monitoring load.
- */
+// The probe must be a single-key command via the template: PING fans out to every cluster
+// master and fails if any one node is down, restarting healthy pods on each liveness tick.
 class ValkeyClusterBackplaneTest {
 
-    @Test
-    void isHealthy_routesThroughTemplateExecute_andDoesNotTouchConnectionFactoryDirectly() {
-        StringRedisTemplate template = mock(StringRedisTemplate.class);
-        when(template.execute(any(RedisCallback.class))).thenReturn("PONG");
-
+    private static ValkeyClusterBackplane backplane(StringRedisTemplate template) {
         ApplicationProperties props = new ApplicationProperties();
         props.getCluster().getNode().setId("n-1");
-        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
-
-        assertTrue(bp.isHealthy());
-        verify(template, times(1)).execute(any(RedisCallback.class));
-        // Critical: never bypass the template's connection management.
-        verify(template, never()).getConnectionFactory();
+        return new ValkeyClusterBackplane(props, template);
     }
 
     @Test
-    void isHealthy_returnsFalseWhenExecuteThrows() {
+    @DisplayName("healthy when the single-key probe answers; PING is never issued")
+    void isHealthy_usesSingleKeyProbe_andNeverPings() {
         StringRedisTemplate template = mock(StringRedisTemplate.class);
-        when(template.execute(any(RedisCallback.class))).thenThrow(new RuntimeException("boom"));
+        when(template.hasKey(anyString())).thenReturn(Boolean.FALSE);
 
-        ApplicationProperties props = new ApplicationProperties();
-        props.getCluster().getNode().setId("n-1");
-        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
+        assertTrue(backplane(template).isHealthy());
 
-        assertFalse(bp.isHealthy());
+        // The signal is "the command completed", not the boolean - the key is never written.
+        verify(template, times(1)).hasKey("stirling:health:n-1");
+        // Critical: a PING would fan out across a cluster and never bypass the template.
+        verify(template, never()).execute(any(RedisCallback.class));
+        verify(template, never()).getConnectionFactory();
+    }
+
+    // Replaces an assertion on a null hasKey() reply: Lettuce only returns null while queueing
+    // inside a pipeline or MULTI, and this template is neither, so that branch was unreachable.
+    @Test
+    @DisplayName("healthy whichever boolean the probe returns - only the round trip matters")
+    void isHealthy_ignoresProbeReplyValue() {
+        StringRedisTemplate exists = mock(StringRedisTemplate.class);
+        when(exists.hasKey(anyString())).thenReturn(Boolean.TRUE);
+        assertTrue(backplane(exists).isHealthy());
+
+        StringRedisTemplate missing = mock(StringRedisTemplate.class);
+        when(missing.hasKey(anyString())).thenReturn(Boolean.FALSE);
+        assertTrue(backplane(missing).isHealthy());
+    }
+
+    @Test
+    @DisplayName("unhealthy when the connection to Valkey is down")
+    void isHealthy_returnsFalseOnConnectionFailure() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        // The exception a real outage surfaces, rather than a synthetic RuntimeException.
+        when(template.hasKey(anyString()))
+                .thenThrow(new RedisConnectionFailureException("connection refused"));
+
+        assertFalse(backplane(template).isHealthy());
+    }
+
+    @Test
+    @DisplayName("unhealthy (not propagated) when the probe throws")
+    void isHealthy_returnsFalseWhenProbeThrows() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        when(template.hasKey(anyString())).thenThrow(new RuntimeException("boom"));
+
+        assertFalse(backplane(template).isHealthy());
     }
 
     @Test
     void shouldRunLocalCleanup_returnsFalse_valkeyOwnsTtlEviction() {
-        StringRedisTemplate template = mock(StringRedisTemplate.class);
-        ApplicationProperties props = new ApplicationProperties();
-        props.getCluster().getNode().setId("n-1");
-        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
-        assertFalse(bp.shouldRunLocalCleanup());
+        assertFalse(backplane(mock(StringRedisTemplate.class)).shouldRunLocalCleanup());
+    }
+
+    @Nested
+    @DisplayName("health indicator")
+    class BackplaneHealthIndicator {
+
+        private final ApplicationContextRunner runner =
+                new ApplicationContextRunner()
+                        .withUserConfiguration(ValkeyBackplaneHealthIndicator.class);
+
+        private static Health health(StringRedisTemplate template) {
+            return new ValkeyBackplaneHealthIndicator(backplane(template)).health();
+        }
+
+        @Test
+        @DisplayName("UP with backplane and node details when the probe answers")
+        void reportsUp() {
+            StringRedisTemplate template = mock(StringRedisTemplate.class);
+            when(template.hasKey(anyString())).thenReturn(Boolean.FALSE);
+
+            Health health = health(template);
+
+            assertEquals(Status.UP, health.getStatus());
+            assertEquals("valkey", health.getDetails().get("backplane"));
+            assertEquals("n-1", health.getDetails().get("nodeId"));
+        }
+
+        @Test
+        @DisplayName("DOWN when Valkey is unreachable, so the node drops out of rotation")
+        void reportsDownOnOutage() {
+            StringRedisTemplate template = mock(StringRedisTemplate.class);
+            when(template.hasKey(anyString()))
+                    .thenThrow(new RedisConnectionFailureException("connection refused"));
+
+            Health health = health(template);
+
+            assertEquals(Status.DOWN, health.getStatus());
+            assertEquals("valkey", health.getDetails().get("backplane"));
+        }
+
+        @Test
+        @DisplayName("DOWN rather than throwing when node-id resolution fails")
+        void reportsDownInsteadOfPropagating() {
+            ValkeyClusterBackplane broken = mock(ValkeyClusterBackplane.class);
+            when(broken.isHealthy()).thenReturn(true);
+            when(broken.backplaneType()).thenReturn("valkey");
+            when(broken.localNodeId()).thenThrow(new IllegalStateException("no node id"));
+
+            Health health = new ValkeyBackplaneHealthIndicator(broken).health();
+
+            assertEquals(Status.DOWN, health.getStatus());
+        }
+
+        @Test
+        @DisplayName("not registered on a single-node install")
+        void absentWhenClusterDisabled() {
+            runner.run(
+                    context ->
+                            assertThat(context)
+                                    .doesNotHaveBean(ValkeyBackplaneHealthIndicator.class));
+        }
+
+        @Test
+        @DisplayName("not registered when clustering uses the in-process backplane")
+        void absentWhenBackplaneIsInProcess() {
+            runner.withPropertyValues("cluster.enabled=true", "cluster.backplane=inprocess")
+                    .run(
+                            context ->
+                                    assertThat(context)
+                                            .doesNotHaveBean(ValkeyBackplaneHealthIndicator.class));
+        }
+
+        @Test
+        @DisplayName("registered when the Valkey backplane is active")
+        void presentWhenValkeyBackplaneActive() {
+            runner.withPropertyValues("cluster.enabled=true", "cluster.backplane=valkey")
+                    .withBean(
+                            ValkeyClusterBackplane.class,
+                            () -> backplane(mock(StringRedisTemplate.class)))
+                    .run(
+                            context ->
+                                    assertThat(context)
+                                            .hasSingleBean(ValkeyBackplaneHealthIndicator.class));
+        }
     }
 }

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStoreTest.java

@@ -0,0 +1,118 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockingDetails;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.Invocation;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.ValueOperations;
+import org.springframework.data.redis.core.script.RedisScript;
+
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.JobStoreEntry.JobState;
+
+class ValkeyJobStoreTest {
+
+    private static final String INDEX_KEY = "stirling:file2job:file-a";
+    private static final String JOB_KEY = "stirling:job:job-1";
+
+    private static JobStoreEntry entry() {
+        return new JobStoreEntry(
+                "job-1",
+                JobState.COMPLETE,
+                "node-1",
+                Instant.EPOCH,
+                Instant.EPOCH,
+                null,
+                List.of("file-a"),
+                Map.of());
+    }
+
+    @Test
+    @DisplayName("a non-positive TTL removes the index row through the ownership guard, not DEL")
+    void expiredOnWriteUsesTheValueGuardedDelete() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+
+        // stirling.jobResultExpiryMinutes=0 reaches put() with an already-expired entry.
+        new ValkeyJobStore(template).put(entry(), Duration.ZERO);
+
+        @SuppressWarnings("rawtypes")
+        ArgumentCaptor<RedisScript> script = ArgumentCaptor.forClass(RedisScript.class);
+        @SuppressWarnings("unchecked")
+        ArgumentCaptor<List<String>> keys = ArgumentCaptor.forClass(List.class);
+        ArgumentCaptor<Object> args = ArgumentCaptor.forClass(Object.class);
+        verify(template).execute(script.capture(), keys.capture(), args.capture());
+
+        assertEquals(List.of(INDEX_KEY), keys.getValue());
+        assertEquals("job-1", args.getValue());
+        assertTrue(
+                script.getValue().getScriptAsString().contains("get"),
+                "the index row must be removed only while this job still owns it");
+        // An unguarded DEL here would drop an index row a newer job already claimed.
+        verify(template, never()).delete(INDEX_KEY);
+        verify(template).delete("stirling:job:job-1");
+    }
+
+    @Test
+    @DisplayName("a live TTL writes the job hash and the index row with that TTL, deleting nothing")
+    void liveTtlWritesTheJobHashAndIndexRow() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        @SuppressWarnings("unchecked")
+        ValueOperations<String, String> values = mock(ValueOperations.class);
+        when(template.opsForValue()).thenReturn(values);
+
+        new ValkeyJobStore(template).put(entry(), Duration.ofMinutes(5));
+
+        // Shape-agnostic: a row may be written by its own command or by one multi-key script.
+        List<String> writtenKeys = new ArrayList<>();
+        List<String> writtenArgs = new ArrayList<>();
+        for (Invocation invocation : mockingDetails(template).getInvocations()) {
+            Object[] raw = invocation.getRawArguments();
+            if (!"execute".equals(invocation.getMethod().getName()) || raw.length < 3) {
+                continue;
+            }
+            if (raw[1] instanceof List<?> keys && raw[2] instanceof Object[] args) {
+                keys.forEach(key -> writtenKeys.add(String.valueOf(key)));
+                for (Object arg : args) {
+                    writtenArgs.add(String.valueOf(arg));
+                }
+            }
+        }
+
+        String ttlMillis = Long.toString(Duration.ofMinutes(5).toMillis());
+        assertTrue(writtenKeys.contains(JOB_KEY), "put() must write the job hash, not just index");
+        assertTrue(writtenArgs.contains(ttlMillis), "the job hash must be written with its TTL");
+        assertTrue(
+                writtenArgs.containsAll(List.of("jobId", "job-1", "state", "COMPLETE")),
+                "the job hash must carry the entry's fields");
+
+        ArgumentCaptor<String> setKey = ArgumentCaptor.forClass(String.class);
+        ArgumentCaptor<String> setValue = ArgumentCaptor.forClass(String.class);
+        ArgumentCaptor<Duration> setTtl = ArgumentCaptor.forClass(Duration.class);
+        verify(values, atLeast(0)).set(setKey.capture(), setValue.capture(), setTtl.capture());
+        boolean indexSetPx =
+                setKey.getAllValues().contains(INDEX_KEY)
+                        && setValue.getAllValues().contains("job-1")
+                        && setTtl.getAllValues().contains(Duration.ofMinutes(5));
+        assertTrue(
+                indexSetPx || writtenKeys.contains(INDEX_KEY),
+                "put() must write the file index row");
+        verify(template, never()).delete(anyString());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicySaveShapeTest.java

@@ -0,0 +1,222 @@
+package stirling.software.proprietary.policy.controller;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.http.MediaType;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.MvcResult;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.servlet.mvc.annotation.ResponseStatusExceptionResolver;
+import org.springframework.web.servlet.mvc.support.DefaultHandlerExceptionResolver;
+
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.inprocess.InProcessJobStore;
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.service.JobOwnershipService;
+import stirling.software.common.util.TempFileManager;
+import stirling.software.proprietary.policy.asset.PolicyAssetCleaner;
+import stirling.software.proprietary.policy.asset.PolicyAssetResolver;
+import stirling.software.proprietary.policy.config.PolicyAccessGuard;
+import stirling.software.proprietary.policy.config.PolicyManagementAuthority;
+import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
+import stirling.software.proprietary.policy.engine.PolicyRunner;
+import stirling.software.proprietary.policy.engine.PolicyValidator;
+import stirling.software.proprietary.policy.ledger.ProcessedLedger;
+import stirling.software.proprietary.policy.model.Policy;
+import stirling.software.proprietary.policy.overview.PolicyOverviewService;
+import stirling.software.proprietary.policy.source.Source;
+import stirling.software.proprietary.policy.source.SourceAccessGuard;
+import stirling.software.proprietary.policy.source.SourceDocCounter;
+import stirling.software.proprietary.policy.source.SourceStore;
+import stirling.software.proprietary.policy.store.PolicyStore;
+import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
+
+/**
+ * The save endpoint's body shape. {@link Policy} has no {@code sourceIds} or {@code trigger} field,
+ * and unknown properties are ignored, so the pre-inputs body used to store a policy that referenced
+ * and watched nothing while still returning 200. It must be an error instead.
+ */
+@ExtendWith(MockitoExtension.class)
+@DisplayName("Policy save body shape")
+class PolicySaveShapeTest {
+
+    @Mock private PolicyRunner policyRunner;
+    @Mock private PolicyRunRegistry runRegistry;
+    @Mock private PolicyStore policyStore;
+    @Mock private SourceStore sourceStore;
+    @Mock private SourceAccessGuard sourceAccessGuard;
+    @Mock private SourceDocCounter docCounter;
+    @Mock private PolicyValidator policyValidator;
+    @Mock private PolicyAccessGuard policyAccessGuard;
+    @Mock private PolicyManagementAuthority policyManagementAuthority;
+    @Mock private PolicyTriggerManager policyTriggerManager;
+    @Mock private PolicyOverviewService policyOverviewService;
+    @Mock private PolicyAssetCleaner assetCleaner;
+    @Mock private PolicyAssetResolver assetResolver;
+    @Mock private ProcessedLedger processedLedger;
+    @Mock private TempFileManager tempFileManager;
+    @Mock private JobOwnershipService jobOwnershipService;
+
+    private final JobStore jobStore = new InProcessJobStore();
+    private MockMvc mockMvc;
+
+    @BeforeEach
+    void setUp() {
+        ApplicationProperties applicationProperties = new ApplicationProperties();
+        // Login off: policy editing is then open to the local operator, so the shape is what's
+        // under test rather than the role gate.
+        applicationProperties.getSecurity().setEnableLogin(false);
+        PolicyController controller =
+                new PolicyController(
+                        policyRunner,
+                        runRegistry,
+                        policyStore,
+                        sourceStore,
+                        sourceAccessGuard,
+                        docCounter,
+                        policyValidator,
+                        policyAccessGuard,
+                        policyManagementAuthority,
+                        policyTriggerManager,
+                        policyOverviewService,
+                        assetCleaner,
+                        assetResolver,
+                        processedLedger,
+                        List.of(),
+                        applicationProperties,
+                        tempFileManager,
+                        jobOwnershipService,
+                        jobStore);
+        mockMvc =
+                MockMvcBuilders.standaloneSetup(controller)
+                        // standaloneSetup's defaults don't handle ResponseStatusException.
+                        .setHandlerExceptionResolvers(
+                                new ResponseStatusExceptionResolver(),
+                                new DefaultHandlerExceptionResolver())
+                        .build();
+    }
+
+    private MvcResult save(String body) throws Exception {
+        return mockMvc.perform(
+                        post("/api/v1/policies")
+                                .contentType(MediaType.APPLICATION_JSON)
+                                .content(body))
+                .andReturn();
+    }
+
+    @Test
+    @DisplayName("the legacy sourceIds key is rejected instead of silently dropped")
+    void legacySourceIdsRejected() throws Exception {
+        MvcResult result =
+                save(
+                        """
+                        {"name":"legacy","enabled":true,"sourceIds":["src-1"],
+                         "steps":[{"operation":"/api/v1/misc/compress-pdf","parameters":{}}],
+                         "output":{"type":"inline","options":{}}}
+                        """);
+
+        assertThat(result.getResponse().getStatus()).isEqualTo(400);
+        assertThat(result.getResponse().getErrorMessage()).contains("sourceIds").contains("inputs");
+        verify(policyStore, never()).save(any());
+    }
+
+    @Test
+    @DisplayName("the legacy policy-level trigger is rejected instead of silently dropped")
+    void legacyTriggerRejected() throws Exception {
+        MvcResult result =
+                save(
+                        """
+                        {"name":"legacy","enabled":true,
+                         "trigger":{"type":"folder-watch","options":{}},
+                         "steps":[{"operation":"/api/v1/misc/compress-pdf","parameters":{}}],
+                         "output":{"type":"inline","options":{}}}
+                        """);
+
+        assertThat(result.getResponse().getStatus()).isEqualTo(400);
+        assertThat(result.getResponse().getErrorMessage()).contains("trigger").contains("inputs");
+        verify(policyStore, never()).save(any());
+    }
+
+    @Test
+    @DisplayName("the inputs shape is stored with its source reference intact")
+    void modernInputsAccepted() throws Exception {
+        Source source = new Source("src-1", "Incoming", "folder", Map.of(), true, "alice", 1L);
+        when(sourceStore.get("src-1")).thenReturn(Optional.of(source));
+        when(sourceAccessGuard.canAccess(source)).thenReturn(true);
+        when(policyStore.save(any())).thenAnswer(invocation -> invocation.getArgument(0));
+
+        mockMvc.perform(
+                        post("/api/v1/policies")
+                                .contentType(MediaType.APPLICATION_JSON)
+                                .content(
+                                        """
+                                        {"name":"modern","enabled":true,
+                                         "inputs":[{"sourceId":"src-1","trigger":null}],
+                                         "steps":[{"operation":"/api/v1/misc/compress-pdf",
+                                                   "parameters":{}}],
+                                         "output":{"type":"inline","options":{}}}
+                                        """))
+                .andExpect(status().isOk());
+
+        ArgumentCaptor<Policy> stored = ArgumentCaptor.forClass(Policy.class);
+        verify(policyStore).save(stored.capture());
+        assertThat(stored.getValue().sourceIds()).containsExactly("src-1");
+    }
+
+    @Test
+    @DisplayName("a body with neither key still saves")
+    void bodyWithNeitherKeyAccepted() throws Exception {
+        when(policyStore.save(any())).thenAnswer(invocation -> invocation.getArgument(0));
+
+        mockMvc.perform(
+                        post("/api/v1/policies")
+                                .contentType(MediaType.APPLICATION_JSON)
+                                .content(
+                                        """
+                                        {"name":"editor","enabled":true,
+                                         "steps":[{"operation":"/api/v1/misc/compress-pdf",
+                                                   "parameters":{}}],
+                                         "output":{"type":"inline","options":{}}}
+                                        """))
+                .andExpect(status().isOk());
+
+        verify(policyStore).save(any());
+    }
+
+    @Test
+    @DisplayName("a null trigger still saves - the editor and portal send one on every save")
+    void nullTriggerAccepted() throws Exception {
+        when(policyStore.save(any())).thenAnswer(invocation -> invocation.getArgument(0));
+
+        mockMvc.perform(
+                        post("/api/v1/policies")
+                                .contentType(MediaType.APPLICATION_JSON)
+                                .content(
+                                        """
+                                        {"name":"editor","enabled":true,"trigger":null,
+                                         "steps":[{"operation":"/api/v1/misc/compress-pdf",
+                                                   "parameters":{}}],
+                                         "output":{"type":"inline","options":{}}}
+                                        """))
+                .andExpect(status().isOk());
+
+        verify(policyStore).save(any());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/policy/seed/DefaultClassificationPolicySeederTest.java

@@ -124,7 +124,7 @@ class DefaultClassificationPolicySeederTest {
         Team defaultTeam = new Team();
         defaultTeam.setId(1L);
         defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                 .thenReturn(Optional.of(defaultTeam));
         when(policyStore.findByTeam(1L)).thenReturn(List.of());
 
@@ -135,7 +135,8 @@ class DefaultClassificationPolicySeederTest {
 
     @Test
     void doesNotSeedOnStartupWhenThereIsNoDefaultTeam() {
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME)).thenReturn(Optional.empty());
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
+                .thenReturn(Optional.empty());
 
         seeder().seedDefaultTeamOnStartup();
 

app/proprietary/src/test/java/stirling/software/proprietary/security/InitialSecuritySetupTest.java

@@ -2,7 +2,9 @@ package stirling.software.proprietary.security;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -17,6 +19,7 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.core.env.Environment;
+import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import stirling.software.common.model.ApplicationProperties;
@@ -107,6 +110,27 @@ class InitialSecuritySetupTest {
         assertThat(saved.getPassword()).isEqualTo("password");
     }
 
+    @Test
+    void initRetriesUntilPeerRacesStopColliding() {
+        // Cluster cold boot: peers collide on a different row each pass, so one retry is not
+        // enough. Two collisions then success proves the loop converges instead of exiting.
+        when(userService.hasUsers()).thenReturn(true);
+        when(userService.getUsersWithoutTeam()).thenReturn(Collections.emptyList());
+        when(userService.usernameExistsIgnoreCase(any())).thenReturn(true);
+        doThrow(new DataIntegrityViolationException("duplicate key: users_username_key"))
+                .doThrow(
+                        new DataIntegrityViolationException("duplicate key: license_settings_pkey"))
+                .doNothing()
+                .when(licenseSettingsService)
+                .initializeGrandfatheredCount();
+
+        initialSecuritySetup.init();
+
+        verify(licenseSettingsService, times(3)).initializeGrandfatheredCount();
+        // Only the pass that got through may complete the remaining steps.
+        verify(licenseSettingsService, times(1)).updateLicenseMaxUsers();
+    }
+
     @Test
     void configureJwtSettingsDisablesKeyCleanupWhenJwtDisabled() {
         ReflectionTestUtils.setField(initialSecuritySetup, "v2Enabled", true);

app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/InviteLinkControllerTest.java

@@ -117,7 +117,7 @@ class InviteLinkControllerTest {
         Team defaultTeam = new Team();
         defaultTeam.setId(1L);
         defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                 .thenReturn(Optional.of(defaultTeam));
 
         mockMvc.perform(
@@ -132,7 +132,7 @@ class InviteLinkControllerTest {
         Team defaultTeam = new Team();
         defaultTeam.setId(5L);
         defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                 .thenReturn(Optional.of(defaultTeam));
         when(userService.usernameExistsIgnoreCase("new@example.com")).thenReturn(false);
         when(inviteTokenRepository.findByEmail("new@example.com")).thenReturn(Optional.empty());

app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerMoreTest.java

@@ -253,7 +253,7 @@ class UserControllerMoreTest {
             Team defaultTeam = new Team();
             defaultTeam.setId(1L);
             defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
-            when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+            when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                     .thenReturn(Optional.of(defaultTeam));
 
             mockMvc.perform(
@@ -461,7 +461,7 @@ class UserControllerMoreTest {
             Team defaultTeam = new Team();
             defaultTeam.setId(1L);
             defaultTeam.setName(TeamService.DEFAULT_TEAM_NAME);
-            when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+            when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                     .thenReturn(Optional.of(defaultTeam));
 
             mockMvc.perform(

app/proprietary/src/test/java/stirling/software/proprietary/security/controller/api/UserControllerTest.java

@@ -107,7 +107,7 @@ class UserControllerTest {
         when(userService.usernameExistsIgnoreCase("new@example.com")).thenReturn(false);
         when(userService.isUsernameValid("new@example.com")).thenReturn(true);
         when(licenseSettingsService.wouldExceedLimit(1)).thenReturn(false);
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                 .thenReturn(Optional.of(defaultTeam));
 
         User savedUser = new User();

app/proprietary/src/test/java/stirling/software/proprietary/security/service/TeamServiceDuplicateNameDbTest.java

@@ -0,0 +1,106 @@
+package stirling.software.proprietary.security.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringBootConfiguration;
+import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
+import org.springframework.boot.persistence.autoconfigure.EntityScan;
+import org.springframework.dao.IncorrectResultSizeDataAccessException;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+
+import stirling.software.proprietary.model.Team;
+import stirling.software.proprietary.security.repository.TeamRepository;
+
+/**
+ * Peers cold-booting a shared DB can each commit a "Default" row, because teams.name carries no
+ * unique constraint. Every node must still converge on one team instead of failing to boot.
+ */
+@DataJpaTest
+class TeamServiceDuplicateNameDbTest {
+
+    @Autowired private TeamRepository teamRepository;
+
+    private Long saveTeam(String name) {
+        Team team = new Team();
+        team.setName(name);
+        return teamRepository.saveAndFlush(team).getId();
+    }
+
+    @Test
+    @DisplayName("two same-named rows really can be committed - the race is not hypothetical")
+    void duplicateNamesArePersistable() {
+        Long first = saveTeam(TeamService.DEFAULT_TEAM_NAME);
+        Long second = saveTeam(TeamService.DEFAULT_TEAM_NAME);
+
+        assertEquals(
+                2,
+                teamRepository.findAll().stream()
+                        .filter(t -> TeamService.DEFAULT_TEAM_NAME.equals(t.getName()))
+                        .count(),
+                "no unique constraint on teams.name, so both inserts commit");
+        assertEquals(true, first < second, "ids are monotonic, so 'lowest' is well defined");
+    }
+
+    @Test
+    @DisplayName("the old finder throws on duplicates - this is what killed startup")
+    void findByNameThrowsOnDuplicates() {
+        saveTeam(TeamService.DEFAULT_TEAM_NAME);
+        saveTeam(TeamService.DEFAULT_TEAM_NAME);
+
+        assertThrows(
+                IncorrectResultSizeDataAccessException.class,
+                () -> teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME),
+                "a derived Optional finder cannot survive two rows");
+    }
+
+    @Test
+    @DisplayName("getOrCreateDefaultTeam converges on the lowest id instead of throwing")
+    void getOrCreateConvergesOnLowestId() {
+        Long first = saveTeam(TeamService.DEFAULT_TEAM_NAME);
+        saveTeam(TeamService.DEFAULT_TEAM_NAME);
+
+        TeamService teamService = new TeamService(teamRepository);
+
+        assertEquals(first, teamService.getOrCreateDefaultTeam().getId());
+        assertEquals(
+                first,
+                teamService.getOrCreateDefaultTeam().getId(),
+                "repeated calls must not drift between the duplicates");
+    }
+
+    @Test
+    @DisplayName("the internal team converges the same way")
+    void internalTeamConvergesOnLowestId() {
+        Long first = saveTeam(TeamService.INTERNAL_TEAM_NAME);
+        saveTeam(TeamService.INTERNAL_TEAM_NAME);
+
+        assertEquals(first, new TeamService(teamRepository).getOrCreateInternalTeam().getId());
+    }
+
+    @Test
+    @DisplayName("with no row at all it still creates one")
+    void createsWhenAbsent() {
+        Team created = new TeamService(teamRepository).getOrCreateDefaultTeam();
+
+        assertEquals(TeamService.DEFAULT_TEAM_NAME, created.getName());
+        assertEquals(
+                created.getId(),
+                teamRepository
+                        .findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME)
+                        .orElseThrow()
+                        .getId());
+    }
+
+    @SpringBootConfiguration
+    @EntityScan(
+            basePackages = {
+                "stirling.software.proprietary.model",
+                "stirling.software.proprietary.security.model"
+            })
+    @EnableJpaRepositories(basePackages = "stirling.software.proprietary.security.repository")
+    static class TestApp {}
+}

app/proprietary/src/test/java/stirling/software/proprietary/security/service/TeamServiceTest.java

@@ -27,7 +27,7 @@ class TeamServiceTest {
         var team = new Team();
         team.setName("Marleyans");
 
-        when(teamRepository.findByName(TeamService.DEFAULT_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.DEFAULT_TEAM_NAME))
                 .thenReturn(Optional.of(team));
 
         Team result = teamService.getOrCreateDefaultTeam();
@@ -42,7 +42,7 @@ class TeamServiceTest {
         defaultTeam.setId(1L);
         defaultTeam.setName(teamName);
 
-        when(teamRepository.findByName(teamName)).thenReturn(Optional.empty());
+        when(teamRepository.findFirstByNameOrderByIdAsc(teamName)).thenReturn(Optional.empty());
         when(teamRepository.save(any(Team.class))).thenReturn(defaultTeam);
 
         Team result = teamService.getOrCreateDefaultTeam();
@@ -55,7 +55,7 @@ class TeamServiceTest {
         var team = new Team();
         team.setName("Eldians");
 
-        when(teamRepository.findByName(TeamService.INTERNAL_TEAM_NAME))
+        when(teamRepository.findFirstByNameOrderByIdAsc(TeamService.INTERNAL_TEAM_NAME))
                 .thenReturn(Optional.of(team));
 
         Team result = teamService.getOrCreateInternalTeam();
@@ -70,10 +70,8 @@ class TeamServiceTest {
         internalTeam.setId(2L);
         internalTeam.setName(teamName);
 
-        when(teamRepository.findByName(teamName)).thenReturn(Optional.empty());
+        when(teamRepository.findFirstByNameOrderByIdAsc(teamName)).thenReturn(Optional.empty());
         when(teamRepository.save(any(Team.class))).thenReturn(internalTeam);
-        when(teamRepository.findByName(TeamService.INTERNAL_TEAM_NAME))
-                .thenReturn(Optional.empty());
 
         Team result = teamService.getOrCreateInternalTeam();
 

app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceMoreTest.java

@@ -258,7 +258,8 @@ class UserServiceMoreTest {
             User u = user("p");
             Team defaultTeam = new Team();
             defaultTeam.setName("Default");
-            when(teamRepository.findByName("Default")).thenReturn(Optional.of(defaultTeam));
+            when(teamRepository.findFirstByNameOrderByIdAsc("Default"))
+                    .thenReturn(Optional.of(defaultTeam));
 
             userService.changeUserTeam(u, null);
 

app/proprietary/src/test/java/stirling/software/proprietary/security/service/UserServiceTest.java

@@ -145,7 +145,8 @@ class UserServiceTest {
             throws SQLException, UnsupportedProviderException {
         Team defaultTeam = new Team();
         defaultTeam.setName("Default");
-        when(teamRepository.findByName("Default")).thenReturn(Optional.of(defaultTeam));
+        when(teamRepository.findFirstByNameOrderByIdAsc("Default"))
+                .thenReturn(Optional.of(defaultTeam));
         when(userRepository.save(any(User.class)))
                 .thenAnswer(invocation -> invocation.getArgument(0));
 
@@ -153,7 +154,7 @@ class UserServiceTest {
 
         User saved = userService.saveUserCore(request);
 
-        verify(teamRepository).findByName("Default");
+        verify(teamRepository).findFirstByNameOrderByIdAsc("Default");
         verify(teamRepository, never()).findById(anyLong());
         verify(databaseService).exportDatabase();
         assertEquals(defaultTeam, saved.getTeam(), "Default team should be applied");

testing/compose/docker-compose-multinode.betterdb.yml

@@ -0,0 +1,119 @@
+# BetterDB observability over the Valkey backplane plus an on-demand load generator. Dashboard: http://localhost:3001
+# Up: docker compose -f docker-compose-multinode.yml -f docker-compose-multinode.betterdb.yml up -d   (load: append --profile load run --rm loadgen)
+
+services:
+  # Same Valkey, retuned so the observability panels have data on a local box.
+  valkey:
+    command:
+      - valkey-server
+      - --save
+      - ""
+      - --appendonly
+      - "no"
+      # Default 10000us never trips locally, so the slowlog stays empty; 500us shows real command spread.
+      - --slowlog-log-slower-than
+      - "500"
+      - --slowlog-max-len
+      - "1024"
+      # 0 disables latency monitoring; 50ms populates LATENCY HISTORY/LATEST.
+      - --latency-monitor-threshold
+      - "50"
+      - --maxmemory-policy
+      - "noeviction"
+    ports:
+      # Loopback-only: this Valkey has no auth, so never expose it on all interfaces.
+      - "127.0.0.1:6379:6379" # host access for valkey-cli inspection
+
+  # One-shot: BetterDB runs as UID 1001 and cannot write a root-owned volume.
+  betterdb-init:
+    image: alpine:3.20
+    container_name: multinode-betterdb-init
+    command: ["sh", "-c", "chown -R 1001:1001 /data && echo 'betterdb data dir ready'"]
+    volumes:
+      - betterdb-data:/data
+    networks:
+      - stirling-multinode
+
+  # One-shot: its own database on the stack's Postgres, so BetterDB history survives restarts.
+  betterdb-db-init:
+    image: postgres:17-alpine
+    container_name: multinode-betterdb-db-init
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PGHOST: postgres
+      PGUSER: stirling
+      PGPASSWORD: stirling
+      PGDATABASE: stirling
+    command:
+      - sh
+      - -c
+      - psql -tAc "SELECT 1 FROM pg_database WHERE datname='betterdb'" | grep -q 1 || psql -c "CREATE DATABASE betterdb"
+    networks:
+      - stirling-multinode
+
+  betterdb:
+    # Digest-pinned: this third-party image holds the stack's Postgres credentials, so no floating :latest.
+    image: betterdb/monitor:0.39.0-no-ai@sha256:630cf435a129f285ca2d877ea1686d9788a7c2cb346cb6e38a3d227034d294f6
+    container_name: multinode-betterdb
+    restart: unless-stopped
+    depends_on:
+      valkey:
+        condition: service_healthy
+      betterdb-init:
+        condition: service_completed_successfully
+      betterdb-db-init:
+        condition: service_completed_successfully
+    environment:
+      DB_HOST: valkey
+      DB_PORT: "6379"
+      DB_TYPE: valkey
+      # postgres (not the default memory store) so history survives a restart; sqlite is not
+      # compiled into the published image despite the docs listing it.
+      STORAGE_TYPE: postgres
+      STORAGE_URL: "postgresql://stirling:stirling@postgres:5432/betterdb"
+      BETTERDB_DATA_DIR: /app/data
+      # Encrypts stored connection secrets at rest; test-only value.
+      ENCRYPTION_KEY: "multinode-betterdb-test-encryption-key"
+      ANOMALY_DETECTION_ENABLED: "true"
+      ANOMALY_POLL_INTERVAL_MS: "1000"
+      AUDIT_POLL_INTERVAL_MS: "15000"
+      CLIENT_ANALYTICS_POLL_INTERVAL_MS: "15000"
+      KEY_ANALYTICS_INTERVAL_MS: "60000"
+      # No phoning home from a local test stack.
+      BETTERDB_TELEMETRY: "false"
+    ports:
+      - "127.0.0.1:3001:3001"
+    volumes:
+      - betterdb-data:/app/data
+    # The image's own healthcheck probes "localhost", which resolves to ::1 while the server
+    # binds IPv4 only, so it always reports unhealthy. Pin it to 127.0.0.1.
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:3001/api/health >/dev/null || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 20s
+    networks:
+      - stirling-multinode
+
+  # Profile-gated so `up` skips it; run on demand to drive traffic through the LB.
+  loadgen:
+    image: python:3.12-alpine
+    container_name: multinode-loadgen
+    profiles: ["load"]
+    environment:
+      BASE_URL: "http://nginx:8080"
+      DURATION_SECONDS: "${DURATION_SECONDS:-300}"
+      CONCURRENCY: "${CONCURRENCY:-24}"
+      ASYNC_RATIO: "${ASYNC_RATIO:-0.35}"
+      USER_COUNT: "${USER_COUNT:-40}"
+    volumes:
+      - ./multinode/loadgen.py:/loadgen.py:ro
+    command: ["python3", "-u", "/loadgen.py"]
+    networks:
+      - stirling-multinode
+
+volumes:
+  betterdb-data:

testing/compose/docker-compose-multinode.valkey-cluster.yml

@@ -0,0 +1,133 @@
+# Must come LAST on the command line: it overrides valkey.command and compose REPLACES command.
+# Host tooling cannot follow MOVED on Docker Desktop - run valkey-cli from an in-network container.
+
+x-valkey-cluster-node: &valkey-cluster-node
+  image: valkey/valkey:8-alpine
+  restart: unless-stopped
+  command:
+    - valkey-server
+    - --save
+    - ""
+    - --appendonly
+    - "no"
+    # nodes.conf in the container layer, not a volume: a persisted membership file plus reshuffled
+    # container IPs wedges the cluster on recreate.
+    - --dir
+    - /tmp
+    - --cluster-enabled
+    - "yes"
+    - --cluster-config-file
+    - nodes.conf
+    - --cluster-node-timeout
+    - "5000"
+    # No --cluster-announce-ip: a service name is invalid and 127.0.0.1 breaks every in-network
+    # client with unroutable MOVED targets. Nodes must announce their own container IP.
+    - --slowlog-log-slower-than
+    - "500"
+    - --slowlog-max-len
+    - "1024"
+    - --latency-monitor-threshold
+    - "50"
+    - --maxmemory-policy
+    - "noeviction"
+  # Liveness only. A node answers PING long before it owns any slots, so this cannot gate the app -
+  # valkey-cluster-init does that.
+  healthcheck:
+    test: ["CMD-SHELL", "valkey-cli ping | grep -q PONG"]
+    interval: 3s
+    timeout: 5s
+    retries: 30
+  networks:
+    - stirling-multinode
+
+services:
+  # Node 1 keeps the base container_name (multinode-valkey) and the 'valkey' DNS name so existing
+  # docker exec / `valkey-cli -h valkey` steps still resolve. Note they now see ONE SHARD only.
+  valkey:
+    <<: *valkey-cluster-node
+
+  valkey-2:
+    <<: *valkey-cluster-node
+    container_name: multinode-valkey-2
+  valkey-3:
+    <<: *valkey-cluster-node
+    container_name: multinode-valkey-3
+  valkey-4:
+    <<: *valkey-cluster-node
+    container_name: multinode-valkey-4
+  valkey-5:
+    <<: *valkey-cluster-node
+    container_name: multinode-valkey-5
+  valkey-6:
+    <<: *valkey-cluster-node
+    container_name: multinode-valkey-6
+
+  # Creates against resolved IPs: `--cluster create` bakes whatever it is given into the membership
+  # table, and IPs are what the nodes gossip anyway. Idempotent, so repeated `up` is safe.
+  valkey-cluster-init:
+    image: valkey/valkey:8-alpine
+    container_name: multinode-valkey-cluster-init
+    restart: "no"
+    depends_on:
+      valkey: {condition: service_healthy}
+      valkey-2: {condition: service_healthy}
+      valkey-3: {condition: service_healthy}
+      valkey-4: {condition: service_healthy}
+      valkey-5: {condition: service_healthy}
+      valkey-6: {condition: service_healthy}
+    entrypoint:
+      - /bin/sh
+      - -c
+      - |
+        set -e
+        NODES="valkey valkey-2 valkey-3 valkey-4 valkey-5 valkey-6"
+        # cluster_state flips to ok before every slot is served, so gate on the slot count too.
+        formed() { valkey-cli -h "$$1" cluster info 2>/dev/null | tr -d '\r' | grep -q '^cluster_state:ok' \
+                   && valkey-cli -h "$$1" cluster info 2>/dev/null | tr -d '\r' | grep -q '^cluster_slots_ok:16384'; }
+        if formed valkey; then
+          echo "cluster already formed"; valkey-cli -h valkey cluster info | head -3; exit 0
+        fi
+        ADDRS=""
+        for n in $$NODES; do
+          IP=$$(getent hosts $$n | awk '{print $$1; exit}')
+          [ -n "$$IP" ] || { echo "FATAL: cannot resolve $$n"; exit 1; }
+          echo "$$n -> $$IP"
+          ADDRS="$$ADDRS $$IP:6379"
+        done
+        echo "creating cluster on:$$ADDRS"
+        valkey-cli --cluster create $$ADDRS --cluster-replicas 1 --cluster-yes
+        # `--cluster create` returns before every node has the full slot map. Gate on all six
+        # agreeing, otherwise the first app command can hit CLUSTERDOWN.
+        for i in $$(seq 1 30); do
+          ok=1
+          for n in $$NODES; do
+            formed $$n || ok=0
+          done
+          [ "$$ok" = "1" ] && { echo "cluster_state:ok + 16384/16384 slots on all 6 nodes"; exit 0; }
+          echo "waiting for slot propagation..."; sleep 2
+        done
+        echo "FATAL: cluster did not reach cluster_state:ok with all 16384 slots served"; exit 1
+    networks:
+      - stirling-multinode
+
+  # service_completed_successfully on the init is the real readiness gate; valkey's own healthcheck
+  # only proves the process is alive.
+  stirling-1:
+    depends_on:
+      valkey-cluster-init:
+        condition: service_completed_successfully
+    environment:
+      CLUSTER_VALKEY_MODE: "cluster"
+      CLUSTER_VALKEY_NODES: "valkey:6379,valkey-2:6379,valkey-3:6379,valkey-4:6379,valkey-5:6379,valkey-6:6379"
+      CLUSTER_VALKEY_MAXREDIRECTS: "3"
+      CLUSTER_VALKEY_TOPOLOGYREFRESHMS: "30000"
+
+  stirling-2:
+    depends_on:
+      valkey-cluster-init:
+        condition: service_completed_successfully
+    environment:
+      CLUSTER_VALKEY_MODE: "cluster"
+      CLUSTER_VALKEY_NODES: "valkey:6379,valkey-2:6379,valkey-3:6379,valkey-4:6379,valkey-5:6379,valkey-6:6379"
+      CLUSTER_VALKEY_MAXREDIRECTS: "3"
+      CLUSTER_VALKEY_TOPOLOGYREFRESHMS: "30000"

testing/compose/docker-compose-multinode.valkey-sentinel.yml

@@ -0,0 +1,115 @@
+# Valkey Sentinel HA overlay: 1 primary + 2 replicas + 3 sentinels. Up: ./start-multinode-test.sh --valkey sentinel
+# Never overrides valkey.command, so the base 'valkey' service survives and this composes with any other overlay in any order.
+
+x-sentinel-deps: &sentinel-deps
+  sentinel-1:
+    condition: service_healthy
+  sentinel-2:
+    condition: service_healthy
+  sentinel-3:
+    condition: service_healthy
+
+services:
+  # No --replica-announce-ip: a hostname makes sentinel discard the replica, and
+  # 'resolve-hostnames yes' is not a workaround (see the TILT note on the sentinel below).
+  valkey-replica-1: &valkey-replica
+    image: valkey/valkey:8-alpine
+    container_name: multinode-valkey-replica-1
+    restart: unless-stopped
+    depends_on:
+      valkey:
+        condition: service_healthy
+    command:
+      - valkey-server
+      - --save
+      - ""
+      - --appendonly
+      - "no"
+      # Runtime files in the container layer, not the image's anonymous /data volume, so a
+      # `compose up` recreate always starts from clean state.
+      - --dir
+      - /tmp
+      - --replicaof
+      - valkey
+      - "6379"
+      - --replica-read-only
+      - "yes"
+    healthcheck:
+      test: ["CMD-SHELL", "valkey-cli ping | grep -q PONG"]
+      interval: 3s
+      timeout: 5s
+      retries: 30
+    networks:
+      - stirling-multinode
+
+  valkey-replica-2:
+    <<: *valkey-replica
+    container_name: multinode-valkey-replica-2
+
+  # No host-mounted conf: sentinel rewrites it at runtime, so a read-only mount kills boot and a
+  # read-write one replays stale known-replica IPs. The entrypoint generates it into /tmp instead.
+  sentinel-1: &valkey-sentinel
+    image: valkey/valkey:8-alpine
+    container_name: multinode-valkey-sentinel-1
+    restart: unless-stopped
+    depends_on:
+      valkey:
+        condition: service_healthy
+    # Monitor a RESOLVED IP, never the name 'valkey': killing the primary drops its DNS record,
+    # sentinel's synchronous lookup stalls its event loop, and TILT suspends failover.
+    entrypoint:
+      - /bin/sh
+      - -c
+      - |
+        set -e
+        for i in $$(seq 1 30); do
+          PRIMARY_IP=$$(getent hosts valkey | awk '{print $$1; exit}')
+          [ -n "$$PRIMARY_IP" ] && break
+          echo "waiting for DNS: valkey"; sleep 2
+        done
+        [ -n "$$PRIMARY_IP" ] || { echo "FATAL: could not resolve 'valkey'"; exit 1; }
+        echo "sentinel monitoring primary valkey -> $$PRIMARY_IP"
+        cat > /tmp/sentinel.conf <<EOF
+        port 26379
+        dir /tmp
+        sentinel monitor mymaster $$PRIMARY_IP 6379 2
+        sentinel down-after-milliseconds mymaster 5000
+        sentinel failover-timeout mymaster 10000
+        sentinel parallel-syncs mymaster 1
+        EOF
+        exec valkey-sentinel /tmp/sentinel.conf
+    # ckquorum alone is NOT enough: it reports "OK 3 usable Sentinels" even with ZERO discovered
+    # replicas, i.e. no failover candidate. Assert both.
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - valkey-cli -p 26379 sentinel ckquorum mymaster | grep -q '^OK' && [ "$$(valkey-cli -p 26379 sentinel replicas mymaster | grep -c '^name$$')" -ge 2 ]
+      interval: 3s
+      timeout: 5s
+      retries: 40
+    networks:
+      - stirling-multinode
+
+  sentinel-2:
+    <<: *valkey-sentinel
+    container_name: multinode-valkey-sentinel-2
+
+  sentinel-3:
+    <<: *valkey-sentinel
+    container_name: multinode-valkey-sentinel-3
+
+  # ---- App nodes: point the backplane at the sentinel set ------------------
+  # Only the keys that differ from the base file - compose merges `environment` across -f files.
+  stirling-1:
+    depends_on: *sentinel-deps
+    environment:
+      CLUSTER_VALKEY_MODE: "sentinel"
+      CLUSTER_VALKEY_SENTINEL_MASTER: "mymaster"
+      CLUSTER_VALKEY_SENTINEL_NODES: "sentinel-1:26379,sentinel-2:26379,sentinel-3:26379"
+
+  stirling-2:
+    depends_on: *sentinel-deps
+    environment:
+      CLUSTER_VALKEY_MODE: "sentinel"
+      CLUSTER_VALKEY_SENTINEL_MASTER: "mymaster"
+      CLUSTER_VALKEY_SENTINEL_NODES: "sentinel-1:26379,sentinel-2:26379,sentinel-3:26379"

testing/compose/docker-compose-multinode.yml

@@ -1,5 +1,5 @@
-# Multi-node Stirling-PDF processor test stack: shared Postgres/MinIO/Valkey behind an nginx LB fronting N app nodes, with enterprise features unlocked via a local test licence key.
-# Bring up: ./start-multinode-test.sh   Validate: ./validate-multinode-test.sh   Access: http://localhost:8080 (admin / stirling)
+# Multi-node test stack. Up: ./start-multinode-test.sh [--valkey sentinel|cluster] - default is
+# standalone. Every topology keeps container 'multinode-valkey', so docker-exec steps still work.
 
 x-stirling-node: &stirling-node
   build:
@@ -67,12 +67,25 @@ x-stirling-node: &stirling-node
     CLUSTER_BACKPLANE: "valkey"
     CLUSTER_ARTIFACTSTORE: "s3"
     CLUSTER_VALKEY_URL: "redis://valkey:6379"
+    # Named connections so CLIENT LIST attributes load per node in any Valkey monitor; overridden per node below.
+    CLUSTER_VALKEY_CLIENTNAME: "stirling-node-base"
+    # Pinned to the shipped settings.yml.template defaults so the test stack exercises what users run.
+    CLUSTER_VALKEY_POOL_ENABLED: "true"
+    CLUSTER_VALKEY_POOL_MAXACTIVE: "16"
+    CLUSTER_VALKEY_POOL_MAXIDLE: "16"
+    CLUSTER_VALKEY_POOL_MINIDLE: "0"
+    CLUSTER_VALKEY_POOL_MAXWAITMILLIS: "2000"
     # SPRING_DATA_REDIS_REPOSITORIES_ENABLED is not needed: DataRedisRepositoriesAutoConfiguration is excluded (see application.properties, multinode/README.md).
 
     # --- Shared credential-encryption key (REQUIRED in cluster mode) ---
     # AES-256 key that encrypts stored integration/S3 secrets - must match on every node or secrets encrypted on one can't decrypt on another; boot fails if unset with cluster.enabled=true (test-only value; JWT keys persist separately in the shared DB).
     STIRLING_CREDENTIAL_ENCRYPTION_KEY: "dMobekyUEnEV7WHBah2FkbboP4Coqifd3JRXB00LiIY="
 
+    # --- Shared AutomaticallyGenerated key/UUID (REQUIRED in cluster mode) ---
+    # Left unset, each node mints its own UUID into its local settings.yml: workflow metadata encrypted on one node then fails to decrypt on another and licence-seat HMACs do not verify across nodes. Must be valid UUIDs (anything else is replaced at startup); boot fails if unset with cluster.enabled=true (test-only values).
+    AUTOMATICALLYGENERATED_KEY: "11111111-1111-1111-1111-111111111111"
+    AUTOMATICALLYGENERATED_UUID: "22222222-2222-2222-2222-222222222222"
+
     # --- Policy / processor subsystem (the thing under test) ---
     POLICIES_ENABLED: "true"
     # Let policy S3 sources/webhook-staging connections point at the in-cluster MinIO.
@@ -166,6 +179,9 @@ services:
     environment:
       <<: *stirling-env
       SYSTEM_NODEID: "node-1"
+      # Pinned instead of the per-boot UUID default so tests can assert on registered node ids.
+      CLUSTER_NODE_ID: "multinode-1"
+      CLUSTER_VALKEY_CLIENTNAME: "stirling-node-1"
 
   # ---- App node 2 -----------------------------------------------------------
   # No ordering vs node-1 needed - each node mints its own signing key at boot and publishes the public half to the shared DB, so both verify each other's tokens.
@@ -175,6 +191,9 @@ services:
     environment:
       <<: *stirling-env
       SYSTEM_NODEID: "node-2"
+      # Pinned instead of the per-boot UUID default so tests can assert on registered node ids.
+      CLUSTER_NODE_ID: "multinode-2"
+      CLUSTER_VALKEY_CLIENTNAME: "stirling-node-2"
 
   # ---- One-shot seeder: teams, ~40 users, S3 connection, policies -----------
   # Profile-gated so plain `docker compose up` skips it; the start script runs it once, using postgres:alpine for psql with curl+jq added for the HTTP calls.

testing/compose/multinode/loadgen.py

@@ -0,0 +1,460 @@
+#!/usr/bin/env python3
+"""Load generator for the multi-node stack: real PDF work through the nginx LB as many distinct users,
+mixed sync/async so the Valkey backplane sees job, lock, rate-limit and cache traffic. Stdlib only."""
+
+import json
+import os
+import random
+import sys
+import threading
+import time
+import traceback
+import urllib.error
+import urllib.request
+import uuid
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+BASE_URL = os.environ.get("BASE_URL", "http://nginx:8080").rstrip("/")
+DURATION = int(os.environ.get("DURATION_SECONDS", "300"))
+CONCURRENCY = int(os.environ.get("CONCURRENCY", "24"))
+USER_COUNT = int(os.environ.get("USER_COUNT", "40"))
+USER_PASSWORD = os.environ.get("USER_PASSWORD", "Password123!")
+ADMIN_USER = os.environ.get("ADMIN_USER", "admin")
+ADMIN_PASS = os.environ.get("ADMIN_PASS", "stirling")
+ASYNC_RATIO = float(os.environ.get("ASYNC_RATIO", "0.35"))
+RAMP_SECONDS = int(os.environ.get("RAMP_SECONDS", "20"))
+REQUEST_TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT_SECONDS", "120"))
+# Above this share of failed requests the run exits non-zero, so CI cannot green on a broken stack.
+MAX_ERROR_RATE = float(os.environ.get("MAX_ERROR_RATE", "0.05"))
+# Grace for in-flight async polls after the clock runs out; bounds how far a run can overrun.
+POLL_GRACE_SECONDS = int(os.environ.get("POLL_GRACE_SECONDS", "30"))
+
+# Cheap plumbing calls: excluded from the PDF-operation rate so the headline number is comparable.
+NON_PDF_OPS = {"info/status", "job-poll"}
+
+run_started = 0.0
+stop_at = 0.0
+stats_lock = threading.Lock()
+stats = defaultdict(lambda: {"ok": 0, "err": 0, "lat": [], "bytes": 0})
+nodes_seen = defaultdict(int)
+errors = defaultdict(int)
+job_stats = {"submitted": 0, "completed": 0, "failed": 0, "sticky_410": 0}
+
+
+# ---------------------------------------------------------------- PDF corpus
+def make_pdf(pages: int, filler_lines: int) -> bytes:
+    """Build a valid multi-page PDF from raw syntax so the corpus needs no PDF library."""
+    objs = []  # obj number -> body bytes, 1-indexed by position
+
+    font_obj = 3 + pages * 2  # catalog=1, pages=2, then page/content pairs
+    kids = " ".join(f"{3 + i * 2} 0 R" for i in range(pages))
+    objs.append(b"<</Type/Catalog/Pages 2 0 R>>")
+    objs.append(f"<</Type/Pages/Kids[{kids}]/Count {pages}>>".encode())
+
+    for i in range(pages):
+        content_num = 4 + i * 2
+        objs.append(
+            f"<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]"
+            f"/Contents {content_num} 0 R"
+            f"/Resources<</Font<</F1 {font_obj} 0 R>>>>>>".encode()
+        )
+        lines = [b"BT /F1 14 Tf 54 740 Td (Stirling load-test page " + str(i + 1).encode() + b") Tj ET"]
+        for n in range(filler_lines):
+            y = 710 - (n * 18) % 640
+            text = f"lorem ipsum dolor sit amet {uuid.uuid4().hex}"
+            lines.append(f"BT /F1 9 Tf 54 {y} Td ({text}) Tj ET".encode())
+        stream = b"\n".join(lines)
+        objs.append(b"<</Length " + str(len(stream)).encode() + b">>stream\n" + stream + b"\nendstream")
+
+    objs.append(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>")
+
+    out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
+    offsets = []
+    for idx, body in enumerate(objs, start=1):
+        offsets.append(len(out))
+        # Newlines around the body keep 'endstream' and 'endobj' from fusing into one token.
+        out += f"{idx} 0 obj\n".encode() + body + b"\nendobj\n"
+
+    xref_at = len(out)
+    out += f"xref\n0 {len(objs) + 1}\n".encode()
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode()
+    out += f"trailer<</Size {len(objs) + 1}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n".encode()
+    return bytes(out)
+
+
+def build_corpus():
+    print("==> Building PDF corpus...", flush=True)
+    corpus = {
+        "tiny": make_pdf(2, 6),
+        "small": make_pdf(8, 14),
+        "medium": make_pdf(30, 20),
+        "large": make_pdf(80, 26),
+    }
+    for name, data in corpus.items():
+        print(f"    {name:7s} {len(data) / 1024:8.1f} KB", flush=True)
+    return corpus
+
+
+# ---------------------------------------------------------------- HTTP plumbing
+def encode_multipart(fields, files):
+    """fields: dict of scalars. files: list of (fieldname, filename, bytes) - repeats allowed."""
+    boundary = uuid.uuid4().hex
+    body = bytearray()
+    for key, value in fields.items():
+        body += f"--{boundary}\r\n".encode()
+        body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()
+        body += f"{value}\r\n".encode()
+    for key, filename, data in files:
+        body += f"--{boundary}\r\n".encode()
+        body += (
+            f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'
+            f"Content-Type: application/pdf\r\n\r\n"
+        ).encode()
+        body += data + b"\r\n"
+    body += f"--{boundary}--\r\n".encode()
+    return bytes(body), f"multipart/form-data; boundary={boundary}"
+
+
+def request(method, path, token=None, body=None, content_type=None, timeout=REQUEST_TIMEOUT):
+    url = path if path.startswith("http") else BASE_URL + path
+    req = urllib.request.Request(url, data=body, method=method)
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    if content_type:
+        req.add_header("Content-Type", content_type)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status, resp.read(), dict(resp.headers)
+    except urllib.error.HTTPError as exc:
+        return exc.code, exc.read(), dict(exc.headers)
+    except Exception as exc:  # connection reset, timeout, DNS
+        return 0, str(exc).encode(), {}
+
+
+def budget(default):
+    """Clamp a timeout to what is left of the run so one hung call cannot outlive DURATION."""
+    return max(1.0, min(default, (stop_at + POLL_GRACE_SECONDS) - time.time()))
+
+
+def record(op, status, elapsed_ms, size, headers, ok=None):
+    # ok overrides the 2xx check for expected non-2xx replies (async 410 re-routes).
+    if ok is None:
+        ok = 200 <= status < 300
+    with stats_lock:
+        entry = stats[op]
+        if ok:
+            entry["ok"] += 1
+            entry["bytes"] += size
+        else:
+            entry["err"] += 1
+            errors[f"{op} -> {status}"] += 1
+        entry["lat"].append(elapsed_ms)
+        served = headers.get("X-Served-By")
+        if served:
+            nodes_seen[served] += 1
+
+
+def percentile(values, pct):
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    idx = min(len(ordered) - 1, int(round(pct / 100.0 * (len(ordered) - 1))))
+    return ordered[idx]
+
+
+# ---------------------------------------------------------------- auth
+def login(username, password):
+    body = json.dumps({"username": username, "password": password}).encode()
+    status, data, _ = request("POST", "/api/v1/auth/login", body=body,
+                              content_type="application/json", timeout=30)
+    if status != 200:
+        return None
+    try:
+        return json.loads(data).get("session", {}).get("access_token") or json.loads(data).get("access_token")
+    except Exception:
+        return None
+
+
+def collect_tokens():
+    print("==> Logging in...", flush=True)
+    tokens = []
+    admin = login(ADMIN_USER, ADMIN_PASS)
+    if admin:
+        tokens.append(("admin", admin))
+    for i in range(1, USER_COUNT + 1):
+        user = f"user{i:02d}@stirling.test"
+        tok = login(user, USER_PASSWORD)
+        if tok:
+            tokens.append((user, tok))
+    print(f"    {len(tokens)} authenticated principals", flush=True)
+    return tokens
+
+
+# ---------------------------------------------------------------- workload
+def op_rotate(corpus):
+    return "/api/v1/general/rotate-pdf", {"angle": random.choice([90, 180, 270])}, [
+        ("fileInput", "load.pdf", corpus[random.choice(["tiny", "small", "medium"])])
+    ]
+
+
+def op_merge(corpus):
+    files = [("fileInput", f"m{i}.pdf", corpus[random.choice(["tiny", "small"])]) for i in range(random.randint(2, 4))]
+    return "/api/v1/general/merge-pdfs", {"sortType": "orderProvided"}, files
+
+
+def op_compress(corpus):
+    return "/api/v1/misc/compress-pdf", {"optimizeLevel": random.choice([1, 2, 3])}, [
+        ("fileInput", "compress.pdf", corpus[random.choice(["medium", "large"])])
+    ]
+
+
+def op_remove_pages(corpus):
+    return "/api/v1/general/remove-pages", {"pageNumbers": "1,3"}, [
+        ("fileInput", "rm.pdf", corpus[random.choice(["small", "medium"])])
+    ]
+
+
+def op_split(corpus):
+    return "/api/v1/general/split-pages", {"pageNumbers": "2,4"}, [
+        ("fileInput", "split.pdf", corpus[random.choice(["small", "medium"])])
+    ]
+
+
+def op_page_numbers(corpus):
+    return "/api/v1/misc/add-page-numbers", {
+        "customMargin": "medium",
+        "position": 8,
+        "startingNumber": 1,
+        "pagesToNumber": "all",
+        "customText": "{n} of {total}",
+    }, [("fileInput", "num.pdf", corpus[random.choice(["small", "medium"])])]
+
+
+def op_flatten(corpus):
+    return "/api/v1/misc/flatten", {"flattenOnlyForms": "false"}, [
+        ("fileInput", "flat.pdf", corpus[random.choice(["tiny", "small"])])
+    ]
+
+
+def op_metadata(corpus):
+    return "/api/v1/misc/update-metadata", {
+        "deleteAll": "false",
+        "author": "load-test",
+        "title": f"run-{uuid.uuid4().hex[:8]}",
+    }, [("fileInput", "meta.pdf", corpus[random.choice(["tiny", "small"])])]
+
+
+# (weight, name, builder) - heavier tools are rarer so throughput stays high.
+WORKLOAD = [
+    (22, "rotate", op_rotate),
+    (14, "merge", op_merge),
+    (10, "compress", op_compress),
+    (14, "remove-pages", op_remove_pages),
+    (12, "split-pages", op_split),
+    (12, "add-page-numbers", op_page_numbers),
+    (8, "flatten", op_flatten),
+    (8, "update-metadata", op_metadata),
+]
+WEIGHTS = [w for w, _, _ in WORKLOAD]
+
+
+def poll_job(job_id, token, deadline):
+    """Poll until the job completes. A 410 means we hit a non-owner node; retry re-routes us."""
+    while time.time() < deadline:
+        started = time.time()
+        status, data, headers = request("GET", f"/api/v1/general/job/{job_id}", token=token,
+                                        timeout=budget(30))
+        # A 410 is an expected cross-node re-route, counted in job_stats rather than as an error.
+        record("job-poll", status, (time.time() - started) * 1000, len(data), headers,
+               ok=(status == 410 or 200 <= status < 300))
+        if status == 410:
+            with stats_lock:
+                job_stats["sticky_410"] += 1
+            time.sleep(0.3)
+            continue
+        if status != 200:
+            return False
+        try:
+            payload = json.loads(data)
+        except Exception:
+            return False
+        result = payload.get("jobResult", payload)
+        if result.get("complete"):
+            return result.get("error") is None
+        time.sleep(0.4)
+    return False
+
+
+def worker(worker_id, corpus, tokens):
+    rng = random.Random(worker_id * 7919)
+    # Stagger startup so all workers do not slam the LB in the same instant.
+    time.sleep(max(0.0, min(rng.uniform(0, RAMP_SECONDS), stop_at - time.time())))
+    while time.time() < stop_at:
+        _, token = rng.choice(tokens)
+        _, name, builder = rng.choices(WORKLOAD, weights=WEIGHTS, k=1)[0]
+        path, fields, files = builder(corpus)
+        use_async = rng.random() < ASYNC_RATIO
+        if use_async:
+            path += "?async=true"
+        body, content_type = encode_multipart(fields, files)
+
+        started = time.time()
+        status, data, headers = request("POST", path, token=token, body=body,
+                                        content_type=content_type, timeout=budget(REQUEST_TIMEOUT))
+        elapsed = (time.time() - started) * 1000
+        label = f"{name}{' (async)' if use_async else ''}"
+        record(label, status, elapsed, len(data), headers)
+
+        if use_async and 200 <= status < 300:
+            try:
+                job_id = json.loads(data).get("jobId")
+            except Exception:
+                job_id = None
+            if job_id:
+                with stats_lock:
+                    job_stats["submitted"] += 1
+                deadline = min(time.time() + 240, stop_at + POLL_GRACE_SECONDS)
+                if poll_job(job_id, token, deadline):
+                    with stats_lock:
+                        job_stats["completed"] += 1
+                else:
+                    with stats_lock:
+                        job_stats["failed"] += 1
+
+        # Cheap reads between jobs: extra request volume and node-registry reads.
+        if time.time() < stop_at and rng.random() < 0.3:
+            started = time.time()
+            s, d, h = request("GET", "/api/v1/info/status", token=token, timeout=budget(20))
+            record("info/status", s, (time.time() - started) * 1000, len(d), h)
+
+
+def progress_printer():
+    last_ops = 0
+    while time.time() < stop_at:
+        time.sleep(15)
+        with stats_lock:
+            total = sum(v["ok"] + v["err"] for v in stats.values())
+            ok = sum(v["ok"] for v in stats.values())
+            ops = sum(v["ok"] + v["err"] for op, v in stats.items() if op not in NON_PDF_OPS)
+            mb = sum(v["bytes"] for v in stats.values()) / 1024 / 1024
+            jobs = dict(job_stats)
+        remaining = max(0, int(stop_at - time.time()))
+        rate = (ops - last_ops) / 15.0
+        last_ops = ops
+        print(
+            f"    [{remaining:4d}s left] {total:6d} reqs  {ok:6d} ok  {rate:5.1f} pdf-op/s  "
+            f"{mb:7.1f} MB down  async {jobs['completed']}/{jobs['submitted']}",
+            flush=True,
+        )
+
+
+def report():
+    """Print per-endpoint counts and latency percentiles. Returns (ok, err) totals."""
+    elapsed = max(0.001, time.time() - run_started)
+    with stats_lock:
+        snapshot = {op: dict(entry, lat=list(entry["lat"])) for op, entry in stats.items()}
+    print("\n" + "=" * 84)
+    print(" LOAD TEST SUMMARY")
+    print("=" * 84)
+    print(f"{'operation':22s} {'ok':>7s} {'err':>6s} {'p50':>7s} {'p95':>7s} {'p99':>7s} "
+          f"{'req/s':>7s} {'MB':>7s}")
+    print("-" * 84)
+    total_ok = total_err = 0
+    for op in sorted(snapshot):
+        entry = snapshot[op]
+        lat = entry["lat"]
+        calls = entry["ok"] + entry["err"]
+        total_ok += entry["ok"]
+        total_err += entry["err"]
+        print(
+            f"{op:22s} {entry['ok']:7d} {entry['err']:6d} {percentile(lat, 50):7.0f} "
+            f"{percentile(lat, 95):7.0f} {percentile(lat, 99):7.0f} {calls / elapsed:7.2f} "
+            f"{entry['bytes'] / 1024 / 1024:7.1f}"
+        )
+    print("-" * 84)
+    total = total_ok + total_err
+    print(f"{'TOTAL':22s} {total_ok:7d} {total_err:6d} {'':7s} {'':7s} {'':7s} "
+          f"{total / elapsed:7.2f}")
+    pdf_calls = sum(e["ok"] + e["err"] for op, e in snapshot.items() if op not in NON_PDF_OPS)
+    print(f"\n PDF operations only: {pdf_calls} calls, {pdf_calls / elapsed:.2f} op/s over "
+          f"{elapsed:.0f}s (health probes and job polls excluded).")
+
+    print("\n Load-balancer spread (X-Served-By):")
+    for node, count in sorted(nodes_seen.items(), key=lambda kv: -kv[1]):
+        print(f"   {node:24s} {count:7d} responses")
+
+    print("\n Async jobs (these are the Valkey JobStore writes):")
+    print(f"   submitted {job_stats['submitted']}   completed {job_stats['completed']}"
+          f"   failed {job_stats['failed']}   cross-node 410 re-routes {job_stats['sticky_410']}")
+
+    if errors:
+        print("\n Top errors:")
+        for key, count in sorted(errors.items(), key=lambda kv: -kv[1])[:15]:
+            print(f"   {count:6d}  {key}")
+    print("=" * 84, flush=True)
+    return total_ok, total_err
+
+
+def wait_for_app():
+    print(f"==> Waiting for {BASE_URL} ...", flush=True)
+    for _ in range(120):
+        status, _, _ = request("GET", "/api/v1/info/status", timeout=10)
+        if status == 200:
+            print("    app is up", flush=True)
+            return True
+        time.sleep(2)
+    return False
+
+
+def main():
+    global stop_at, run_started
+    if not wait_for_app():
+        print("app never came up", file=sys.stderr)
+        return 1
+
+    corpus = build_corpus()
+    tokens = collect_tokens()
+    if not tokens:
+        print("no logins succeeded - cannot generate authenticated load", file=sys.stderr)
+        return 1
+
+    run_started = time.time()
+    stop_at = run_started + DURATION
+    print(
+        f"\n==> Driving load for {DURATION}s: {CONCURRENCY} workers, "
+        f"{len(tokens)} users, {int(ASYNC_RATIO * 100)}% async\n",
+        flush=True,
+    )
+    ticker = threading.Thread(target=progress_printer, daemon=True)
+    ticker.start()
+    crashed = 0
+    with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
+        futures = [pool.submit(worker, i, corpus, tokens) for i in range(CONCURRENCY)]
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception:
+                crashed += 1
+                traceback.print_exc()
+    total_ok, total_err = report()
+
+    total = total_ok + total_err
+    if crashed:
+        print(f"\n{crashed} worker(s) crashed - see the tracebacks above", file=sys.stderr)
+        return 1
+    if not total:
+        print("\nno requests were issued", file=sys.stderr)
+        return 1
+    err_rate = total_err / total
+    if err_rate > MAX_ERROR_RATE:
+        print(f"\nerror rate {err_rate:.1%} exceeds MAX_ERROR_RATE {MAX_ERROR_RATE:.1%}",
+              file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

testing/compose/multinode/seed.sh

@@ -1,7 +1,6 @@
 #!/bin/sh
-# Seeds a running multi-node stack: 4 teams, ~40 users, an S3 connection, a scheduled S3 policy, and a webhook source if the build supports it.
-# Auth uses the Bearer JWT from the login response body (not a cookie) since the global API key can't create teams.
-# Idempotent-ish: re-running skips existing teams/users; each step is best-effort and logs failures without aborting.
+# Seeds a running stack (teams, users, S3 connection + policy). Idempotent; exits non-zero if a create fails.
+# Auth uses the Bearer JWT from the login body, not a cookie - the global API key can't create teams.
 set -u
 
 BASE_URL="${BASE_URL:-http://localhost:8080}"
@@ -41,11 +40,23 @@ TOKEN=$(jq -r '.session.access_token' </tmp/login.json)
 
 auth() { curl -sS -H "Authorization: Bearer $TOKEN" "$@"; }
 
+errors=0
+fail_note() { errors=$((errors+1)); log "ERROR: $*"; }
+
+# The regression runner re-seeds an already-seeded stack, so every create is check-then-create by name.
+find_id_by_name() { # $1=list path  $2=name -> the existing id, or empty
+  auth "$BASE_URL$1" 2>/dev/null \
+    | jq -r --arg n "$2" '[.. | objects | select(.name? == $n) | .id?] | map(select(. != null)) | first // empty' 2>/dev/null
+}
+
 # --- teams -------------------------------------------------------------------
 for t in $TEAMS; do
   code=$(auth -o /dev/null -w '%{http_code}' -X POST "$BASE_URL/api/v1/team/create" \
     --data-urlencode "name=$t")
-  log "team '$t': HTTP $code"
+  case "$code" in
+    2*|409) log "team '$t': HTTP $code" ;;
+    *)      fail_note "team '$t': HTTP $code" ;;
+  esac
 done
 
 # Resolve team ids from the DB (no admin list endpoint self-hosted).
@@ -59,7 +70,9 @@ team_count=$#
 log "seedable team ids:$seed_team_ids (count=$team_count)"
 
 # --- users: spread across teams, first two are admins ------------------------
-created=0; failed=0
+# The licence caps seats, so USER_COUNT is a target, not a promise: hitting the cap stops the
+# loop and is reported, not counted as a failure. Every other rejection stays fatal.
+created=0; failed=0; capped=""
 n=1
 while [ "$n" -le "$USER_COUNT" ]; do
   uname=$(printf "user%02d@stirling.test" "$n")
@@ -75,14 +88,29 @@ while [ "$n" -le "$USER_COUNT" ]; do
     ${team_id:+--data-urlencode "teamId=$team_id"} \
     --data-urlencode "authType=WEB" \
     --data-urlencode "forceChange=false")
+  body=$(cat /tmp/user.json)
   case "$code" in
     200|201) created=$((created+1));;
     409)     log "user $uname already exists";;
-    *)       failed=$((failed+1)); [ "$failed" -le 3 ] && log "user $uname failed HTTP $code: $(cat /tmp/user.json)";;
+    *)
+      case "$body" in
+        *"Maximum number of users reached"*|*"Available slots"*)
+          capped="$body"
+          break
+          ;;
+      esac
+      failed=$((failed+1))
+      [ "$failed" -le 3 ] && log "user $uname failed HTTP $code: $body"
+      ;;
   esac
   n=$((n+1))
 done
+if [ -n "$capped" ]; then
+  log "user seats exhausted after $created create(s); licence caps this stack below USER_COUNT=$USER_COUNT"
+  log "  server said: $capped"
+fi
 log "users created: $created (failed: $failed, requested: $USER_COUNT)"
+[ "$failed" -eq 0 ] || fail_note "$failed of $USER_COUNT user creates failed"
 
 # --- S3 connection -> the in-cluster MinIO 'policy-data' bucket ---------------
 conn_body=$(cat <<JSON
@@ -90,9 +118,15 @@ conn_body=$(cat <<JSON
  "config":{"bucket":"policy-data","region":"us-east-1","endpoint":"http://minio:9000","accessKeyId":"minioadmin","secretAccessKey":"minioadmin","pathStyleAccess":true}}
 JSON
 )
-conn_id=$(auth -X POST "$BASE_URL/api/v1/integrations" -H 'Content-Type: application/json' -d "$conn_body" \
-  | jq -r '.id // empty' 2>/dev/null)
-log "S3 connection id: ${conn_id:-<none>}"
+conn_id=$(find_id_by_name "/api/v1/integrations" "MinIO policy bucket")
+if [ -n "${conn_id:-}" ]; then
+  log "S3 connection already exists: id $conn_id"
+else
+  conn_id=$(auth -X POST "$BASE_URL/api/v1/integrations" -H 'Content-Type: application/json' -d "$conn_body" \
+    | jq -r '.id // empty' 2>/dev/null)
+  log "S3 connection id: ${conn_id:-<none>}"
+  [ -n "${conn_id:-}" ] || fail_note "S3 connection was not created"
+fi
 
 # --- a scheduled S3 -> compress -> S3 policy ---------------------------------
 if [ -n "${conn_id:-}" ]; then
@@ -101,22 +135,39 @@ if [ -n "${conn_id:-}" ]; then
  "options":{"connectionId":$conn_id,"prefix":"incoming/","mode":"consume"}}
 JSON
 )
-  src_id=$(auth -X POST "$BASE_URL/api/v1/sources" -H 'Content-Type: application/json' -d "$src_body" \
-    | jq -r '.id // empty' 2>/dev/null)
-  log "S3 source id: ${src_id:-<none>}"
+  src_id=$(find_id_by_name "/api/v1/sources" "Incoming S3")
+  if [ -n "${src_id:-}" ]; then
+    log "S3 source already exists: id $src_id"
+  else
+    src_id=$(auth -X POST "$BASE_URL/api/v1/sources" -H 'Content-Type: application/json' -d "$src_body" \
+      | jq -r '.id // empty' 2>/dev/null)
+    log "S3 source id: ${src_id:-<none>}"
+    [ -n "${src_id:-}" ] || fail_note "S3 source was not created"
+  fi
 
   if [ -n "${src_id:-}" ]; then
+    # 'inputs', not the legacy {trigger, sourceIds}: Policy has no sourceIds field, so the old
+    # shape binds inputs=null and seeds a policy that watches nothing while still returning 200.
     pol_body=$(cat <<JSON
 {"name":"Compress incoming PDFs","enabled":true,
- "trigger":{"type":"schedule","options":{"schedule":{"type":"every","count":5,"unit":"MINUTES"}}},
- "sourceIds":["$src_id"],
+ "inputs":[{"sourceId":"$src_id",
+            "trigger":{"type":"schedule","options":{"schedule":{"type":"every","count":5,"unit":"MINUTES"}}}}],
  "steps":[{"operation":"/api/v1/misc/compress-pdf","parameters":{}}],
  "output":{"type":"s3","options":{"connectionId":$conn_id,"prefix":"processed/"}}}
 JSON
 )
-    code=$(auth -o /tmp/pol.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/policies" \
-      -H 'Content-Type: application/json' -d "$pol_body")
-    log "policy create: HTTP $code $( [ "$code" != 200 ] && head -c 160 /tmp/pol.json )"
+    pol_id=$(find_id_by_name "/api/v1/policies" "Compress incoming PDFs")
+    if [ -n "${pol_id:-}" ]; then
+      log "policy already exists: id $pol_id"
+    else
+      code=$(auth -o /tmp/pol.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/policies" \
+        -H 'Content-Type: application/json' -d "$pol_body")
+      log "policy create: HTTP $code $( [ "$code" != 200 ] && head -c 160 /tmp/pol.json )"
+      case "$code" in
+        2*|409) ;;
+        *)      fail_note "policy create: HTTP $code" ;;
+      esac
+    fi
   fi
 
   # --- webhook source + policy (only if this build has the webhook type) -----
@@ -125,16 +176,27 @@ JSON
  "options":{"connectionId":$conn_id,"mode":"consume"}}
 JSON
 )
-  wh=$(auth -o /tmp/wh.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/sources" \
-    -H 'Content-Type: application/json' -d "$wh_body")
-  if [ "$wh" = "200" ] || [ "$wh" = "201" ]; then
-    wh_url=$(jq -r '.options.webhookId // empty' </tmp/wh.json 2>/dev/null)
-    log "webhook source created (deliver to /api/v1/webhooks/$wh_url)"
+  wh_id=$(find_id_by_name "/api/v1/sources" "Partner webhook")
+  if [ -n "${wh_id:-}" ]; then
+    log "webhook source already exists: id $wh_id"
   else
-    log "webhook source not created (HTTP $wh) - expected on builds without the webhook branch"
+    wh=$(auth -o /tmp/wh.json -w '%{http_code}' -X POST "$BASE_URL/api/v1/sources" \
+      -H 'Content-Type: application/json' -d "$wh_body")
+    if [ "$wh" = "200" ] || [ "$wh" = "201" ]; then
+      wh_url=$(jq -r '.options.webhookId // empty' </tmp/wh.json 2>/dev/null)
+      log "webhook source created (deliver to /api/v1/webhooks/$wh_url)"
+    else
+      # Not counted as an error: builds without the webhook branch legitimately reject this type.
+      log "webhook source not created (HTTP $wh) - expected on builds without the webhook branch"
+    fi
   fi
 fi
 
+if [ "$errors" -gt 0 ]; then
+  log "seed FAILED with $errors error(s)."
+  exit 1
+fi
+
 log "seed complete."
 log "  login: $ADMIN_USER / $ADMIN_PASS at $BASE_URL"
 log "  users: user01..$(printf '%02d' "$USER_COUNT")@stirling.test / $USER_PASS"

testing/compose/multinode/valkey-topology.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Sourced by start-multinode-test.sh and run-multinode-regression.sh.
+
+# Prints the `docker compose -f ...` prefix for a topology; returns 1 on an unknown one. Overlays must
+# follow the base file - they override valkey.command / depends_on, and compose REPLACES command.
+compose_cmd_for_topology() {
+  base="docker compose -f docker-compose-multinode.yml"
+  case "$1" in
+    standalone) echo "$base" ;;
+    sentinel)   echo "$base -f docker-compose-multinode.valkey-sentinel.yml" ;;
+    cluster)    echo "$base -f docker-compose-multinode.valkey-cluster.yml" ;;
+    *) return 1 ;;
+  esac
+}

testing/compose/run-multinode-regression.sh

@@ -1,27 +1,43 @@
 #!/usr/bin/env bash
-# Runs the multi-node regression suite (behave features/multinode) against the clustered stack: brings it up if needed, runs non-destructive scenarios then @destructive failover ones, and restores any killed node.
-# Usage: ./run-multinode-regression.sh [--no-failover] [--no-seed]
-# @known_gap scenarios are expected to fail - they mark work not yet done, so a non-zero exit is fine while those are open.
+# Usage: ./run-multinode-regression.sh [--valkey standalone|sentinel|cluster] [--no-failover] [--no-seed]
+# @known_gap scenarios are expected to fail, so a non-zero exit is fine while those are open.
 set -uo pipefail
 cd "$(dirname "$0")"
 
-COMPOSE="docker compose -f docker-compose-multinode.yml"
+. ./multinode/valkey-topology.sh
+
 CUKE_DIR="../cucumber"
 RUN_FAILOVER=1
 SEED=1
-for arg in "$@"; do
-  case "$arg" in
-    --no-failover) RUN_FAILOVER=0 ;;
-    --no-seed)     SEED=0 ;;
+VALKEY_TOPOLOGY="${VALKEY_TOPOLOGY:-standalone}"
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --no-failover) RUN_FAILOVER=0; shift ;;
+    --no-seed)     SEED=0; shift ;;
+    --valkey)      [ "$#" -ge 2 ] || { echo "--valkey needs a value (standalone|sentinel|cluster)"; exit 2; }
+                   VALKEY_TOPOLOGY="$2"; shift 2 ;;
+    --valkey=*)    VALKEY_TOPOLOGY="${1#*=}"; shift ;;
+    *) echo "Unknown argument '$1'"; exit 2 ;;
   esac
 done
 
-echo "==> Ensuring the multi-node stack is up..."
-if ! docker inspect -f '{{.State.Health.Status}}' multinode-stirling-1 2>/dev/null | grep -q healthy; then
-  ./start-multinode-test.sh $([ "$SEED" = 0 ] && echo --no-seed) || exit 1
+# Same overlay set as start-multinode-test.sh, so `up -d` here restores killed nodes without
+# silently dropping the sentinel/cluster services back to the base standalone Valkey.
+COMPOSE=$(compose_cmd_for_topology "$VALKEY_TOPOLOGY") \
+  || { echo "Unknown --valkey topology '$VALKEY_TOPOLOGY' (expected standalone|sentinel|cluster)"; exit 2; }
+
+SEED_LOG="$(pwd)/multinode/seed.log"
+
+echo "==> Ensuring the multi-node stack is up (valkey=$VALKEY_TOPOLOGY)..."
+# Exact match: "unhealthy" contains "healthy", so a substring test reads a broken node as up.
+node_health=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-1 2>/dev/null | tr -d '\r')
+if [ "$node_health" != "healthy" ]; then
+  ./start-multinode-test.sh --valkey "$VALKEY_TOPOLOGY" $([ "$SEED" = 0 ] && echo --no-seed) || exit 1
 elif [ "$SEED" = 1 ]; then
   echo "    stack already up; seeding (idempotent)..."
-  $COMPOSE --profile seed run --rm seed >/dev/null 2>&1 || echo "    (seed reported issues, continuing)"
+  if ! $COMPOSE --profile seed run --rm seed 2>&1 | tee "$SEED_LOG"; then
+    echo "    (seed reported issues, continuing; full output in $SEED_LOG)"
+  fi
 fi
 
 echo "==> Checking Python + behave..."
@@ -46,18 +62,31 @@ run_behave "~@destructive" "core" || rc=1
 if [ "$RUN_FAILOVER" = 1 ]; then
   run_behave "@destructive" "failover" || rc=1
   echo "==> Restoring any killed nodes..."
-  $COMPOSE up -d >/dev/null 2>&1
+  if ! $COMPOSE up -d --remove-orphans; then
+    echo "    ERROR: could not restore the stack ('up -d' failed)"
+    rc=1
+  fi
   for n in multinode-stirling-1 multinode-stirling-2; do
+    restored=0
     for i in $(seq 1 24); do
-      [ "$(docker inspect -f '{{.State.Health.Status}}' "$n" 2>/dev/null)" = "healthy" ] && break
+      [ "$(docker inspect -f '{{.State.Health.Status}}' "$n" 2>/dev/null | tr -d '\r')" = "healthy" ] \
+        && { restored=1; break; }
       sleep 5
     done
+    if [ "$restored" = 1 ]; then
+      echo "    $n: healthy"
+    else
+      echo "    ERROR: $n did not return to healthy after failover; last 40 log lines:"
+      docker logs --tail 40 "$n" 2>&1 | sed 's/^/      /'
+      rc=1
+    fi
   done
 fi
 
 echo
 echo "============================================================"
 echo " Regression run complete. Reports: $REPORT_DIR"
+echo " Valkey topology: $VALKEY_TOPOLOGY"
 echo " Exit $rc (non-zero = at least one scenario failed;"
 echo " @known_gap scenarios are expected to fail - see the report)."
 echo " Stack left running: http://localhost:8080  (admin / stirling)"

testing/compose/start-multinode-test.sh

@@ -1,19 +1,51 @@
 #!/usr/bin/env bash
 # Brings up the multi-node stack (Postgres/Valkey/MinIO/2 app nodes/nginx LB), seeds teams/users/an S3 connection/policies, then leaves it running for manual testing.
-# Usage: ./start-multinode-test.sh [--no-seed | --down]
+# Usage: ./start-multinode-test.sh [--valkey standalone|sentinel|cluster] [--no-seed | --down]
 set -euo pipefail
 cd "$(dirname "$0")"
 
-COMPOSE="docker compose -f docker-compose-multinode.yml"
+# Valkey topology, layered as a compose overlay so the default (standalone) stack is unchanged.
+. ./multinode/valkey-topology.sh
+VALKEY_TOPOLOGY="${VALKEY_TOPOLOGY:-standalone}"
+DOWN=0
+SEED=1
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --valkey)   [ "$#" -ge 2 ] || { echo "--valkey needs a value (standalone|sentinel|cluster)"; exit 2; }
+                VALKEY_TOPOLOGY="$2"; shift 2 ;;
+    --valkey=*) VALKEY_TOPOLOGY="${1#*=}"; shift ;;
+    --down)     DOWN=1; shift ;;
+    --no-seed)  SEED=0; shift ;;
+    *) echo "Unknown argument '$1'. Usage: $0 [--valkey standalone|sentinel|cluster] [--no-seed | --down]"; exit 2 ;;
+  esac
+done
 
-if [ "${1:-}" = "--down" ]; then
+COMPOSE=$(compose_cmd_for_topology "$VALKEY_TOPOLOGY") \
+  || { echo "Unknown --valkey topology '$VALKEY_TOPOLOGY' (expected standalone|sentinel|cluster)"; exit 2; }
+
+# Records which topology is live so switching overlays tears the old services down instead of
+# orphaning them (stale containers otherwise confuse validate-multinode-test.sh's detection).
+TOPOLOGY_MARKER="multinode/.active-topology"
+PREV_TOPOLOGY=""
+if [ -f "$TOPOLOGY_MARKER" ]; then
+  PREV_TOPOLOGY=$(tr -d '[:space:]' < "$TOPOLOGY_MARKER" 2>/dev/null || echo "")
+fi
+
+if [ "$DOWN" = "1" ]; then
   echo "Tearing down multi-node stack + volumes..."
   $COMPOSE --profile seed down -v --remove-orphans
+  rm -f "$TOPOLOGY_MARKER"
   exit 0
 fi
 
-SEED=1
-[ "${1:-}" = "--no-seed" ] && SEED=0
+if [ -n "$PREV_TOPOLOGY" ] && [ "$PREV_TOPOLOGY" != "$VALKEY_TOPOLOGY" ]; then
+  echo "==> Topology change ($PREV_TOPOLOGY -> $VALKEY_TOPOLOGY); tearing the old stack down first..."
+  PREV_COMPOSE=$(compose_cmd_for_topology "$PREV_TOPOLOGY") || PREV_COMPOSE=""
+  if [ -n "$PREV_COMPOSE" ]; then
+    $PREV_COMPOSE --profile seed down -v --remove-orphans || true
+  fi
+  rm -f "$TOPOLOGY_MARKER"
+fi
 
 # Cluster mode is licence-gated. Without a valid key the nodes fail the cluster licence gate at boot.
 if [ -z "${PREMIUM_KEY:-}" ]; then
@@ -24,8 +56,9 @@ fi
 echo "==> Building the Stirling image (first run compiles the app; be patient)..."
 $COMPOSE build
 
-echo "==> Starting Postgres + Valkey + MinIO + 2 app nodes + nginx..."
-$COMPOSE up -d
+echo "==> Starting Postgres + Valkey ($VALKEY_TOPOLOGY) + MinIO + 2 app nodes + nginx..."
+$COMPOSE up -d --remove-orphans
+printf '%s\n' "$VALKEY_TOPOLOGY" > "$TOPOLOGY_MARKER"
 
 echo "==> Waiting for both app nodes to report healthy..."
 for node in multinode-stirling-1 multinode-stirling-2; do
@@ -45,7 +78,7 @@ fi
 cat <<EOF
 
 ============================================================================
- Multi-node Stirling is UP.
+ Multi-node Stirling is UP.   Valkey topology: $VALKEY_TOPOLOGY
 
    App (via load balancer): http://localhost:8080     (admin / stirling)
    MinIO console:           http://localhost:9001     (minioadmin / minioadmin)
@@ -57,7 +90,11 @@ cat <<EOF
  Try it:
    ./validate-multinode-test.sh        # multi-node smoke tests (optional)
    $COMPOSE logs -f stirling-1         # tail a node
-   ./start-multinode-test.sh --down    # stop + wipe
+   ./start-multinode-test.sh --valkey $VALKEY_TOPOLOGY --down    # stop + wipe
+
+ Other Valkey topologies (each wipes and rebuilds the backplane):
+   ./start-multinode-test.sh --valkey sentinel   # 1 primary + 2 replicas + 3 sentinels
+   ./start-multinode-test.sh --valkey cluster    # 3 primaries + 3 replicas, sharded
 
  Nodes are reachable directly for cross-node checks:
    docker compose -f docker-compose-multinode.yml exec stirling-1 curl -s localhost:8080/api/v1/info/status

testing/compose/validate-multinode-test.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
-# Multi-node smoke tests against a running stack (start-multinode-test.sh): load-balancer spread, cross-node JWT validation (signing keys persist in the shared DB), and processor state visible from every node.
-# Auth: extracts the Bearer JWT from the login body via sed (no jq needed host-side) and hits nodes directly with docker exec.
-# Non-destructive - safe to re-run against the stack at http://localhost:8080.
+# Non-destructive smoke tests against a running stack; safe to re-run.
+# JWT is extracted from the login body with sed so no host-side jq is needed.
 set -uo pipefail
 cd "$(dirname "$0")"
 
@@ -13,6 +12,47 @@ PROBE="/api/v1/sources"
 pass=0; fail=0
 ok()   { echo "  PASS - $*"; pass=$((pass+1)); }
 bad()  { echo "  FAIL - $*"; fail=$((fail+1)); }
+skip() { echo "  SKIP - $*"; }
+
+# Which Valkey topology is live, read off the running containers rather than a flag, so this script
+# is correct no matter how the stack was brought up.
+detect_topology() {
+  # cluster_enabled lives in INFO cluster, NOT in CLUSTER INFO (which only reports state/slots).
+  if docker exec multinode-valkey valkey-cli info cluster 2>/dev/null | tr -d '\r' | grep -q '^cluster_enabled:1'; then
+    echo cluster
+  # Must be a RUNNING sentinel: docker inspect also succeeds for a stopped leftover from a previous run.
+  elif [ -n "$(docker ps --filter name=multinode-valkey-sentinel-1 --filter status=running -q 2>/dev/null)" ]; then
+    echo sentinel
+  else
+    echo standalone
+  fi
+}
+TOPOLOGY=$(detect_topology)
+
+# Every stirling:* key in any topology: a cluster shards them so --cluster call fans out, plain KEYS is
+# the non-cluster fallback. Must be docker exec - `docker run --entrypoint /bin/sh` is mangled by MSYS.
+backplane_keys_raw() {
+  docker exec multinode-valkey valkey-cli --cluster call --cluster-only-masters 127.0.0.1:6379 keys 'stirling:*' 2>/dev/null \
+    || docker exec multinode-valkey valkey-cli keys 'stirling:*' 2>/dev/null
+}
+# Same, minus the "host:port: " prefix a cluster call prepends to every line.
+backplane_keys() {
+  backplane_keys_raw | tr -d '\r' | sed 's/^[A-Za-z0-9_.-]*:[0-9][0-9]*: //' | grep '^stirling:'
+}
+
+# INFO field off the Valkey the app writes to (the primary in every topology).
+valkey_info() { valkey_info_on multinode-valkey "$1"; }
+valkey_info_on() { docker exec "$1" valkey-cli info "$2" 2>/dev/null | tr -d '\r'; }
+
+# Every reachable Valkey DATA container (1 / 3 / 6 by topology); sampling only multinode-valkey would
+# cover a sixth of a cluster. Sentinels are the monitoring plane, not the backplane, so they are excluded.
+valkey_nodes() {
+  docker ps --format '{{.Names}}' --filter name=multinode-valkey 2>/dev/null | tr -d '\r' \
+    | grep -v -e sentinel -e cluster-init | sort | while read -r c; do
+      docker exec "$c" valkey-cli ping 2>/dev/null | tr -d '\r' | grep -q '^PONG' && echo "$c"
+    done
+}
+VALKEY_NODES=$(valkey_nodes)
 
 login() { # -> prints the bearer token
   curl -s -X POST "$LB/api/v1/auth/login" -H 'Content-Type: application/json' \
@@ -73,9 +113,121 @@ else
   bad "integration list failed (HTTP $lc) - credential key may not be shared across nodes"
 fi
 
+echo "== 6. Every Valkey connection is attributable (CLIENT SETNAME) =="
+# Census every data node: one container carries a sixth of the connections in a 6-shard cluster.
+# Scoped to lib-name=Lettuce - valkey-cli and monitoring agents legitimately have no name.
+sampled=0; anon=0; lettuce=0; all_names=""
+for c in $VALKEY_NODES; do
+  sampled=$((sampled+1))
+  clients=$(docker exec "$c" valkey-cli client list 2>/dev/null | tr -d '\r' | grep 'lib-name=Lettuce')
+  lettuce=$(( lettuce + $(printf '%s\n' "$clients" | grep -c 'lib-name=Lettuce') ))
+  anon=$(( anon + $(printf '%s\n' "$clients" | grep -c 'name= ') ))
+  all_names="$all_names
+$(printf '%s\n' "$clients" | grep -o 'name=stirling-[^ ]*')"
+done
+named=$(printf '%s\n' "$all_names" | grep . | sort -u)
+distinct=$(printf '%s\n' "$named" | grep -c .)
+echo "  sampled $sampled Valkey node(s); $lettuce Lettuce connection(s); app connection names: $(printf '%s' "$named" | paste -sd, -)"
+if [ "$sampled" -lt 1 ]; then
+  bad "no reachable Valkey container found - cannot census client names"
+elif [ "${lettuce:-0}" -eq 0 ]; then
+  # Population floor: with zero Lettuce clients the anon count is also zero and would falsely pass.
+  bad "no Lettuce connections found across $sampled node(s) - nothing to census (are the app nodes connected?)"
+else
+  [ "${anon:-0}" -eq 0 ] && ok "no unnamed app connections across $sampled node(s) (all carry CLIENT SETNAME)" \
+                          || bad "$anon Lettuce connection(s) have an empty name= - CLIENT SETNAME is not applied"
+  [ "${distinct:-0}" -ge 2 ] && ok "$distinct distinct stirling-* client names (load attributable per node)" \
+                              || bad "only ${distinct:-0} distinct stirling-* client name(s) (expected one per app node)"
+fi
+
+echo "== 7. Backplane traffic multiplexes over one shared connection =="
+# Job load must not open a stream of new connections: every backplane command rides the shared native
+# connection. This measures multiplexing, NOT whether the Lettuce pool is enabled.
+JOBS=200
+CHURN_BUDGET=50
+conns_total() {
+  total=0
+  for c in $VALKEY_NODES; do
+    n=$(valkey_info_on "$c" stats | sed -n 's/^total_connections_received:\([0-9]*\).*/\1/p')
+    total=$(( total + ${n:-0} ))
+  done
+  echo "$total"
+}
+vk_count=$(printf '%s\n' "$VALKEY_NODES" | grep -c .)
+if [ -z "${jwt:-}" ]; then
+  skip "no JWT - cannot drive load to measure connection growth"
+elif [ "${vk_count:-0}" -lt 1 ]; then
+  bad "no reachable Valkey container found - cannot measure connection growth"
+else
+  probe_pdf=$(mktemp -t mn-probe-XXXXXX.pdf 2>/dev/null || echo /tmp/mn-probe.pdf)
+  printf '%%PDF-1.4\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj\n3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]>>endobj\ntrailer<</Root 1 0 R>>\n%%%%EOF\n' > "$probe_pdf"
+  before=$(conns_total)
+  submitted=0
+  for i in $(seq 1 "$JOBS"); do
+    hc=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $jwt" \
+         -F "fileInput=@$probe_pdf" -F "angle=90" "$LB/api/v1/general/rotate-pdf?async=true")
+    [ "$hc" = "200" ] && submitted=$((submitted+1))
+  done
+  after=$(conns_total)
+  rm -f "$probe_pdf"
+  delta=$(( ${after:-0} - ${before:-0} ))
+  echo "  total_connections_received over $vk_count Valkey node(s): $before -> $after (delta $delta over $submitted/$JOBS async jobs)"
+  if [ "$submitted" -lt 1 ]; then
+    bad "no async jobs were accepted - cannot measure connection growth"
+  elif [ "$delta" -lt "$CHURN_BUDGET" ]; then
+    ok "only $delta new connections for $submitted async jobs - backplane multiplexes over the shared connection"
+  else
+    bad "$delta new connections for $submitted async jobs - a connection is opened per operation (expected < $CHURN_BUDGET)"
+  fi
+fi
+
+echo "== 8. Every app node registered a heartbeat in the backplane =="
+nodes_registered=$(backplane_keys | grep -c '^stirling:nodes:')
+expected=$(printf '%s\n' $NODES | grep -c .)
+echo "  stirling:nodes:* heartbeats: ${nodes_registered:-0} (expected >= $expected)"
+[ "${nodes_registered:-0}" -ge "$expected" ] && ok "all $expected app nodes registered in Valkey" \
+                                             || bad "only ${nodes_registered:-0} of $expected app nodes registered"
+
+echo "== 9. Valkey topology is redundant (detected: $TOPOLOGY) =="
+case "$TOPOLOGY" in
+  sentinel)
+    reps=$(valkey_info replication | sed -n 's/^connected_slaves:\([0-9]*\).*/\1/p')
+    [ "${reps:-0}" -ge 2 ] && ok "primary is replicating to ${reps} replicas" \
+                            || bad "primary has ${reps:-0} connected replicas (expected 2) - no failover target"
+    quorum=$(docker exec multinode-valkey-sentinel-1 valkey-cli -p 26379 sentinel ckquorum mymaster 2>/dev/null | tr -d '\r')
+    case "$quorum" in
+      OK*) ok "sentinels have quorum: $quorum" ;;
+      *)   bad "sentinel quorum check failed: ${quorum:-<no reply>}" ;;
+    esac
+    # ckquorum reports OK even with zero discovered replicas, so assert the failover candidates too.
+    known=$(docker exec multinode-valkey-sentinel-1 valkey-cli -p 26379 sentinel replicas mymaster 2>/dev/null | tr -d '\r' | grep -c '^name$')
+    [ "${known:-0}" -ge 2 ] && ok "sentinel-1 knows $known failover candidates" \
+                             || bad "sentinel-1 knows only ${known:-0} replicas - a failover would have no target"
+    ;;
+  cluster)
+    info=$(docker exec multinode-valkey valkey-cli cluster info 2>/dev/null | tr -d '\r')
+    printf '%s\n' "$info" | grep -q '^cluster_state:ok' && ok "cluster_state:ok" || bad "cluster_state is not ok"
+    slots=$(printf '%s\n' "$info" | sed -n 's/^cluster_slots_ok:\([0-9]*\).*/\1/p')
+    [ "${slots:-0}" = "16384" ] && ok "all 16384 slots served" || bad "only ${slots:-0}/16384 slots served"
+    # Masters carry '-' in the master-id column; replicas carry their primary's id there.
+    masters=$(docker exec multinode-valkey valkey-cli cluster nodes 2>/dev/null | tr -d '\r' | grep -c 'master -')
+    known=$(printf '%s\n' "$info" | sed -n 's/^cluster_known_nodes:\([0-9]*\).*/\1/p')
+    [ "${masters:-0}" -ge 3 ] && ok "$masters primaries sharding the keyspace (${known:-?} nodes known)" \
+                               || bad "only ${masters:-0} primaries (expected 3)"
+    # Keys must actually spread; everything on one shard would mean something pinned them to a slot.
+    shards=$(backplane_keys_raw | tr -d '\r' | grep -c '^[A-Za-z0-9_.-]*:[0-9][0-9]*: stirling:')
+    [ "${shards:-0}" -ge 1 ] && ok "backplane keys present on ${shards} shard(s)" \
+                              || bad "no backplane keys found on any shard"
+    ;;
+  *)
+    skip "standalone Valkey - no replication to verify (use --valkey sentinel|cluster for HA)"
+    ;;
+esac
+
 echo
 echo "============================================================"
 echo " Multi-node validation: $pass passed, $fail failed."
+echo " Valkey topology: $TOPOLOGY"
 echo " Stack left running: $LB  (admin / stirling)"
 echo "============================================================"
 [ "$fail" -eq 0 ]

testing/cucumber/features/multinode/cluster_health.feature

@@ -12,6 +12,11 @@ Feature: Multi-node cluster health
         Given the multi-node stack is running
         And both nodes are cluster members using the Valkey backplane
 
+    @smoke
+    Scenario: Every node published a heartbeat to the Valkey backplane
+        Given the multi-node stack is running
+        Then every application node should be registered in the backplane
+
     Scenario: The load balancer answers the health endpoint
         Given the multi-node stack is running
         When I request "/api/v1/info/status" 4 times through the load balancer

testing/cucumber/features/multinode/shared_state.feature

@@ -14,7 +14,9 @@ Feature: Shared state across nodes
         And I am logged in as admin
         Then every node should report the same number of sources
 
+    # Not the seed's USER_COUNT: the licence caps seats, so the seed stops short of its target.
+    # 10 proves the seeded org reached the shared database without encoding a seat count.
     Scenario: The seeded org is present in the shared database
         Given the multi-node stack is running
-        Then the "users" table should contain at least 40 row(s)
+        Then the "users" table should contain at least 10 row(s)
         And the "integration_configs" table should contain at least 1 row(s)

testing/cucumber/features/steps/multinode_step_definitions.py

@@ -2,6 +2,7 @@
 
 import io
 import json
+import re
 import subprocess
 import time
 import uuid
@@ -11,8 +12,13 @@ from behave import given, then, when
 
 LB_URL = "http://localhost:8080"
 NODES = ["multinode-stirling-1", "multinode-stirling-2"]
+# Must match CLUSTER_NODE_ID in testing/compose/docker-compose-multinode.yml.
+NODE_IDS = ["multinode-1", "multinode-2"]
 PG = "multinode-postgres"
 MINIO = "multinode-minio"
+# The container name is stable but its Valkey role is not (cluster shards, sentinel failover), so
+# every keyspace probe must fan out in cluster mode - see _backplane_keys.
+VALKEY = "multinode-valkey"
 BUCKET = "policy-data"
 SOURCE_PREFIX = "incoming/"
 OUTPUT_PREFIX = "processed/"
@@ -95,9 +101,11 @@ def _names_on_node(node, path, token):
 
 
 def _policy_body(name, source_ids=None, enabled=True):
+    # Must be the 'inputs' shape. Policy has no sourceIds field, so a legacy
+    # {sourceIds, trigger} body binds inputs=null and silently stores a policy referencing nothing.
     return json.dumps({
-        "name": name, "enabled": enabled, "trigger": None,
-        "sourceIds": source_ids or [],
+        "name": name, "enabled": enabled,
+        "inputs": [{"sourceId": sid, "trigger": None} for sid in (source_ids or [])],
         "steps": [{"operation": "/api/v1/misc/compress-pdf", "parameters": {}}],
         "output": {"type": "inline", "options": {}},
     })
@@ -397,15 +405,74 @@ def step_run_visible_every(context):
 
 
 # --------------------------------------------------------------------------- rate limiting
+# --cluster call prefixes each node's reply with 'host:port: '; a stirling: key never looks like that.
+_NODE_PREFIX = re.compile(r"^[A-Za-z0-9_.\-]+:\d+:\s?")
+
+
+def _cluster_mode():
+    """True when this Valkey runs in cluster mode, so a plain KEYS would see one shard only."""
+    rc, out, err = _sh(["docker", "exec", VALKEY, "valkey-cli", "info", "cluster"], timeout=30)
+    assert rc == 0, f"valkey INFO cluster failed: {err.strip() or out.strip()}"
+    return "cluster_enabled:1" in out
+
+
+def _backplane_keys():
+    """Every stirling:* key in the backplane, whatever the Valkey topology."""
+    if _cluster_mode():
+        # A fan-out failure must be loud: falling back to a single-shard KEYS would silently
+        # report a fraction of the keyspace and every downstream count would be wrong.
+        rc, out, err = _sh(["docker", "exec", VALKEY, "valkey-cli", "--cluster", "call",
+                            "--cluster-only-masters", "127.0.0.1:6379", "keys", "stirling:*"],
+                           timeout=60)
+        assert rc == 0, f"valkey cluster fan-out failed: {err.strip() or out.strip()}"
+    else:
+        rc, out, err = _sh(["docker", "exec", VALKEY, "valkey-cli", "keys", "stirling:*"], timeout=30)
+        assert rc == 0, f"valkey probe failed: {err.strip() or out.strip()}"
+    return [k for k in (_NODE_PREFIX.sub("", ln.strip()) for ln in out.splitlines())
+            if k.startswith("stirling:")]
+
+
+def _post_through_lb(context, count=3):
+    """POST through the LB; returns True if a node reported the X-Rate-Limit-Remaining header."""
+    if not getattr(context, "jwt_token", None):
+        _lb_login(context)
+    limited = False
+    for _ in range(count):
+        marker = uuid.uuid4().hex[:8]
+        r = requests.post(f"{LB_URL}/api/v1/general/rotate-pdf",
+                          headers={"Authorization": f"Bearer {context.jwt_token}"},
+                          files={"fileInput": (f"rl-{marker}.pdf", _pdf_bytes(marker),
+                                               "application/pdf")},
+                          data={"angle": 90}, timeout=60)
+        limited = limited or "X-Rate-Limit-Remaining" in r.headers
+    return limited
+
+
 @then("the rate-limit counter should be shared across nodes")
 def step_ratelimit_shared(context):
-    # In cluster mode the ValkeyRateLimitStore holds counters in Valkey; probe that a key exists.
-    net = context._net or _network()
-    rc, out, err = _sh(["docker", "run", "--rm", "--network", net, "--entrypoint", "/bin/sh",
-                        "valkey/valkey:8-alpine", "-c",
-                        "valkey-cli -h valkey keys '*'"], timeout=30)
-    assert rc == 0, f"valkey probe failed: {err.strip()}"
-    assert out.strip(), "no keys in Valkey - rate-limit/backplane state is not shared"
+    # Heartbeat keys are always present, so only a stirling:rl: bucket created by real traffic
+    # proves the counters live in the backplane rather than per node.
+    if not _post_through_lb(context):
+        context.scenario.skip(
+            "rate limiting is not active on this stack (no X-Rate-Limit-Remaining header on a "
+            "POST through the LB), so no stirling:rl: bucket can exist - shared rate limiting is "
+            "unproven here, not proven")
+        return
+    buckets = [k for k in _backplane_keys() if k.startswith("stirling:rl:")]
+    assert buckets, (
+        "POSTs through the LB were rate limited but the backplane holds no stirling:rl: key - "
+        "the counters are per node, not shared")
+
+
+@then("every application node should be registered in the backplane")
+def step_nodes_registered(context):
+    # Assert the exact node ids: counting keys lets a stale heartbeat from an earlier run stand in
+    # for a node that never registered.
+    keys = set(_backplane_keys())
+    missing = [nid for nid in NODE_IDS if f"stirling:nodes:{nid}" not in keys]
+    assert not missing, (
+        f"no stirling:nodes: heartbeat for {missing}; the backplane holds "
+        f"{sorted(k for k in keys if k.startswith('stirling:nodes:'))}")
 
 
 # --------------------------------------------------------------------------- failover