Multi node cluster fixes (#7025)

- Exclude DataRedisRepositoriesAutoConfiguration (cluster crash-loop
fix)
- Share JWT signing keys via the DB + require a shared credential key in
cluster mode
- Make policy run status/listing visible across nodes


---

## Checklist

### General

- [ ] I have read the [Contribution
Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md)
- [ ] I have read the [Stirling-PDF Developer
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md)
(if applicable)
- [ ] I have read the [How to add new languages to
Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md)
(if applicable)
- [ ] I have performed a self-review of my own code
- [ ] My changes generate no new warnings

### Documentation

- [ ] I have updated relevant docs on [Stirling-PDF's doc
repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/)
(if functionality has heavily changed)
- [ ] I have read the section [Add New Translation
Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags)
(for new translation tags only)

### Translations (if applicable)

- [ ] I ran
[`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md)

### UI Changes (if applicable)

- [ ] Screenshots or videos demonstrating the UI changes are attached
(e.g., as comments or direct attachments in the PR)

### Testing (if applicable)

- [ ] I have run `task check` to verify linters, typechecks, and tests
pass
- [ ] I have tested my changes locally. Refer to the [Testing
Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing)
for more details.

app/common/src/main/java/stirling/software/common/model/job/JobResult.java

@@ -4,6 +4,8 @@ import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -47,6 +49,9 @@ public class JobResult {
      */
     private final List<String> notes = new CopyOnWriteArrayList<>();
 
+    /** Key/value metadata that survives the write-through into the shared job store. */
+    private final Map<String, String> metadata = new ConcurrentHashMap<>();
+
     /**
      * Create a new JobResult with the given job ID
      *
@@ -161,4 +166,16 @@ public class JobResult {
     public List<String> getNotes() {
         return Collections.unmodifiableList(notes);
     }
+
+    /** Attach a metadata value, e.g. a policy id so cluster peers can identify a policy run. */
+    public void putMetadata(String key, String value) {
+        if (key != null && value != null) {
+            this.metadata.put(key, value);
+        }
+    }
+
+    /** An unmodifiable view of this job's metadata. */
+    public Map<String, String> getMetadata() {
+        return Collections.unmodifiableMap(metadata);
+    }
 }

app/common/src/main/java/stirling/software/common/service/TaskManager.java

@@ -230,6 +230,18 @@ public class TaskManager {
         return false;
     }
 
+    /** Attach metadata to a job and write it through to the shared store for cluster peers. */
+    public boolean putMetadata(String jobId, String key, String value) {
+        JobResult jobResult = jobResults.get(jobId);
+        if (jobResult != null) {
+            jobResult.putMetadata(key, value);
+            writeThrough(jobId, jobResult);
+            return true;
+        }
+        log.warn("Attempted to set metadata on non-existent job ID: {}", jobId);
+        return false;
+    }
+
     /**
      * Get statistics about all jobs in the system
      *
@@ -378,7 +390,7 @@ public class TaskManager {
                 fileIds.add(rf.getFileId());
             }
         }
-        Map<String, String> meta = new HashMap<>();
+        Map<String, String> meta = new HashMap<>(result.getMetadata());
         if (result.getNotes() != null && !result.getNotes().isEmpty()) {
             meta.put("notesCount", Integer.toString(result.getNotes().size()));
         }

app/core/src/main/resources/application.properties

@@ -98,7 +98,8 @@ spring.main.allow-bean-definition-overriding=true
 # spring-data-redis is on the classpath only for the optional Valkey backplane (which wires its own
 # factory); exclude Spring Boot's stock Redis auto-config so a default install doesn't create a dead
 # localhost:6379 factory that flips /actuator/health to DOWN.
-spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration
+# Also exclude the repositories auto-config: in cluster mode it needs a redisTemplate bean we don't define.
+spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisRepositoriesAutoConfiguration
 
 # Set up a consistent temporary directory location
 java.io.tmpdir=${stirling.tempfiles.directory:${java.io.tmpdir}/stirling-pdf}

app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java

@@ -44,10 +44,13 @@ public class CredentialEncryption {
     private static volatile SecretKey key;
 
     private final String configuredKey;
+    private final boolean clusterEnabled;
 
     public CredentialEncryption(
-            @Value("${stirling.security.credentialEncryptionKey:}") String configuredKey) {
+            @Value("${stirling.security.credentialEncryptionKey:}") String configuredKey,
+            @Value("${cluster.enabled:false}") boolean clusterEnabled) {
         this.configuredKey = configuredKey;
+        this.clusterEnabled = clusterEnabled;
     }
 
     @PostConstruct
@@ -64,6 +67,14 @@ public class CredentialEncryption {
         if (configured != null && !configured.isBlank()) {
             return new SecretKeySpec(Base64.getDecoder().decode(configured.trim()), ALGORITHM);
         }
+        // Cluster nodes must share this key, so fail fast rather than generate a node-local one.
+        if (clusterEnabled) {
+            throw new IllegalStateException(
+                    "cluster.enabled=true requires a shared credential encryption key. Set"
+                            + " STIRLING_CREDENTIAL_ENCRYPTION_KEY (or"
+                            + " stirling.security.credentialEncryptionKey) to the same value on every"
+                            + " node.");
+        }
         return loadOrCreateKeyFile();
     }
 

app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java

@@ -6,6 +6,7 @@ import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
@@ -39,6 +40,8 @@ import jakarta.validation.Valid;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.job.JobResponse;
 import stirling.software.common.service.JobOwnershipService;
@@ -102,6 +105,8 @@ public class PolicyController {
     private final ApplicationProperties applicationProperties;
     private final TempFileManager tempFileManager;
     private final JobOwnershipService jobOwnershipService;
+    // Shared job store: lets the run endpoints see runs that executed on other nodes.
+    private final JobStore jobStore;
 
     @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     @Operation(
@@ -172,10 +177,19 @@ public class PolicyController {
             description = "Returns the current status, step cursor, and output files of a run.")
     public ResponseEntity<PolicyRunView> status(@PathVariable String runId) {
         PolicyRun run = runRegistry.get(runId);
-        if (run == null) {
-            return ResponseEntity.notFound().build();
+        if (run != null) {
+            return ResponseEntity.ok(PolicyRunView.of(run));
         }
-        return ResponseEntity.ok(PolicyRunView.of(run));
+        // Not local: read the run's shared projection so any node can serve its status.
+        if (ownedByCurrentUser(runId)) {
+            Optional<JobStoreEntry> entry = jobStore.get(runId);
+            if (entry.isPresent()
+                    && entry.get().resultMeta() != null
+                    && entry.get().resultMeta().containsKey("policyId")) {
+                return ResponseEntity.ok(PolicyRunView.ofEntry(entry.get()));
+            }
+        }
+        return ResponseEntity.notFound().build();
     }
 
     @GetMapping("/runs")
@@ -188,11 +202,26 @@ public class PolicyController {
                             + " collected, rather than orphaned on the backend. Ad-hoc runs (no"
                             + " policy id) are excluded.")
     public List<PolicyRunView> listRuns() {
-        return runRegistry.all().stream()
+        // Local runs first (they carry live step state); keyed by runId to dedupe shared entries.
+        Map<String, PolicyRunView> byRunId = new LinkedHashMap<>();
+        runRegistry.all().stream()
                 .filter(run -> run.getPolicyId() != null)
                 .filter(run -> ownedByCurrentUser(run.getRunId()))
-                .map(PolicyRunView::of)
-                .toList();
+                .forEach(run -> byRunId.put(run.getRunId(), PolicyRunView.of(run)));
+        // Then runs from other nodes, read from the shared job store.
+        for (JobStoreEntry entry : jobStore.all()) {
+            if (byRunId.containsKey(entry.jobId())) {
+                continue;
+            }
+            Map<String, String> meta = entry.resultMeta();
+            if (meta == null || !meta.containsKey("policyId")) {
+                continue; // ad-hoc job, not a stored-policy run
+            }
+            if (ownedByCurrentUser(entry.jobId())) {
+                byRunId.put(entry.jobId(), PolicyRunView.ofEntry(entry));
+            }
+        }
+        return List.copyOf(byRunId.values());
     }
 
     /**

app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java

@@ -134,6 +134,10 @@ public class PolicyEngine {
         // ownership check passes. No-op when security is off.
         String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString());
         taskManager.createTask(runId);
+        // Tag the shared job entry with the policy id so peers can list it as a policy run.
+        if (policyId != null) {
+            taskManager.putMetadata(runId, "policyId", policyId);
+        }
         PolicyRun run = new PolicyRun(runId, policyId, definition);
         registry.register(run);
         CompletableFuture<PolicyRun> completion = new CompletableFuture<>();

app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java

@@ -1,7 +1,9 @@
 package stirling.software.proprietary.policy.model;
 
 import java.util.List;
+import java.util.Map;
 
+import stirling.software.common.cluster.JobStoreEntry;
 import stirling.software.common.model.job.ResultFile;
 
 /**
@@ -34,4 +36,33 @@ public record PolicyRunView(
                 run.getOutputs(),
                 run.getCreatedAt().toEpochMilli());
     }
+
+    /** Cross-node view from a shared job-store entry; step cursor is node-local so it reads 0. */
+    public static PolicyRunView ofEntry(JobStoreEntry entry) {
+        Map<String, String> meta = entry.resultMeta() == null ? Map.of() : entry.resultMeta();
+        PolicyRunStatus status =
+                switch (entry.state()) {
+                    case COMPLETE -> PolicyRunStatus.COMPLETED;
+                    case FAILED -> PolicyRunStatus.FAILED;
+                    case RUNNING, PENDING -> PolicyRunStatus.RUNNING;
+                };
+        List<ResultFile> outputs =
+                entry.fileIds() == null
+                        ? List.of()
+                        : entry.fileIds().stream()
+                                .map(id -> ResultFile.builder().fileId(id).build())
+                                .toList();
+        long createdAt = entry.createdAt() == null ? 0L : entry.createdAt().toEpochMilli();
+        return new PolicyRunView(
+                entry.jobId(),
+                meta.get("policyId"),
+                status,
+                0,
+                0,
+                entry.error(),
+                null,
+                null,
+                outputs,
+                createdAt);
+    }
 }

app/proprietary/src/main/java/stirling/software/proprietary/security/model/JwtSigningKeyEntity.java

@@ -0,0 +1,52 @@
+package stirling.software.proprietary.security.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import org.hibernate.annotations.CreationTimestamp;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Convert;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import stirling.software.proprietary.integration.crypto.EncryptedStringConverter;
+
+/** A JWT signing keypair in the shared DB; the private key is encrypted at rest. */
+@Entity
+@Table(name = "jwt_signing_keys")
+@NoArgsConstructor
+@Getter
+@Setter
+public class JwtSigningKeyEntity implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    @Id
+    @Column(name = "key_id", length = 128)
+    private String keyId;
+
+    // Base64 X.509 public key (non-secret).
+    @Column(name = "verifying_key", columnDefinition = "text", nullable = false)
+    private String verifyingKey;
+
+    // Base64 PKCS#8 private key, encrypted at rest.
+    @Convert(converter = EncryptedStringConverter.class)
+    @Column(name = "signing_key", columnDefinition = "text", nullable = false)
+    private String signingKey;
+
+    @CreationTimestamp
+    @Column(name = "created_at", updatable = false)
+    private LocalDateTime createdAt;
+
+    public JwtSigningKeyEntity(String keyId, String verifyingKey, String signingKey) {
+        this.keyId = keyId;
+        this.verifyingKey = verifyingKey;
+        this.signingKey = signingKey;
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/security/repository/JwtSigningKeyRepository.java

@@ -0,0 +1,26 @@
+package stirling.software.proprietary.security.repository;
+
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Optional;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+import stirling.software.proprietary.security.model.JwtSigningKeyEntity;
+
+/** Shared-DB store of JWT signing keys - the source of truth every cluster node reads from. */
+@Repository
+public interface JwtSigningKeyRepository extends JpaRepository<JwtSigningKeyEntity, String> {
+
+    /** Newest first, so the most recently created key is the active signing key. */
+    List<JwtSigningKeyEntity> findAllByOrderByCreatedAtDesc();
+
+    /**
+     * The current active signing key: the single newest row. Used for cheap cluster convergence.
+     */
+    Optional<JwtSigningKeyEntity> findFirstByOrderByCreatedAtDesc();
+
+    /** Keys created before the cutoff, eligible for rotation cleanup. */
+    List<JwtSigningKeyEntity> findByCreatedAtBefore(LocalDateTime cutoff);
+}

app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPairCleanupService.java

@@ -3,8 +3,10 @@ package stirling.software.proprietary.security.service;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.time.LocalDateTime;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,6 +19,8 @@ import jakarta.annotation.PostConstruct;
 
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.DistributedLock.LockHandle;
 import stirling.software.common.configuration.InstallationPathConfig;
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.proprietary.security.model.JwtVerificationKey;
@@ -26,15 +30,23 @@ import stirling.software.proprietary.security.model.JwtVerificationKey;
 @ConditionalOnBooleanProperty("v2")
 public class KeyPairCleanupService {
 
+    // Cluster-wide single-writer: keys live in the shared DB, so only one node may prune + rotate
+    // per cycle. Otherwise every node runs this and they race to delete each other's keys.
+    private static final String CLEANUP_LOCK = "jwt-key-cleanup";
+    private static final Duration LOCK_LEASE = Duration.ofMinutes(5);
+
     private final KeyPersistenceService keyPersistenceService;
     private final ApplicationProperties.Security.Jwt jwtProperties;
+    private final DistributedLock distributedLock;
 
     @Autowired
     public KeyPairCleanupService(
             KeyPersistenceService keyPersistenceService,
-            ApplicationProperties applicationProperties) {
+            ApplicationProperties applicationProperties,
+            DistributedLock distributedLock) {
         this.keyPersistenceService = keyPersistenceService;
         this.jwtProperties = applicationProperties.getSecurity().getJwt();
+        this.distributedLock = distributedLock;
     }
 
     @Transactional
@@ -44,7 +56,28 @@ public class KeyPairCleanupService {
         if (!jwtProperties.isEnableKeyCleanup() || !keyPersistenceService.isKeystoreEnabled()) {
             return;
         }
+        // A lock-backend error must never fail this @PostConstruct/scheduled run: degrade to
+        // "skip this cycle" so a transient Valkey blip can't stop a node from booting.
+        Optional<LockHandle> lock;
+        try {
+            lock = distributedLock.tryAcquire(CLEANUP_LOCK, LOCK_LEASE);
+        } catch (RuntimeException e) {
+            log.warn(
+                    "Could not acquire the JWT key-cleanup lock ({}); skipping this cycle",
+                    e.getMessage());
+            return;
+        }
+        // No lock means another node is already pruning; skip until the next tick.
+        if (lock.isEmpty()) {
+            log.debug("Another node holds the JWT key-cleanup lock; skipping this cycle");
+            return;
+        }
+        try (LockHandle held = lock.get()) {
+            runCleanup();
+        }
+    }
 
+    private void runCleanup() {
         LocalDateTime cutoffDate =
                 LocalDateTime.now().minusDays(jwtProperties.getKeyRetentionDays());
 

app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPersistenceService.java

@@ -18,15 +18,17 @@ import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Base64;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cache.Cache;
 import org.springframework.cache.CacheManager;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.cache.caffeine.CaffeineCache;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 
 import jakarta.annotation.PostConstruct;
 
@@ -34,27 +36,39 @@ import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.common.configuration.InstallationPathConfig;
 import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.security.model.JwtSigningKeyEntity;
 import stirling.software.proprietary.security.model.JwtVerificationKey;
+import stirling.software.proprietary.security.repository.JwtSigningKeyRepository;
 
+/** Persists JWT signing keys in the shared DB so all nodes sign/verify with the same key. */
 @Slf4j
 @Service
+// CredentialEncryption must init first: startup persists an encrypted private key.
+@DependsOn("credentialEncryption")
 public class KeyPersistenceService implements KeyPersistenceServiceInterface {
 
     public static final String KEY_SUFFIX = ".key";
     public static final String PUB_KEY_SUFFIX = ".pub";
 
     private final ApplicationProperties.Security.Jwt jwtProperties;
-    private final CacheManager cacheManager;
     private final Cache verifyingKeyCache;
+    private final JwtSigningKeyRepository keyRepository;
+    private final boolean clusterEnabled;
+
+    // kid -> KeyPair; safe to cache since key material is immutable.
+    private final Map<String, KeyPair> keyPairCache = new ConcurrentHashMap<>();
 
     private volatile JwtVerificationKey activeKey;
 
-    @Autowired
     public KeyPersistenceService(
-            ApplicationProperties applicationProperties, CacheManager cacheManager) {
+            ApplicationProperties applicationProperties,
+            CacheManager cacheManager,
+            JwtSigningKeyRepository keyRepository,
+            @Value("${cluster.enabled:false}") boolean clusterEnabled) {
         this.jwtProperties = applicationProperties.getSecurity().getJwt();
-        this.cacheManager = cacheManager;
         this.verifyingKeyCache = cacheManager.getCache("verifyingKeys");
+        this.keyRepository = keyRepository;
+        this.clusterEnabled = clusterEnabled;
     }
 
     @PostConstruct
@@ -63,138 +77,86 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
             log.info("JWT keystore is disabled - keys will be generated in memory");
             return;
         }
-
         try {
-            ensurePrivateKeyDirectoryExists();
-            loadExistingKeysFromDisk();
+            importLegacyDiskKeysIfPresent();
+            loadKeysFromDb();
         } catch (Exception e) {
-            log.error("Failed to initialize keystore, using in-memory generation", e);
-        }
-    }
-
-    /**
-     * Load all existing JWT keys from disk into memory on startup.
-     *
-     * <p>This ensures tokens signed with previous keys remain valid after server restart. If no
-     * keys exist on disk, generates a new keypair.
-     */
-    private void loadExistingKeysFromDisk() {
-        try {
-            Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath());
-
-            if (!Files.exists(keyDirectory)) {
-                log.info("No existing keys found, generating new keypair");
-                generateAndStoreKeypair();
-                return;
-            }
-
-            List<Path> keyFiles;
-            try (var stream = Files.list(keyDirectory)) {
-                keyFiles =
-                        stream.filter(path -> path.toString().endsWith(KEY_SUFFIX))
-                                .sorted(
-                                        (a, b) ->
-                                                b.getFileName().compareTo(a.getFileName())) // Most
-                                // recent
-                                // first
-                                .toList();
-            }
-
-            if (keyFiles.isEmpty()) {
-                log.info("No existing keys found in directory, generating new keypair");
-                generateAndStoreKeypair();
-                return;
-            }
-
-            log.info("Loading {} existing JWT keys from disk", keyFiles.size());
-            int loadedCount = 0;
-
-            for (Path keyFile : keyFiles) {
-                try {
-                    String keyId = keyFile.getFileName().toString().replace(KEY_SUFFIX, "");
-
-                    // Load private key first
-                    PrivateKey privateKey = loadPrivateKey(keyId);
-
-                    // Try to load public key, or generate it from private key if missing
-                    // (migration)
-                    String encodedPublicKey;
-                    try {
-                        encodedPublicKey = loadPublicKey(keyId);
-                    } catch (IOException e) {
-                        // Public key file doesn't exist - generate it from private key (migration)
-                        log.info("Migrating legacy key: generating public key file for {}", keyId);
-                        KeyPair keyPair = reconstructKeyPair(privateKey);
-
-                        // Save the public key file
-                        Path publicKeyFile = keyDirectory.resolve(keyId + PUB_KEY_SUFFIX);
-                        encodedPublicKey = encodePublicKey(keyPair.getPublic());
-                        Files.writeString(publicKeyFile, encodedPublicKey);
-                        publicKeyFile.toFile().setReadable(true, true);
-                        publicKeyFile.toFile().setWritable(true, true);
-                        publicKeyFile.toFile().setExecutable(false, false);
-
-                        log.info("Successfully migrated key: {}", keyId);
-                    }
-
-                    // Create verification key and add to cache
-                    JwtVerificationKey verifyingKey =
-                            new JwtVerificationKey(keyId, encodedPublicKey);
-                    verifyingKeyCache.put(keyId, verifyingKey);
-                    loadedCount++;
-
-                    // Set the most recent key as active (first in sorted list)
-                    if (activeKey == null) {
-                        activeKey = verifyingKey;
-                        log.info("Set active JWT signing key: {}", keyId);
-                    } else {
-                        log.debug(
-                                "Loaded historical JWT key: {} (created: {})",
-                                keyId,
-                                verifyingKey.getCreatedAt());
-                    }
-                } catch (Exception e) {
-                    log.warn(
-                            "Failed to load key: {}, skipping. Error: {}",
-                            keyFile.getFileName(),
-                            e.getMessage());
-                }
-            }
-
-            if (loadedCount == 0) {
-                log.warn("No valid keys could be loaded from disk, generating new keypair");
-                generateAndStoreKeypair();
-            } else {
-                log.info(
-                        "Successfully loaded {} JWT keys, active key: {}",
-                        loadedCount,
-                        activeKey.getKeyId());
-            }
-
-        } catch (IOException e) {
-            log.error("Failed to load keys from disk, generating new keypair", e);
+            log.error("Failed to initialize keystore, generating a fresh keypair", e);
             generateAndStoreKeypair();
         }
     }
 
-    @Transactional
-    private JwtVerificationKey generateAndStoreKeypair() {
-        JwtVerificationKey verifyingKey = null;
+    /**
+     * Cluster convergence: adopt the newest signing key in the shared DB as this node's active key.
+     * Runs on every node so a key a peer just minted becomes the shared active signer within one
+     * interval, keeping cluster rotation equivalent to single-node. Cluster-only: a single node
+     * always holds its own newest key, so this is skipped entirely off-cluster.
+     */
+    @Scheduled(fixedDelayString = "${stirling.security.jwt.activeKeyReloadMs:300000}")
+    public void reloadActiveKeyFromDb() {
+        if (!clusterEnabled || !isKeystoreEnabled()) {
+            return;
+        }
+        try {
+            Optional<JwtSigningKeyEntity> newestOpt =
+                    keyRepository.findFirstByOrderByCreatedAtDesc();
+            if (newestOpt.isEmpty()) {
+                return;
+            }
+            JwtSigningKeyEntity newest = newestOpt.get();
+            JwtVerificationKey current = activeKey;
+            if (current != null && newest.getKeyId().equals(current.getKeyId())) {
+                return;
+            }
+            JwtVerificationKey adopted =
+                    new JwtVerificationKey(newest.getKeyId(), newest.getVerifyingKey());
+            verifyingKeyCache.put(newest.getKeyId(), adopted);
+            activeKey = adopted;
+            log.info(
+                    "Adopted newest JWT signing key {} from the shared DB as active",
+                    newest.getKeyId());
+        } catch (Exception e) {
+            log.warn("Could not reload active JWT key from the shared DB: {}", e.getMessage());
+        }
+    }
 
+    /** Load every signing key from the shared DB into the caches; most recent becomes active. */
+    private void loadKeysFromDb() {
+        List<JwtSigningKeyEntity> keys = keyRepository.findAllByOrderByCreatedAtDesc();
+        if (keys.isEmpty()) {
+            log.info("No JWT keys in the database, generating a new keypair");
+            generateAndStoreKeypair();
+            return;
+        }
+        for (JwtSigningKeyEntity key : keys) {
+            verifyingKeyCache.put(
+                    key.getKeyId(), new JwtVerificationKey(key.getKeyId(), key.getVerifyingKey()));
+        }
+        activeKey = new JwtVerificationKey(keys.get(0).getKeyId(), keys.get(0).getVerifyingKey());
+        log.info("Loaded {} JWT key(s) from DB, active key: {}", keys.size(), activeKey.getKeyId());
+    }
+
+    private JwtVerificationKey generateAndStoreKeypair() {
         try {
             KeyPair keyPair = generateRSAKeypair();
             String keyId = generateKeyId();
+            String verifyingKey = encodePublicKey(keyPair.getPublic());
+            String signingKey =
+                    Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded());
 
-            storeKeyPair(keyId, keyPair);
-            verifyingKey = new JwtVerificationKey(keyId, encodePublicKey(keyPair.getPublic()));
-            verifyingKeyCache.put(keyId, verifyingKey);
-            activeKey = verifyingKey;
+            // Converter encrypts the private key at rest with the shared credential-encryption key.
+            keyRepository.save(new JwtSigningKeyEntity(keyId, verifyingKey, signingKey));
+
+            keyPairCache.put(keyId, keyPair);
+            JwtVerificationKey verificationKey = new JwtVerificationKey(keyId, verifyingKey);
+            verifyingKeyCache.put(keyId, verificationKey);
+            activeKey = verificationKey;
             log.info("Generated and stored new JWT keypair: {}", keyId);
-        } catch (IOException e) {
+            return verificationKey;
+        } catch (RuntimeException e) {
             log.error("Failed to generate and store keypair", e);
+            return null;
         }
-
-        return verifyingKey;
     }
 
     @Override
@@ -207,25 +169,29 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
 
     @Override
     public Optional<KeyPair> getKeyPair(String keyId) {
-        if (!isKeystoreEnabled()) {
+        if (!isKeystoreEnabled() || keyId == null) {
             return Optional.empty();
         }
-
+        KeyPair cached = keyPairCache.get(keyId);
+        if (cached != null) {
+            return Optional.of(cached);
+        }
+        Optional<JwtSigningKeyEntity> entityOpt = keyRepository.findById(keyId);
+        if (entityOpt.isEmpty()) {
+            log.warn("No signing key found in DB for keyId: {}", keyId);
+            return Optional.empty();
+        }
+        JwtSigningKeyEntity entity = entityOpt.get();
         try {
-            JwtVerificationKey verifyingKey =
-                    verifyingKeyCache.get(keyId, JwtVerificationKey.class);
-
-            if (verifyingKey == null) {
-                log.warn("No signing key found in database for keyId: {}", keyId);
-                return Optional.empty();
-            }
-
-            PrivateKey privateKey = loadPrivateKey(keyId);
-            PublicKey publicKey = decodePublicKey(verifyingKey.getVerifyingKey());
-
-            return Optional.of(new KeyPair(publicKey, privateKey));
-        } catch (Exception e) {
-            log.error("Failed to load keypair for keyId: {}", keyId, e);
+            KeyPair keyPair =
+                    new KeyPair(
+                            decodePublicKey(entity.getVerifyingKey()),
+                            decodePrivateKey(entity.getSigningKey()));
+            keyPairCache.put(keyId, keyPair);
+            verifyingKeyCache.put(keyId, new JwtVerificationKey(keyId, entity.getVerifyingKey()));
+            return Optional.of(keyPair);
+        } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
+            log.error("Failed to decode keypair for keyId: {}", keyId, e);
             return Optional.empty();
         }
     }
@@ -241,44 +207,69 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
     }
 
     @Override
-    @CacheEvict(
-            value = {"verifyingKeys"},
-            key = "#keyId",
-            condition = "#root.target.isKeystoreEnabled()")
     public void removeKey(String keyId) {
+        keyRepository.deleteById(keyId);
         verifyingKeyCache.evict(keyId);
+        keyPairCache.remove(keyId);
     }
 
     @Override
     public List<JwtVerificationKey> getKeysEligibleForCleanup(LocalDateTime cutoffDate) {
-        CaffeineCache caffeineCache = (CaffeineCache) verifyingKeyCache;
-        com.github.benmanes.caffeine.cache.Cache<Object, Object> nativeCache =
-                caffeineCache.getNativeCache();
-
-        log.debug(
-                "Cache size: {}, Checking {} keys for cleanup",
-                nativeCache.estimatedSize(),
-                nativeCache.asMap().size());
-
-        return nativeCache.asMap().values().stream()
-                .filter(value -> value instanceof JwtVerificationKey)
-                .map(value -> (JwtVerificationKey) value)
-                .filter(
-                        key -> {
-                            boolean eligible = key.getCreatedAt().isBefore(cutoffDate);
-                            log.debug(
-                                    "Key {} created at {}, eligible for cleanup: {}",
-                                    key.getKeyId(),
-                                    key.getCreatedAt(),
-                                    eligible);
-                            return eligible;
-                        })
+        return keyRepository.findByCreatedAtBefore(cutoffDate).stream()
+                .map(e -> new JwtVerificationKey(e.getKeyId(), e.getVerifyingKey()))
                 .toList();
     }
 
+    /** Import any pre-existing on-disk keys into the DB once, so upgrades keep sessions valid. */
+    private void importLegacyDiskKeysIfPresent() {
+        if (keyRepository.count() > 0) {
+            return;
+        }
+        Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath());
+        if (!Files.exists(keyDirectory)) {
+            return;
+        }
+        List<Path> keyFiles;
+        try (var stream = Files.list(keyDirectory)) {
+            keyFiles = stream.filter(p -> p.toString().endsWith(KEY_SUFFIX)).toList();
+        } catch (IOException e) {
+            log.warn("Could not list legacy key directory {}: {}", keyDirectory, e.getMessage());
+            return;
+        }
+        int imported = 0;
+        for (Path keyFile : keyFiles) {
+            String keyId = keyFile.getFileName().toString().replace(KEY_SUFFIX, "");
+            try {
+                PrivateKey privateKey = loadPrivateKey(keyId);
+                String verifyingKey = resolveLegacyPublicKey(keyId, privateKey);
+                String signingKey = Base64.getEncoder().encodeToString(privateKey.getEncoded());
+                keyRepository.save(new JwtSigningKeyEntity(keyId, verifyingKey, signingKey));
+                imported++;
+            } catch (Exception e) {
+                log.warn("Skipping legacy key {}: {}", keyId, e.getMessage());
+            }
+        }
+        if (imported > 0) {
+            log.info("Imported {} legacy JWT key(s) from disk into the shared DB", imported);
+        }
+    }
+
+    private String resolveLegacyPublicKey(String keyId, PrivateKey privateKey)
+            throws NoSuchAlgorithmException, InvalidKeySpecException {
+        try {
+            return loadPublicKey(keyId);
+        } catch (IOException e) {
+            // No .pub file: derive the public key from the RSA private key.
+            return encodePublicKey(reconstructKeyPair(privateKey).getPublic());
+        }
+    }
+
+    // UUID suffix so two nodes booting the same second don't collide on keyId.
     private String generateKeyId() {
         return "jwt-key-"
-                + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmmss"));
+                + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmmss"))
+                + "-"
+                + UUID.randomUUID().toString().substring(0, 8);
     }
 
     private KeyPair generateRSAKeypair() {
@@ -291,125 +282,49 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface {
         }
     }
 
-    private void ensurePrivateKeyDirectoryExists() throws IOException {
-        Path keyPath = Path.of(InstallationPathConfig.getPrivateKeyPath());
-
-        if (!Files.exists(keyPath)) {
-            Files.createDirectories(keyPath);
-        }
-    }
-
-    /**
-     * Store both private and public keys to disk.
-     *
-     * <p>Private key stored as: keyId.key
-     *
-     * <p>Public key stored as: keyId.pub
-     */
-    private void storeKeyPair(String keyId, KeyPair keyPair) throws IOException {
-        Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath());
-
-        // Store private key
-        Path privateKeyFile = keyDirectory.resolve(keyId + KEY_SUFFIX);
-        String encodedPrivateKey =
-                Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded());
-        Files.writeString(privateKeyFile, encodedPrivateKey);
-
-        // Set read/write to only the owner (security)
-        privateKeyFile.toFile().setReadable(true, true);
-        privateKeyFile.toFile().setWritable(true, true);
-        privateKeyFile.toFile().setExecutable(false, false);
-
-        // Store public key
-        Path publicKeyFile = keyDirectory.resolve(keyId + PUB_KEY_SUFFIX);
-        String encodedPublicKey =
-                Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded());
-        Files.writeString(publicKeyFile, encodedPublicKey);
-
-        // Public key can be more permissive but still restrict to owner
-        publicKeyFile.toFile().setReadable(true, true);
-        publicKeyFile.toFile().setWritable(true, true);
-        publicKeyFile.toFile().setExecutable(false, false);
-
-        log.debug(
-                "Stored keypair to disk: {} (private: {}, public: {})",
-                keyId,
-                privateKeyFile.getFileName(),
-                publicKeyFile.getFileName());
-    }
-
     private PrivateKey loadPrivateKey(String keyId)
             throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
         Path keyFile =
                 Path.of(InstallationPathConfig.getPrivateKeyPath()).resolve(keyId + KEY_SUFFIX);
-
         if (!Files.exists(keyFile)) {
             throw new IOException("Private key not found: " + keyFile);
         }
-
-        String encodedKey = Files.readString(keyFile);
-        byte[] keyBytes = Base64.getDecoder().decode(encodedKey);
-        PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes);
-        KeyFactory keyFactory = KeyFactory.getInstance("RSA");
-
-        return keyFactory.generatePrivate(keySpec);
+        return decodePrivateKey(Files.readString(keyFile));
     }
 
-    /**
-     * Load public key from disk.
-     *
-     * @param keyId the key identifier
-     * @return Base64-encoded public key string
-     * @throws IOException if the public key file is not found
-     */
     private String loadPublicKey(String keyId) throws IOException {
         Path publicKeyFile =
                 Path.of(InstallationPathConfig.getPrivateKeyPath()).resolve(keyId + PUB_KEY_SUFFIX);
-
         if (!Files.exists(publicKeyFile)) {
             throw new IOException("Public key not found: " + publicKeyFile);
         }
-
         return Files.readString(publicKeyFile).trim();
     }
 
-    /**
-     * Reconstruct a KeyPair from a PrivateKey.
-     *
-     * <p>For RSA keys, derives the public key from the private key.
-     *
-     * @param privateKey the RSA private key
-     * @return reconstructed KeyPair
-     * @throws NoSuchAlgorithmException if RSA algorithm is not available
-     * @throws InvalidKeySpecException if the key specification is invalid
-     */
     private KeyPair reconstructKeyPair(PrivateKey privateKey)
             throws NoSuchAlgorithmException, InvalidKeySpecException {
-        // For RSA, we can derive the public key from the private key
         KeyFactory keyFactory = KeyFactory.getInstance("RSA");
-
-        // Get the private key spec
         RSAPrivateCrtKey rsaPrivateKey = (RSAPrivateCrtKey) privateKey;
-
-        // Create public key spec from private key parameters
         RSAPublicKeySpec publicKeySpec =
                 new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPublicExponent());
-
-        // Generate public key
-        PublicKey publicKey = keyFactory.generatePublic(publicKeySpec);
-
-        return new KeyPair(publicKey, privateKey);
+        return new KeyPair(keyFactory.generatePublic(publicKeySpec), privateKey);
     }
 
     private String encodePublicKey(PublicKey publicKey) {
         return Base64.getEncoder().encodeToString(publicKey.getEncoded());
     }
 
+    @Override
     public PublicKey decodePublicKey(String encodedKey)
             throws NoSuchAlgorithmException, InvalidKeySpecException {
-        byte[] keyBytes = Base64.getDecoder().decode(encodedKey);
-        X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes);
-        KeyFactory keyFactory = KeyFactory.getInstance("RSA");
-        return keyFactory.generatePublic(keySpec);
+        X509EncodedKeySpec keySpec = new X509EncodedKeySpec(Base64.getDecoder().decode(encodedKey));
+        return KeyFactory.getInstance("RSA").generatePublic(keySpec);
+    }
+
+    private PrivateKey decodePrivateKey(String encodedKey)
+            throws NoSuchAlgorithmException, InvalidKeySpecException {
+        PKCS8EncodedKeySpec keySpec =
+                new PKCS8EncodedKeySpec(Base64.getDecoder().decode(encodedKey));
+        return KeyFactory.getInstance("RSA").generatePrivate(keySpec);
     }
 }

app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java

@@ -27,6 +27,8 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.server.ResponseStatusException;
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.inprocess.InProcessJobStore;
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.model.job.JobResponse;
 import stirling.software.common.service.JobOwnershipService;
@@ -78,6 +80,7 @@ class PolicyControllerTest {
     @Mock private JobOwnershipService jobOwnershipService;
 
     private ApplicationProperties applicationProperties;
+    private final JobStore jobStore = new InProcessJobStore();
     private PolicyController controller;
 
     private final java.util.List<stirling.software.proprietary.policy.trigger.PolicyTrigger>
@@ -106,7 +109,8 @@ class PolicyControllerTest {
                         policyTriggers,
                         applicationProperties,
                         tempFileManager,
-                        jobOwnershipService);
+                        jobOwnershipService,
+                        jobStore);
     }
 
     private static stirling.software.proprietary.policy.trigger.PolicyTrigger trigger(
@@ -264,6 +268,8 @@ class PolicyControllerTest {
         @DisplayName("returns 404 when run is unknown")
         void notFound() {
             when(runRegistry.get("missing")).thenReturn(null);
+            when(jobOwnershipService.extractJobId("missing")).thenReturn("missing");
+            when(jobOwnershipService.createScopedJobKey("missing")).thenReturn("missing");
 
             ResponseEntity<PolicyRunView> response = controller.status("missing");
 

app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPairCleanupServiceTest.java

@@ -0,0 +1,102 @@
+package stirling.software.proprietary.security.service;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.DistributedLock.LockHandle;
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.security.model.JwtVerificationKey;
+
+/** Cluster safety: pruning JWT keys is single-writer, gated on the shared cleanup lock. */
+@ExtendWith(MockitoExtension.class)
+class KeyPairCleanupServiceTest {
+
+    @Mock private KeyPersistenceService keyPersistenceService;
+    @Mock private ApplicationProperties applicationProperties;
+    @Mock private ApplicationProperties.Security security;
+    @Mock private ApplicationProperties.Security.Jwt jwtProperties;
+    @Mock private DistributedLock distributedLock;
+    @Mock private LockHandle lockHandle;
+
+    private KeyPairCleanupService cleanupService;
+
+    @BeforeEach
+    void setUp() {
+        lenient().when(applicationProperties.getSecurity()).thenReturn(security);
+        lenient().when(security.getJwt()).thenReturn(jwtProperties);
+        lenient().when(jwtProperties.isEnableKeyCleanup()).thenReturn(true);
+        lenient().when(keyPersistenceService.isKeystoreEnabled()).thenReturn(true);
+        cleanupService =
+                new KeyPairCleanupService(
+                        keyPersistenceService, applicationProperties, distributedLock);
+    }
+
+    @Test
+    void skipsPruningWhenAnotherNodeHoldsTheLock() {
+        when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.empty());
+
+        cleanupService.cleanup();
+
+        // No node-local pruning happened; the lock holder owns this cycle.
+        verify(keyPersistenceService, never()).getKeysEligibleForCleanup(any());
+        verify(keyPersistenceService, never()).refreshActiveKeyPair();
+    }
+
+    @Test
+    void prunesAndRotatesWhenLockAcquiredThenReleasesIt() {
+        when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.of(lockHandle));
+        when(keyPersistenceService.getKeysEligibleForCleanup(any()))
+                .thenReturn(List.of(new JwtVerificationKey("old-key", "cHVi")));
+
+        cleanupService.cleanup();
+
+        verify(keyPersistenceService).removeKey("old-key");
+        verify(keyPersistenceService).refreshActiveKeyPair();
+        verify(lockHandle).close();
+    }
+
+    @Test
+    void releasesTheLockEvenWhenNoKeysAreEligible() {
+        when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.of(lockHandle));
+        when(keyPersistenceService.getKeysEligibleForCleanup(any())).thenReturn(List.of());
+
+        cleanupService.cleanup();
+
+        verify(keyPersistenceService, never()).refreshActiveKeyPair();
+        verify(lockHandle).close();
+    }
+
+    @Test
+    void skipsPruningWhenTheLockBackendErrors() {
+        // A Valkey blip at boot must not fail startup: tryAcquire throwing degrades to skip.
+        when(distributedLock.tryAcquire(any(), any()))
+                .thenThrow(new RuntimeException("valkey unreachable"));
+
+        cleanupService.cleanup();
+
+        verify(keyPersistenceService, never()).getKeysEligibleForCleanup(any());
+        verify(keyPersistenceService, never()).refreshActiveKeyPair();
+    }
+
+    @Test
+    void doesNothingWhenCleanupDisabled() {
+        when(jwtProperties.isEnableKeyCleanup()).thenReturn(false);
+
+        cleanupService.cleanup();
+
+        verify(distributedLock, never()).tryAcquire(any(), any());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPersistenceServiceInterfaceTest.java

@@ -4,45 +4,41 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.lenient;
-import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.NoSuchAlgorithmException;
 import java.util.Base64;
+import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.Mock;
-import org.mockito.MockedStatic;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.cache.CacheManager;
 import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
 
-import stirling.software.common.configuration.InstallationPathConfig;
 import stirling.software.common.model.ApplicationProperties;
+import stirling.software.proprietary.security.model.JwtSigningKeyEntity;
 import stirling.software.proprietary.security.model.JwtVerificationKey;
+import stirling.software.proprietary.security.repository.JwtSigningKeyRepository;
 
+/** DB-backed keystore: a key present only in the shared DB still resolves (the cross-node case). */
 @ExtendWith(MockitoExtension.class)
 class KeyPersistenceServiceInterfaceTest {
 
     @Mock private ApplicationProperties applicationProperties;
-
     @Mock private ApplicationProperties.Security security;
-
     @Mock private ApplicationProperties.Security.Jwt jwtConfig;
-
-    @TempDir Path tempDir;
+    @Mock private JwtSigningKeyRepository keyRepository;
 
     private KeyPersistenceService keyPersistenceService;
     private KeyPair testKeyPair;
@@ -58,175 +54,128 @@ class KeyPersistenceServiceInterfaceTest {
 
         lenient().when(applicationProperties.getSecurity()).thenReturn(security);
         lenient().when(security.getJwt()).thenReturn(jwtConfig);
-        lenient().when(jwtConfig.isEnableKeystore()).thenReturn(true); // Default value
+        lenient().when(jwtConfig.isEnableKeystore()).thenReturn(true);
+        lenient().when(keyRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
+        // clusterEnabled=true so the convergence-reload path is exercised.
+        keyPersistenceService =
+                new KeyPersistenceService(applicationProperties, cacheManager, keyRepository, true);
+    }
+
+    private JwtSigningKeyEntity entityFrom(String keyId) {
+        return new JwtSigningKeyEntity(
+                keyId,
+                Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded()),
+                Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded()));
     }
 
     @ParameterizedTest
     @ValueSource(booleans = {true, false})
     void testKeystoreEnabled(boolean keystoreEnabled) {
         when(jwtConfig.isEnableKeystore()).thenReturn(keystoreEnabled);
-
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-
-            assertEquals(keystoreEnabled, keyPersistenceService.isKeystoreEnabled());
-        }
+        assertEquals(keystoreEnabled, keyPersistenceService.isKeystoreEnabled());
     }
 
     @Test
-    void testGetActiveKeypairWhenNoActiveKeyExists() {
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-            keyPersistenceService.initializeKeystore();
+    void generatesAndPersistsAKeyWhenNoneIsActive() {
+        // getActiveKey with no active key mints one and persists it - no disk involved.
+        JwtVerificationKey active = keyPersistenceService.getActiveKey();
 
-            JwtVerificationKey result = keyPersistenceService.getActiveKey();
-
-            assertNotNull(result);
-            assertNotNull(result.getKeyId());
-            assertNotNull(result.getVerifyingKey());
-        }
+        assertNotNull(active);
+        assertNotNull(active.getKeyId());
+        assertNotNull(active.getVerifyingKey());
+        verify(keyRepository).save(any(JwtSigningKeyEntity.class));
     }
 
     @Test
-    void testGetActiveKeyPairWithExistingKey() throws Exception {
-        String keyId = "test-key-2024-01-01-120000";
-        String publicKeyBase64 =
-                Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded());
-        String privateKeyBase64 =
-                Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded());
+    void loadsTheMostRecentExistingKeyAsActive() {
+        when(keyRepository.count()).thenReturn(1L);
+        when(keyRepository.findAllByOrderByCreatedAtDesc())
+                .thenReturn(List.of(entityFrom("jwt-key-2026-07-13-000000-abcd1234")));
 
-        JwtVerificationKey existingKey = new JwtVerificationKey(keyId, publicKeyBase64);
+        keyPersistenceService.initializeKeystore();
+        JwtVerificationKey active = keyPersistenceService.getActiveKey();
 
-        Path keyFile = tempDir.resolve(keyId + ".key");
-        Files.writeString(keyFile, privateKeyBase64);
-
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-            keyPersistenceService.initializeKeystore();
-
-            JwtVerificationKey result = keyPersistenceService.getActiveKey();
-
-            assertNotNull(result);
-            assertNotNull(result.getKeyId());
-        }
+        assertEquals("jwt-key-2026-07-13-000000-abcd1234", active.getKeyId());
     }
 
     @Test
-    void testGetKeyPair() throws Exception {
-        String keyId = "test-key-123";
-        String publicKeyBase64 =
-                Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded());
-        String privateKeyBase64 =
-                Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded());
+    void getKeyPairResolvesAKeyPresentOnlyInTheSharedDb() {
+        // Never initialised locally: the key lives only in the DB, as if another node minted it.
+        String keyId = "jwt-key-from-another-node";
+        when(keyRepository.findById(keyId)).thenReturn(Optional.of(entityFrom(keyId)));
 
-        JwtVerificationKey signingKey = new JwtVerificationKey(keyId, publicKeyBase64);
+        Optional<KeyPair> result = keyPersistenceService.getKeyPair(keyId);
 
-        Path keyFile = tempDir.resolve(keyId + ".key");
-        Files.writeString(keyFile, privateKeyBase64);
-
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-
-            keyPersistenceService
-                    .getClass()
-                    .getDeclaredField("verifyingKeyCache")
-                    .setAccessible(true);
-            var cache = cacheManager.getCache("verifyingKeys");
-            cache.put(keyId, signingKey);
-
-            Optional<KeyPair> result = keyPersistenceService.getKeyPair(keyId);
-
-            assertTrue(result.isPresent());
-            assertNotNull(result.get().getPublic());
-            assertNotNull(result.get().getPrivate());
-        }
+        assertTrue(result.isPresent());
+        assertNotNull(result.get().getPublic());
+        assertNotNull(result.get().getPrivate());
     }
 
     @Test
-    void testGetKeyPairNotFound() {
-        String keyId = "non-existent-key";
-
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-
-            Optional<KeyPair> result = keyPersistenceService.getKeyPair(keyId);
-
-            assertFalse(result.isPresent());
-        }
+    void getKeyPairIsEmptyWhenTheKeyIsUnknown() {
+        when(keyRepository.findById("nope")).thenReturn(Optional.empty());
+        assertFalse(keyPersistenceService.getKeyPair("nope").isPresent());
     }
 
     @Test
-    void testGetKeyPairWhenKeystoreDisabled() {
+    void getKeyPairIsEmptyWhenKeystoreDisabled() {
         when(jwtConfig.isEnableKeystore()).thenReturn(false);
-
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-
-            Optional<KeyPair> result = keyPersistenceService.getKeyPair("any-key");
-
-            assertFalse(result.isPresent());
-        }
+        assertFalse(keyPersistenceService.getKeyPair("any-key").isPresent());
     }
 
     @Test
-    void testInitializeKeystoreCreatesDirectory() throws IOException {
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-            keyPersistenceService.initializeKeystore();
+    void eligibleForCleanupIsSourcedFromTheDb() {
+        when(keyRepository.findByCreatedAtBefore(any())).thenReturn(List.of(entityFrom("old-key")));
 
-            assertTrue(Files.exists(tempDir));
-            assertTrue(Files.isDirectory(tempDir));
-        }
+        List<JwtVerificationKey> stale =
+                keyPersistenceService.getKeysEligibleForCleanup(java.time.LocalDateTime.now());
+
+        assertEquals(1, stale.size());
+        assertEquals("old-key", stale.get(0).getKeyId());
     }
 
     @Test
-    void testLoadExistingKeypairWithMissingPrivateKeyFile() throws Exception {
-        String keyId = "test-key-missing-file";
-        String publicKeyBase64 =
-                Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded());
+    void reloadAdoptsTheNewestKeyAPeerMinted() {
+        // Boot with our own key active, then a peer mints a newer one in the shared DB.
+        when(keyRepository.count()).thenReturn(1L);
+        when(keyRepository.findAllByOrderByCreatedAtDesc())
+                .thenReturn(List.of(entityFrom("jwt-key-local-old")));
+        keyPersistenceService.initializeKeystore();
+        assertEquals("jwt-key-local-old", keyPersistenceService.getActiveKey().getKeyId());
 
-        JwtVerificationKey existingKey = new JwtVerificationKey(keyId, publicKeyBase64);
+        when(keyRepository.findFirstByOrderByCreatedAtDesc())
+                .thenReturn(Optional.of(entityFrom("jwt-key-peer-new")));
 
-        try (MockedStatic<InstallationPathConfig> mockedStatic =
-                mockStatic(InstallationPathConfig.class)) {
-            mockedStatic
-                    .when(InstallationPathConfig::getPrivateKeyPath)
-                    .thenReturn(tempDir.toString());
-            keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager);
-            keyPersistenceService.initializeKeystore();
+        keyPersistenceService.reloadActiveKeyFromDb();
 
-            JwtVerificationKey result = keyPersistenceService.getActiveKey();
-            assertNotNull(result);
-            assertNotNull(result.getKeyId());
-            assertNotNull(result.getVerifyingKey());
-        }
+        // Converged: this node now signs with the peer's newer key.
+        assertEquals("jwt-key-peer-new", keyPersistenceService.getActiveKey().getKeyId());
+    }
+
+    @Test
+    void reloadDoesNothingOffCluster() {
+        KeyPersistenceService singleNode =
+                new KeyPersistenceService(
+                        applicationProperties, cacheManager, keyRepository, false);
+
+        singleNode.reloadActiveKeyFromDb();
+
+        // Off-cluster the DB is never consulted for convergence.
+        verify(keyRepository, org.mockito.Mockito.never()).findFirstByOrderByCreatedAtDesc();
+    }
+
+    @Test
+    void reloadIsANoOpWhenAlreadyHoldingTheNewestKey() {
+        when(keyRepository.count()).thenReturn(1L);
+        when(keyRepository.findAllByOrderByCreatedAtDesc())
+                .thenReturn(List.of(entityFrom("jwt-key-current")));
+        keyPersistenceService.initializeKeystore();
+
+        when(keyRepository.findFirstByOrderByCreatedAtDesc())
+                .thenReturn(Optional.of(entityFrom("jwt-key-current")));
+
+        keyPersistenceService.reloadActiveKeyFromDb();
+
+        assertEquals("jwt-key-current", keyPersistenceService.getActiveKey().getKeyId());
     }
 }