From 79dc7d5615875bd8381134f716f5060e047bf4ef Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:51:30 +0100 Subject: [PATCH] Multi node cluster fixes (#7025) - Exclude DataRedisRepositoriesAutoConfiguration (cluster crash-loop fix) - Share JWT signing keys via the DB + require a shared credential key in cluster mode - Make policy run status/listing visible across nodes --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details. --- .../software/common/model/job/JobResult.java | 17 + .../software/common/service/TaskManager.java | 14 +- .../src/main/resources/application.properties | 3 +- .../crypto/CredentialEncryption.java | 13 +- .../policy/controller/PolicyController.java | 41 +- .../policy/engine/PolicyEngine.java | 4 + .../policy/model/PolicyRunView.java | 31 ++ .../security/model/JwtSigningKeyEntity.java | 52 +++ .../repository/JwtSigningKeyRepository.java | 26 ++ .../service/KeyPairCleanupService.java | 35 +- .../service/KeyPersistenceService.java | 429 +++++++----------- .../controller/PolicyControllerTest.java | 8 +- .../service/KeyPairCleanupServiceTest.java | 102 +++++ .../KeyPersistenceServiceInterfaceTest.java | 237 ++++------ 14 files changed, 600 insertions(+), 412 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/security/model/JwtSigningKeyEntity.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/security/repository/JwtSigningKeyRepository.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPairCleanupServiceTest.java diff --git a/app/common/src/main/java/stirling/software/common/model/job/JobResult.java b/app/common/src/main/java/stirling/software/common/model/job/JobResult.java index 52c0826e2b..aa43431a15 100644 --- a/app/common/src/main/java/stirling/software/common/model/job/JobResult.java +++ b/app/common/src/main/java/stirling/software/common/model/job/JobResult.java @@ -4,6 +4,8 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -47,6 +49,9 @@ public class JobResult { */ private final List notes = new CopyOnWriteArrayList<>(); + /** Key/value metadata that survives the write-through into the shared job store. */ + private final Map metadata = new ConcurrentHashMap<>(); + /** * Create a new JobResult with the given job ID * @@ -161,4 +166,16 @@ public class JobResult { public List getNotes() { return Collections.unmodifiableList(notes); } + + /** Attach a metadata value, e.g. a policy id so cluster peers can identify a policy run. */ + public void putMetadata(String key, String value) { + if (key != null && value != null) { + this.metadata.put(key, value); + } + } + + /** An unmodifiable view of this job's metadata. */ + public Map getMetadata() { + return Collections.unmodifiableMap(metadata); + } } diff --git a/app/common/src/main/java/stirling/software/common/service/TaskManager.java b/app/common/src/main/java/stirling/software/common/service/TaskManager.java index 790e0626ac..3fd9ac3fe4 100644 --- a/app/common/src/main/java/stirling/software/common/service/TaskManager.java +++ b/app/common/src/main/java/stirling/software/common/service/TaskManager.java @@ -230,6 +230,18 @@ public class TaskManager { return false; } + /** Attach metadata to a job and write it through to the shared store for cluster peers. */ + public boolean putMetadata(String jobId, String key, String value) { + JobResult jobResult = jobResults.get(jobId); + if (jobResult != null) { + jobResult.putMetadata(key, value); + writeThrough(jobId, jobResult); + return true; + } + log.warn("Attempted to set metadata on non-existent job ID: {}", jobId); + return false; + } + /** * Get statistics about all jobs in the system * @@ -378,7 +390,7 @@ public class TaskManager { fileIds.add(rf.getFileId()); } } - Map meta = new HashMap<>(); + Map meta = new HashMap<>(result.getMetadata()); if (result.getNotes() != null && !result.getNotes().isEmpty()) { meta.put("notesCount", Integer.toString(result.getNotes().size())); } diff --git a/app/core/src/main/resources/application.properties b/app/core/src/main/resources/application.properties index 7c1190926b..c05f2dcbea 100644 --- a/app/core/src/main/resources/application.properties +++ b/app/core/src/main/resources/application.properties @@ -98,7 +98,8 @@ spring.main.allow-bean-definition-overriding=true # spring-data-redis is on the classpath only for the optional Valkey backplane (which wires its own # factory); exclude Spring Boot's stock Redis auto-config so a default install doesn't create a dead # localhost:6379 factory that flips /actuator/health to DOWN. -spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration +# Also exclude the repositories auto-config: in cluster mode it needs a redisTemplate bean we don't define. +spring.autoconfigure.exclude=org.springframework.boot.data.redis.autoconfigure.DataRedisAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisReactiveAutoConfiguration,org.springframework.boot.data.redis.autoconfigure.DataRedisRepositoriesAutoConfiguration # Set up a consistent temporary directory location java.io.tmpdir=${stirling.tempfiles.directory:${java.io.tmpdir}/stirling-pdf} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java index 96e20f55cb..8ac0ddda96 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/crypto/CredentialEncryption.java @@ -44,10 +44,13 @@ public class CredentialEncryption { private static volatile SecretKey key; private final String configuredKey; + private final boolean clusterEnabled; public CredentialEncryption( - @Value("${stirling.security.credentialEncryptionKey:}") String configuredKey) { + @Value("${stirling.security.credentialEncryptionKey:}") String configuredKey, + @Value("${cluster.enabled:false}") boolean clusterEnabled) { this.configuredKey = configuredKey; + this.clusterEnabled = clusterEnabled; } @PostConstruct @@ -64,6 +67,14 @@ public class CredentialEncryption { if (configured != null && !configured.isBlank()) { return new SecretKeySpec(Base64.getDecoder().decode(configured.trim()), ALGORITHM); } + // Cluster nodes must share this key, so fail fast rather than generate a node-local one. + if (clusterEnabled) { + throw new IllegalStateException( + "cluster.enabled=true requires a shared credential encryption key. Set" + + " STIRLING_CREDENTIAL_ENCRYPTION_KEY (or" + + " stirling.security.credentialEncryptionKey) to the same value on every" + + " node."); + } return loadOrCreateKeyFile(); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 71e5b93c4e..e0df6ba2b1 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -6,6 +6,7 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; @@ -39,6 +40,8 @@ import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.JobStoreEntry; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.JobResponse; import stirling.software.common.service.JobOwnershipService; @@ -102,6 +105,8 @@ public class PolicyController { private final ApplicationProperties applicationProperties; private final TempFileManager tempFileManager; private final JobOwnershipService jobOwnershipService; + // Shared job store: lets the run endpoints see runs that executed on other nodes. + private final JobStore jobStore; @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @Operation( @@ -172,10 +177,19 @@ public class PolicyController { description = "Returns the current status, step cursor, and output files of a run.") public ResponseEntity status(@PathVariable String runId) { PolicyRun run = runRegistry.get(runId); - if (run == null) { - return ResponseEntity.notFound().build(); + if (run != null) { + return ResponseEntity.ok(PolicyRunView.of(run)); } - return ResponseEntity.ok(PolicyRunView.of(run)); + // Not local: read the run's shared projection so any node can serve its status. + if (ownedByCurrentUser(runId)) { + Optional entry = jobStore.get(runId); + if (entry.isPresent() + && entry.get().resultMeta() != null + && entry.get().resultMeta().containsKey("policyId")) { + return ResponseEntity.ok(PolicyRunView.ofEntry(entry.get())); + } + } + return ResponseEntity.notFound().build(); } @GetMapping("/runs") @@ -188,11 +202,26 @@ public class PolicyController { + " collected, rather than orphaned on the backend. Ad-hoc runs (no" + " policy id) are excluded.") public List listRuns() { - return runRegistry.all().stream() + // Local runs first (they carry live step state); keyed by runId to dedupe shared entries. + Map byRunId = new LinkedHashMap<>(); + runRegistry.all().stream() .filter(run -> run.getPolicyId() != null) .filter(run -> ownedByCurrentUser(run.getRunId())) - .map(PolicyRunView::of) - .toList(); + .forEach(run -> byRunId.put(run.getRunId(), PolicyRunView.of(run))); + // Then runs from other nodes, read from the shared job store. + for (JobStoreEntry entry : jobStore.all()) { + if (byRunId.containsKey(entry.jobId())) { + continue; + } + Map meta = entry.resultMeta(); + if (meta == null || !meta.containsKey("policyId")) { + continue; // ad-hoc job, not a stored-policy run + } + if (ownedByCurrentUser(entry.jobId())) { + byRunId.put(entry.jobId(), PolicyRunView.ofEntry(entry)); + } + } + return List.copyOf(byRunId.values()); } /** diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java index 2ceabae656..abdb69cb07 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java @@ -134,6 +134,10 @@ public class PolicyEngine { // ownership check passes. No-op when security is off. String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString()); taskManager.createTask(runId); + // Tag the shared job entry with the policy id so peers can list it as a policy run. + if (policyId != null) { + taskManager.putMetadata(runId, "policyId", policyId); + } PolicyRun run = new PolicyRun(runId, policyId, definition); registry.register(run); CompletableFuture completion = new CompletableFuture<>(); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java index 04a2707077..6d98e273fc 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRunView.java @@ -1,7 +1,9 @@ package stirling.software.proprietary.policy.model; import java.util.List; +import java.util.Map; +import stirling.software.common.cluster.JobStoreEntry; import stirling.software.common.model.job.ResultFile; /** @@ -34,4 +36,33 @@ public record PolicyRunView( run.getOutputs(), run.getCreatedAt().toEpochMilli()); } + + /** Cross-node view from a shared job-store entry; step cursor is node-local so it reads 0. */ + public static PolicyRunView ofEntry(JobStoreEntry entry) { + Map meta = entry.resultMeta() == null ? Map.of() : entry.resultMeta(); + PolicyRunStatus status = + switch (entry.state()) { + case COMPLETE -> PolicyRunStatus.COMPLETED; + case FAILED -> PolicyRunStatus.FAILED; + case RUNNING, PENDING -> PolicyRunStatus.RUNNING; + }; + List outputs = + entry.fileIds() == null + ? List.of() + : entry.fileIds().stream() + .map(id -> ResultFile.builder().fileId(id).build()) + .toList(); + long createdAt = entry.createdAt() == null ? 0L : entry.createdAt().toEpochMilli(); + return new PolicyRunView( + entry.jobId(), + meta.get("policyId"), + status, + 0, + 0, + entry.error(), + null, + null, + outputs, + createdAt); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/model/JwtSigningKeyEntity.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/model/JwtSigningKeyEntity.java new file mode 100644 index 0000000000..23cf42502d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/model/JwtSigningKeyEntity.java @@ -0,0 +1,52 @@ +package stirling.software.proprietary.security.model; + +import java.io.Serializable; +import java.time.LocalDateTime; + +import org.hibernate.annotations.CreationTimestamp; + +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import stirling.software.proprietary.integration.crypto.EncryptedStringConverter; + +/** A JWT signing keypair in the shared DB; the private key is encrypted at rest. */ +@Entity +@Table(name = "jwt_signing_keys") +@NoArgsConstructor +@Getter +@Setter +public class JwtSigningKeyEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "key_id", length = 128) + private String keyId; + + // Base64 X.509 public key (non-secret). + @Column(name = "verifying_key", columnDefinition = "text", nullable = false) + private String verifyingKey; + + // Base64 PKCS#8 private key, encrypted at rest. + @Convert(converter = EncryptedStringConverter.class) + @Column(name = "signing_key", columnDefinition = "text", nullable = false) + private String signingKey; + + @CreationTimestamp + @Column(name = "created_at", updatable = false) + private LocalDateTime createdAt; + + public JwtSigningKeyEntity(String keyId, String verifyingKey, String signingKey) { + this.keyId = keyId; + this.verifyingKey = verifyingKey; + this.signingKey = signingKey; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/repository/JwtSigningKeyRepository.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/repository/JwtSigningKeyRepository.java new file mode 100644 index 0000000000..d34f45593b --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/repository/JwtSigningKeyRepository.java @@ -0,0 +1,26 @@ +package stirling.software.proprietary.security.repository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import stirling.software.proprietary.security.model.JwtSigningKeyEntity; + +/** Shared-DB store of JWT signing keys - the source of truth every cluster node reads from. */ +@Repository +public interface JwtSigningKeyRepository extends JpaRepository { + + /** Newest first, so the most recently created key is the active signing key. */ + List findAllByOrderByCreatedAtDesc(); + + /** + * The current active signing key: the single newest row. Used for cheap cluster convergence. + */ + Optional findFirstByOrderByCreatedAtDesc(); + + /** Keys created before the cutoff, eligible for rotation cleanup. */ + List findByCreatedAtBefore(LocalDateTime cutoff); +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPairCleanupService.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPairCleanupService.java index 8745f505fd..ea7245aa75 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPairCleanupService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPairCleanupService.java @@ -3,8 +3,10 @@ package stirling.software.proprietary.security.service; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.time.LocalDateTime; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; import org.springframework.beans.factory.annotation.Autowired; @@ -17,6 +19,8 @@ import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.cluster.DistributedLock; +import stirling.software.common.cluster.DistributedLock.LockHandle; import stirling.software.common.configuration.InstallationPathConfig; import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.security.model.JwtVerificationKey; @@ -26,15 +30,23 @@ import stirling.software.proprietary.security.model.JwtVerificationKey; @ConditionalOnBooleanProperty("v2") public class KeyPairCleanupService { + // Cluster-wide single-writer: keys live in the shared DB, so only one node may prune + rotate + // per cycle. Otherwise every node runs this and they race to delete each other's keys. + private static final String CLEANUP_LOCK = "jwt-key-cleanup"; + private static final Duration LOCK_LEASE = Duration.ofMinutes(5); + private final KeyPersistenceService keyPersistenceService; private final ApplicationProperties.Security.Jwt jwtProperties; + private final DistributedLock distributedLock; @Autowired public KeyPairCleanupService( KeyPersistenceService keyPersistenceService, - ApplicationProperties applicationProperties) { + ApplicationProperties applicationProperties, + DistributedLock distributedLock) { this.keyPersistenceService = keyPersistenceService; this.jwtProperties = applicationProperties.getSecurity().getJwt(); + this.distributedLock = distributedLock; } @Transactional @@ -44,7 +56,28 @@ public class KeyPairCleanupService { if (!jwtProperties.isEnableKeyCleanup() || !keyPersistenceService.isKeystoreEnabled()) { return; } + // A lock-backend error must never fail this @PostConstruct/scheduled run: degrade to + // "skip this cycle" so a transient Valkey blip can't stop a node from booting. + Optional lock; + try { + lock = distributedLock.tryAcquire(CLEANUP_LOCK, LOCK_LEASE); + } catch (RuntimeException e) { + log.warn( + "Could not acquire the JWT key-cleanup lock ({}); skipping this cycle", + e.getMessage()); + return; + } + // No lock means another node is already pruning; skip until the next tick. + if (lock.isEmpty()) { + log.debug("Another node holds the JWT key-cleanup lock; skipping this cycle"); + return; + } + try (LockHandle held = lock.get()) { + runCleanup(); + } + } + private void runCleanup() { LocalDateTime cutoffDate = LocalDateTime.now().minusDays(jwtProperties.getKeyRetentionDays()); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPersistenceService.java b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPersistenceService.java index 49e66e5c53..adb11936ff 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPersistenceService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/security/service/KeyPersistenceService.java @@ -18,15 +18,17 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.caffeine.CaffeineCache; +import org.springframework.context.annotation.DependsOn; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import jakarta.annotation.PostConstruct; @@ -34,27 +36,39 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.common.configuration.InstallationPathConfig; import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.model.JwtSigningKeyEntity; import stirling.software.proprietary.security.model.JwtVerificationKey; +import stirling.software.proprietary.security.repository.JwtSigningKeyRepository; +/** Persists JWT signing keys in the shared DB so all nodes sign/verify with the same key. */ @Slf4j @Service +// CredentialEncryption must init first: startup persists an encrypted private key. +@DependsOn("credentialEncryption") public class KeyPersistenceService implements KeyPersistenceServiceInterface { public static final String KEY_SUFFIX = ".key"; public static final String PUB_KEY_SUFFIX = ".pub"; private final ApplicationProperties.Security.Jwt jwtProperties; - private final CacheManager cacheManager; private final Cache verifyingKeyCache; + private final JwtSigningKeyRepository keyRepository; + private final boolean clusterEnabled; + + // kid -> KeyPair; safe to cache since key material is immutable. + private final Map keyPairCache = new ConcurrentHashMap<>(); private volatile JwtVerificationKey activeKey; - @Autowired public KeyPersistenceService( - ApplicationProperties applicationProperties, CacheManager cacheManager) { + ApplicationProperties applicationProperties, + CacheManager cacheManager, + JwtSigningKeyRepository keyRepository, + @Value("${cluster.enabled:false}") boolean clusterEnabled) { this.jwtProperties = applicationProperties.getSecurity().getJwt(); - this.cacheManager = cacheManager; this.verifyingKeyCache = cacheManager.getCache("verifyingKeys"); + this.keyRepository = keyRepository; + this.clusterEnabled = clusterEnabled; } @PostConstruct @@ -63,138 +77,86 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface { log.info("JWT keystore is disabled - keys will be generated in memory"); return; } - try { - ensurePrivateKeyDirectoryExists(); - loadExistingKeysFromDisk(); + importLegacyDiskKeysIfPresent(); + loadKeysFromDb(); } catch (Exception e) { - log.error("Failed to initialize keystore, using in-memory generation", e); - } - } - - /** - * Load all existing JWT keys from disk into memory on startup. - * - *

This ensures tokens signed with previous keys remain valid after server restart. If no - * keys exist on disk, generates a new keypair. - */ - private void loadExistingKeysFromDisk() { - try { - Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath()); - - if (!Files.exists(keyDirectory)) { - log.info("No existing keys found, generating new keypair"); - generateAndStoreKeypair(); - return; - } - - List keyFiles; - try (var stream = Files.list(keyDirectory)) { - keyFiles = - stream.filter(path -> path.toString().endsWith(KEY_SUFFIX)) - .sorted( - (a, b) -> - b.getFileName().compareTo(a.getFileName())) // Most - // recent - // first - .toList(); - } - - if (keyFiles.isEmpty()) { - log.info("No existing keys found in directory, generating new keypair"); - generateAndStoreKeypair(); - return; - } - - log.info("Loading {} existing JWT keys from disk", keyFiles.size()); - int loadedCount = 0; - - for (Path keyFile : keyFiles) { - try { - String keyId = keyFile.getFileName().toString().replace(KEY_SUFFIX, ""); - - // Load private key first - PrivateKey privateKey = loadPrivateKey(keyId); - - // Try to load public key, or generate it from private key if missing - // (migration) - String encodedPublicKey; - try { - encodedPublicKey = loadPublicKey(keyId); - } catch (IOException e) { - // Public key file doesn't exist - generate it from private key (migration) - log.info("Migrating legacy key: generating public key file for {}", keyId); - KeyPair keyPair = reconstructKeyPair(privateKey); - - // Save the public key file - Path publicKeyFile = keyDirectory.resolve(keyId + PUB_KEY_SUFFIX); - encodedPublicKey = encodePublicKey(keyPair.getPublic()); - Files.writeString(publicKeyFile, encodedPublicKey); - publicKeyFile.toFile().setReadable(true, true); - publicKeyFile.toFile().setWritable(true, true); - publicKeyFile.toFile().setExecutable(false, false); - - log.info("Successfully migrated key: {}", keyId); - } - - // Create verification key and add to cache - JwtVerificationKey verifyingKey = - new JwtVerificationKey(keyId, encodedPublicKey); - verifyingKeyCache.put(keyId, verifyingKey); - loadedCount++; - - // Set the most recent key as active (first in sorted list) - if (activeKey == null) { - activeKey = verifyingKey; - log.info("Set active JWT signing key: {}", keyId); - } else { - log.debug( - "Loaded historical JWT key: {} (created: {})", - keyId, - verifyingKey.getCreatedAt()); - } - } catch (Exception e) { - log.warn( - "Failed to load key: {}, skipping. Error: {}", - keyFile.getFileName(), - e.getMessage()); - } - } - - if (loadedCount == 0) { - log.warn("No valid keys could be loaded from disk, generating new keypair"); - generateAndStoreKeypair(); - } else { - log.info( - "Successfully loaded {} JWT keys, active key: {}", - loadedCount, - activeKey.getKeyId()); - } - - } catch (IOException e) { - log.error("Failed to load keys from disk, generating new keypair", e); + log.error("Failed to initialize keystore, generating a fresh keypair", e); generateAndStoreKeypair(); } } - @Transactional - private JwtVerificationKey generateAndStoreKeypair() { - JwtVerificationKey verifyingKey = null; + /** + * Cluster convergence: adopt the newest signing key in the shared DB as this node's active key. + * Runs on every node so a key a peer just minted becomes the shared active signer within one + * interval, keeping cluster rotation equivalent to single-node. Cluster-only: a single node + * always holds its own newest key, so this is skipped entirely off-cluster. + */ + @Scheduled(fixedDelayString = "${stirling.security.jwt.activeKeyReloadMs:300000}") + public void reloadActiveKeyFromDb() { + if (!clusterEnabled || !isKeystoreEnabled()) { + return; + } + try { + Optional newestOpt = + keyRepository.findFirstByOrderByCreatedAtDesc(); + if (newestOpt.isEmpty()) { + return; + } + JwtSigningKeyEntity newest = newestOpt.get(); + JwtVerificationKey current = activeKey; + if (current != null && newest.getKeyId().equals(current.getKeyId())) { + return; + } + JwtVerificationKey adopted = + new JwtVerificationKey(newest.getKeyId(), newest.getVerifyingKey()); + verifyingKeyCache.put(newest.getKeyId(), adopted); + activeKey = adopted; + log.info( + "Adopted newest JWT signing key {} from the shared DB as active", + newest.getKeyId()); + } catch (Exception e) { + log.warn("Could not reload active JWT key from the shared DB: {}", e.getMessage()); + } + } + /** Load every signing key from the shared DB into the caches; most recent becomes active. */ + private void loadKeysFromDb() { + List keys = keyRepository.findAllByOrderByCreatedAtDesc(); + if (keys.isEmpty()) { + log.info("No JWT keys in the database, generating a new keypair"); + generateAndStoreKeypair(); + return; + } + for (JwtSigningKeyEntity key : keys) { + verifyingKeyCache.put( + key.getKeyId(), new JwtVerificationKey(key.getKeyId(), key.getVerifyingKey())); + } + activeKey = new JwtVerificationKey(keys.get(0).getKeyId(), keys.get(0).getVerifyingKey()); + log.info("Loaded {} JWT key(s) from DB, active key: {}", keys.size(), activeKey.getKeyId()); + } + + private JwtVerificationKey generateAndStoreKeypair() { try { KeyPair keyPair = generateRSAKeypair(); String keyId = generateKeyId(); + String verifyingKey = encodePublicKey(keyPair.getPublic()); + String signingKey = + Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded()); - storeKeyPair(keyId, keyPair); - verifyingKey = new JwtVerificationKey(keyId, encodePublicKey(keyPair.getPublic())); - verifyingKeyCache.put(keyId, verifyingKey); - activeKey = verifyingKey; + // Converter encrypts the private key at rest with the shared credential-encryption key. + keyRepository.save(new JwtSigningKeyEntity(keyId, verifyingKey, signingKey)); + + keyPairCache.put(keyId, keyPair); + JwtVerificationKey verificationKey = new JwtVerificationKey(keyId, verifyingKey); + verifyingKeyCache.put(keyId, verificationKey); + activeKey = verificationKey; log.info("Generated and stored new JWT keypair: {}", keyId); - } catch (IOException e) { + return verificationKey; + } catch (RuntimeException e) { log.error("Failed to generate and store keypair", e); + return null; } - - return verifyingKey; } @Override @@ -207,25 +169,29 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface { @Override public Optional getKeyPair(String keyId) { - if (!isKeystoreEnabled()) { + if (!isKeystoreEnabled() || keyId == null) { return Optional.empty(); } - + KeyPair cached = keyPairCache.get(keyId); + if (cached != null) { + return Optional.of(cached); + } + Optional entityOpt = keyRepository.findById(keyId); + if (entityOpt.isEmpty()) { + log.warn("No signing key found in DB for keyId: {}", keyId); + return Optional.empty(); + } + JwtSigningKeyEntity entity = entityOpt.get(); try { - JwtVerificationKey verifyingKey = - verifyingKeyCache.get(keyId, JwtVerificationKey.class); - - if (verifyingKey == null) { - log.warn("No signing key found in database for keyId: {}", keyId); - return Optional.empty(); - } - - PrivateKey privateKey = loadPrivateKey(keyId); - PublicKey publicKey = decodePublicKey(verifyingKey.getVerifyingKey()); - - return Optional.of(new KeyPair(publicKey, privateKey)); - } catch (Exception e) { - log.error("Failed to load keypair for keyId: {}", keyId, e); + KeyPair keyPair = + new KeyPair( + decodePublicKey(entity.getVerifyingKey()), + decodePrivateKey(entity.getSigningKey())); + keyPairCache.put(keyId, keyPair); + verifyingKeyCache.put(keyId, new JwtVerificationKey(keyId, entity.getVerifyingKey())); + return Optional.of(keyPair); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + log.error("Failed to decode keypair for keyId: {}", keyId, e); return Optional.empty(); } } @@ -241,44 +207,69 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface { } @Override - @CacheEvict( - value = {"verifyingKeys"}, - key = "#keyId", - condition = "#root.target.isKeystoreEnabled()") public void removeKey(String keyId) { + keyRepository.deleteById(keyId); verifyingKeyCache.evict(keyId); + keyPairCache.remove(keyId); } @Override public List getKeysEligibleForCleanup(LocalDateTime cutoffDate) { - CaffeineCache caffeineCache = (CaffeineCache) verifyingKeyCache; - com.github.benmanes.caffeine.cache.Cache nativeCache = - caffeineCache.getNativeCache(); - - log.debug( - "Cache size: {}, Checking {} keys for cleanup", - nativeCache.estimatedSize(), - nativeCache.asMap().size()); - - return nativeCache.asMap().values().stream() - .filter(value -> value instanceof JwtVerificationKey) - .map(value -> (JwtVerificationKey) value) - .filter( - key -> { - boolean eligible = key.getCreatedAt().isBefore(cutoffDate); - log.debug( - "Key {} created at {}, eligible for cleanup: {}", - key.getKeyId(), - key.getCreatedAt(), - eligible); - return eligible; - }) + return keyRepository.findByCreatedAtBefore(cutoffDate).stream() + .map(e -> new JwtVerificationKey(e.getKeyId(), e.getVerifyingKey())) .toList(); } + /** Import any pre-existing on-disk keys into the DB once, so upgrades keep sessions valid. */ + private void importLegacyDiskKeysIfPresent() { + if (keyRepository.count() > 0) { + return; + } + Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath()); + if (!Files.exists(keyDirectory)) { + return; + } + List keyFiles; + try (var stream = Files.list(keyDirectory)) { + keyFiles = stream.filter(p -> p.toString().endsWith(KEY_SUFFIX)).toList(); + } catch (IOException e) { + log.warn("Could not list legacy key directory {}: {}", keyDirectory, e.getMessage()); + return; + } + int imported = 0; + for (Path keyFile : keyFiles) { + String keyId = keyFile.getFileName().toString().replace(KEY_SUFFIX, ""); + try { + PrivateKey privateKey = loadPrivateKey(keyId); + String verifyingKey = resolveLegacyPublicKey(keyId, privateKey); + String signingKey = Base64.getEncoder().encodeToString(privateKey.getEncoded()); + keyRepository.save(new JwtSigningKeyEntity(keyId, verifyingKey, signingKey)); + imported++; + } catch (Exception e) { + log.warn("Skipping legacy key {}: {}", keyId, e.getMessage()); + } + } + if (imported > 0) { + log.info("Imported {} legacy JWT key(s) from disk into the shared DB", imported); + } + } + + private String resolveLegacyPublicKey(String keyId, PrivateKey privateKey) + throws NoSuchAlgorithmException, InvalidKeySpecException { + try { + return loadPublicKey(keyId); + } catch (IOException e) { + // No .pub file: derive the public key from the RSA private key. + return encodePublicKey(reconstructKeyPair(privateKey).getPublic()); + } + } + + // UUID suffix so two nodes booting the same second don't collide on keyId. private String generateKeyId() { return "jwt-key-" - + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmmss")); + + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmmss")) + + "-" + + UUID.randomUUID().toString().substring(0, 8); } private KeyPair generateRSAKeypair() { @@ -291,125 +282,49 @@ public class KeyPersistenceService implements KeyPersistenceServiceInterface { } } - private void ensurePrivateKeyDirectoryExists() throws IOException { - Path keyPath = Path.of(InstallationPathConfig.getPrivateKeyPath()); - - if (!Files.exists(keyPath)) { - Files.createDirectories(keyPath); - } - } - - /** - * Store both private and public keys to disk. - * - *

Private key stored as: keyId.key - * - *

Public key stored as: keyId.pub - */ - private void storeKeyPair(String keyId, KeyPair keyPair) throws IOException { - Path keyDirectory = Path.of(InstallationPathConfig.getPrivateKeyPath()); - - // Store private key - Path privateKeyFile = keyDirectory.resolve(keyId + KEY_SUFFIX); - String encodedPrivateKey = - Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded()); - Files.writeString(privateKeyFile, encodedPrivateKey); - - // Set read/write to only the owner (security) - privateKeyFile.toFile().setReadable(true, true); - privateKeyFile.toFile().setWritable(true, true); - privateKeyFile.toFile().setExecutable(false, false); - - // Store public key - Path publicKeyFile = keyDirectory.resolve(keyId + PUB_KEY_SUFFIX); - String encodedPublicKey = - Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded()); - Files.writeString(publicKeyFile, encodedPublicKey); - - // Public key can be more permissive but still restrict to owner - publicKeyFile.toFile().setReadable(true, true); - publicKeyFile.toFile().setWritable(true, true); - publicKeyFile.toFile().setExecutable(false, false); - - log.debug( - "Stored keypair to disk: {} (private: {}, public: {})", - keyId, - privateKeyFile.getFileName(), - publicKeyFile.getFileName()); - } - private PrivateKey loadPrivateKey(String keyId) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { Path keyFile = Path.of(InstallationPathConfig.getPrivateKeyPath()).resolve(keyId + KEY_SUFFIX); - if (!Files.exists(keyFile)) { throw new IOException("Private key not found: " + keyFile); } - - String encodedKey = Files.readString(keyFile); - byte[] keyBytes = Base64.getDecoder().decode(encodedKey); - PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - - return keyFactory.generatePrivate(keySpec); + return decodePrivateKey(Files.readString(keyFile)); } - /** - * Load public key from disk. - * - * @param keyId the key identifier - * @return Base64-encoded public key string - * @throws IOException if the public key file is not found - */ private String loadPublicKey(String keyId) throws IOException { Path publicKeyFile = Path.of(InstallationPathConfig.getPrivateKeyPath()).resolve(keyId + PUB_KEY_SUFFIX); - if (!Files.exists(publicKeyFile)) { throw new IOException("Public key not found: " + publicKeyFile); } - return Files.readString(publicKeyFile).trim(); } - /** - * Reconstruct a KeyPair from a PrivateKey. - * - *

For RSA keys, derives the public key from the private key. - * - * @param privateKey the RSA private key - * @return reconstructed KeyPair - * @throws NoSuchAlgorithmException if RSA algorithm is not available - * @throws InvalidKeySpecException if the key specification is invalid - */ private KeyPair reconstructKeyPair(PrivateKey privateKey) throws NoSuchAlgorithmException, InvalidKeySpecException { - // For RSA, we can derive the public key from the private key KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - - // Get the private key spec RSAPrivateCrtKey rsaPrivateKey = (RSAPrivateCrtKey) privateKey; - - // Create public key spec from private key parameters RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPublicExponent()); - - // Generate public key - PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); - - return new KeyPair(publicKey, privateKey); + return new KeyPair(keyFactory.generatePublic(publicKeySpec), privateKey); } private String encodePublicKey(PublicKey publicKey) { return Base64.getEncoder().encodeToString(publicKey.getEncoded()); } + @Override public PublicKey decodePublicKey(String encodedKey) throws NoSuchAlgorithmException, InvalidKeySpecException { - byte[] keyBytes = Base64.getDecoder().decode(encodedKey); - X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - return keyFactory.generatePublic(keySpec); + X509EncodedKeySpec keySpec = new X509EncodedKeySpec(Base64.getDecoder().decode(encodedKey)); + return KeyFactory.getInstance("RSA").generatePublic(keySpec); + } + + private PrivateKey decodePrivateKey(String encodedKey) + throws NoSuchAlgorithmException, InvalidKeySpecException { + PKCS8EncodedKeySpec keySpec = + new PKCS8EncodedKeySpec(Base64.getDecoder().decode(encodedKey)); + return KeyFactory.getInstance("RSA").generatePrivate(keySpec); } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java index 8258622e28..cf33de8656 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/controller/PolicyControllerTest.java @@ -27,6 +27,8 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.server.ResponseStatusException; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; +import stirling.software.common.cluster.JobStore; +import stirling.software.common.cluster.inprocess.InProcessJobStore; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.JobResponse; import stirling.software.common.service.JobOwnershipService; @@ -78,6 +80,7 @@ class PolicyControllerTest { @Mock private JobOwnershipService jobOwnershipService; private ApplicationProperties applicationProperties; + private final JobStore jobStore = new InProcessJobStore(); private PolicyController controller; private final java.util.List @@ -106,7 +109,8 @@ class PolicyControllerTest { policyTriggers, applicationProperties, tempFileManager, - jobOwnershipService); + jobOwnershipService, + jobStore); } private static stirling.software.proprietary.policy.trigger.PolicyTrigger trigger( @@ -264,6 +268,8 @@ class PolicyControllerTest { @DisplayName("returns 404 when run is unknown") void notFound() { when(runRegistry.get("missing")).thenReturn(null); + when(jobOwnershipService.extractJobId("missing")).thenReturn("missing"); + when(jobOwnershipService.createScopedJobKey("missing")).thenReturn("missing"); ResponseEntity response = controller.status("missing"); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPairCleanupServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPairCleanupServiceTest.java new file mode 100644 index 0000000000..c3cb0fb1ba --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPairCleanupServiceTest.java @@ -0,0 +1,102 @@ +package stirling.software.proprietary.security.service; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.cluster.DistributedLock; +import stirling.software.common.cluster.DistributedLock.LockHandle; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.model.JwtVerificationKey; + +/** Cluster safety: pruning JWT keys is single-writer, gated on the shared cleanup lock. */ +@ExtendWith(MockitoExtension.class) +class KeyPairCleanupServiceTest { + + @Mock private KeyPersistenceService keyPersistenceService; + @Mock private ApplicationProperties applicationProperties; + @Mock private ApplicationProperties.Security security; + @Mock private ApplicationProperties.Security.Jwt jwtProperties; + @Mock private DistributedLock distributedLock; + @Mock private LockHandle lockHandle; + + private KeyPairCleanupService cleanupService; + + @BeforeEach + void setUp() { + lenient().when(applicationProperties.getSecurity()).thenReturn(security); + lenient().when(security.getJwt()).thenReturn(jwtProperties); + lenient().when(jwtProperties.isEnableKeyCleanup()).thenReturn(true); + lenient().when(keyPersistenceService.isKeystoreEnabled()).thenReturn(true); + cleanupService = + new KeyPairCleanupService( + keyPersistenceService, applicationProperties, distributedLock); + } + + @Test + void skipsPruningWhenAnotherNodeHoldsTheLock() { + when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.empty()); + + cleanupService.cleanup(); + + // No node-local pruning happened; the lock holder owns this cycle. + verify(keyPersistenceService, never()).getKeysEligibleForCleanup(any()); + verify(keyPersistenceService, never()).refreshActiveKeyPair(); + } + + @Test + void prunesAndRotatesWhenLockAcquiredThenReleasesIt() { + when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.of(lockHandle)); + when(keyPersistenceService.getKeysEligibleForCleanup(any())) + .thenReturn(List.of(new JwtVerificationKey("old-key", "cHVi"))); + + cleanupService.cleanup(); + + verify(keyPersistenceService).removeKey("old-key"); + verify(keyPersistenceService).refreshActiveKeyPair(); + verify(lockHandle).close(); + } + + @Test + void releasesTheLockEvenWhenNoKeysAreEligible() { + when(distributedLock.tryAcquire(any(), any())).thenReturn(Optional.of(lockHandle)); + when(keyPersistenceService.getKeysEligibleForCleanup(any())).thenReturn(List.of()); + + cleanupService.cleanup(); + + verify(keyPersistenceService, never()).refreshActiveKeyPair(); + verify(lockHandle).close(); + } + + @Test + void skipsPruningWhenTheLockBackendErrors() { + // A Valkey blip at boot must not fail startup: tryAcquire throwing degrades to skip. + when(distributedLock.tryAcquire(any(), any())) + .thenThrow(new RuntimeException("valkey unreachable")); + + cleanupService.cleanup(); + + verify(keyPersistenceService, never()).getKeysEligibleForCleanup(any()); + verify(keyPersistenceService, never()).refreshActiveKeyPair(); + } + + @Test + void doesNothingWhenCleanupDisabled() { + when(jwtProperties.isEnableKeyCleanup()).thenReturn(false); + + cleanupService.cleanup(); + + verify(distributedLock, never()).tryAcquire(any(), any()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPersistenceServiceInterfaceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPersistenceServiceInterfaceTest.java index 33b971e5ac..f16117fdb9 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPersistenceServiceInterfaceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/security/service/KeyPersistenceServiceInterfaceTest.java @@ -4,45 +4,41 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.util.Base64; +import java.util.List; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.cache.CacheManager; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; -import stirling.software.common.configuration.InstallationPathConfig; import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.security.model.JwtSigningKeyEntity; import stirling.software.proprietary.security.model.JwtVerificationKey; +import stirling.software.proprietary.security.repository.JwtSigningKeyRepository; +/** DB-backed keystore: a key present only in the shared DB still resolves (the cross-node case). */ @ExtendWith(MockitoExtension.class) class KeyPersistenceServiceInterfaceTest { @Mock private ApplicationProperties applicationProperties; - @Mock private ApplicationProperties.Security security; - @Mock private ApplicationProperties.Security.Jwt jwtConfig; - - @TempDir Path tempDir; + @Mock private JwtSigningKeyRepository keyRepository; private KeyPersistenceService keyPersistenceService; private KeyPair testKeyPair; @@ -58,175 +54,128 @@ class KeyPersistenceServiceInterfaceTest { lenient().when(applicationProperties.getSecurity()).thenReturn(security); lenient().when(security.getJwt()).thenReturn(jwtConfig); - lenient().when(jwtConfig.isEnableKeystore()).thenReturn(true); // Default value + lenient().when(jwtConfig.isEnableKeystore()).thenReturn(true); + lenient().when(keyRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + // clusterEnabled=true so the convergence-reload path is exercised. + keyPersistenceService = + new KeyPersistenceService(applicationProperties, cacheManager, keyRepository, true); + } + + private JwtSigningKeyEntity entityFrom(String keyId) { + return new JwtSigningKeyEntity( + keyId, + Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded()), + Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded())); } @ParameterizedTest @ValueSource(booleans = {true, false}) void testKeystoreEnabled(boolean keystoreEnabled) { when(jwtConfig.isEnableKeystore()).thenReturn(keystoreEnabled); - - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - - assertEquals(keystoreEnabled, keyPersistenceService.isKeystoreEnabled()); - } + assertEquals(keystoreEnabled, keyPersistenceService.isKeystoreEnabled()); } @Test - void testGetActiveKeypairWhenNoActiveKeyExists() { - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - keyPersistenceService.initializeKeystore(); + void generatesAndPersistsAKeyWhenNoneIsActive() { + // getActiveKey with no active key mints one and persists it - no disk involved. + JwtVerificationKey active = keyPersistenceService.getActiveKey(); - JwtVerificationKey result = keyPersistenceService.getActiveKey(); - - assertNotNull(result); - assertNotNull(result.getKeyId()); - assertNotNull(result.getVerifyingKey()); - } + assertNotNull(active); + assertNotNull(active.getKeyId()); + assertNotNull(active.getVerifyingKey()); + verify(keyRepository).save(any(JwtSigningKeyEntity.class)); } @Test - void testGetActiveKeyPairWithExistingKey() throws Exception { - String keyId = "test-key-2024-01-01-120000"; - String publicKeyBase64 = - Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded()); - String privateKeyBase64 = - Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded()); + void loadsTheMostRecentExistingKeyAsActive() { + when(keyRepository.count()).thenReturn(1L); + when(keyRepository.findAllByOrderByCreatedAtDesc()) + .thenReturn(List.of(entityFrom("jwt-key-2026-07-13-000000-abcd1234"))); - JwtVerificationKey existingKey = new JwtVerificationKey(keyId, publicKeyBase64); + keyPersistenceService.initializeKeystore(); + JwtVerificationKey active = keyPersistenceService.getActiveKey(); - Path keyFile = tempDir.resolve(keyId + ".key"); - Files.writeString(keyFile, privateKeyBase64); - - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - keyPersistenceService.initializeKeystore(); - - JwtVerificationKey result = keyPersistenceService.getActiveKey(); - - assertNotNull(result); - assertNotNull(result.getKeyId()); - } + assertEquals("jwt-key-2026-07-13-000000-abcd1234", active.getKeyId()); } @Test - void testGetKeyPair() throws Exception { - String keyId = "test-key-123"; - String publicKeyBase64 = - Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded()); - String privateKeyBase64 = - Base64.getEncoder().encodeToString(testKeyPair.getPrivate().getEncoded()); + void getKeyPairResolvesAKeyPresentOnlyInTheSharedDb() { + // Never initialised locally: the key lives only in the DB, as if another node minted it. + String keyId = "jwt-key-from-another-node"; + when(keyRepository.findById(keyId)).thenReturn(Optional.of(entityFrom(keyId))); - JwtVerificationKey signingKey = new JwtVerificationKey(keyId, publicKeyBase64); + Optional result = keyPersistenceService.getKeyPair(keyId); - Path keyFile = tempDir.resolve(keyId + ".key"); - Files.writeString(keyFile, privateKeyBase64); - - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - - keyPersistenceService - .getClass() - .getDeclaredField("verifyingKeyCache") - .setAccessible(true); - var cache = cacheManager.getCache("verifyingKeys"); - cache.put(keyId, signingKey); - - Optional result = keyPersistenceService.getKeyPair(keyId); - - assertTrue(result.isPresent()); - assertNotNull(result.get().getPublic()); - assertNotNull(result.get().getPrivate()); - } + assertTrue(result.isPresent()); + assertNotNull(result.get().getPublic()); + assertNotNull(result.get().getPrivate()); } @Test - void testGetKeyPairNotFound() { - String keyId = "non-existent-key"; - - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - - Optional result = keyPersistenceService.getKeyPair(keyId); - - assertFalse(result.isPresent()); - } + void getKeyPairIsEmptyWhenTheKeyIsUnknown() { + when(keyRepository.findById("nope")).thenReturn(Optional.empty()); + assertFalse(keyPersistenceService.getKeyPair("nope").isPresent()); } @Test - void testGetKeyPairWhenKeystoreDisabled() { + void getKeyPairIsEmptyWhenKeystoreDisabled() { when(jwtConfig.isEnableKeystore()).thenReturn(false); - - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - - Optional result = keyPersistenceService.getKeyPair("any-key"); - - assertFalse(result.isPresent()); - } + assertFalse(keyPersistenceService.getKeyPair("any-key").isPresent()); } @Test - void testInitializeKeystoreCreatesDirectory() throws IOException { - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - keyPersistenceService.initializeKeystore(); + void eligibleForCleanupIsSourcedFromTheDb() { + when(keyRepository.findByCreatedAtBefore(any())).thenReturn(List.of(entityFrom("old-key"))); - assertTrue(Files.exists(tempDir)); - assertTrue(Files.isDirectory(tempDir)); - } + List stale = + keyPersistenceService.getKeysEligibleForCleanup(java.time.LocalDateTime.now()); + + assertEquals(1, stale.size()); + assertEquals("old-key", stale.get(0).getKeyId()); } @Test - void testLoadExistingKeypairWithMissingPrivateKeyFile() throws Exception { - String keyId = "test-key-missing-file"; - String publicKeyBase64 = - Base64.getEncoder().encodeToString(testKeyPair.getPublic().getEncoded()); + void reloadAdoptsTheNewestKeyAPeerMinted() { + // Boot with our own key active, then a peer mints a newer one in the shared DB. + when(keyRepository.count()).thenReturn(1L); + when(keyRepository.findAllByOrderByCreatedAtDesc()) + .thenReturn(List.of(entityFrom("jwt-key-local-old"))); + keyPersistenceService.initializeKeystore(); + assertEquals("jwt-key-local-old", keyPersistenceService.getActiveKey().getKeyId()); - JwtVerificationKey existingKey = new JwtVerificationKey(keyId, publicKeyBase64); + when(keyRepository.findFirstByOrderByCreatedAtDesc()) + .thenReturn(Optional.of(entityFrom("jwt-key-peer-new"))); - try (MockedStatic mockedStatic = - mockStatic(InstallationPathConfig.class)) { - mockedStatic - .when(InstallationPathConfig::getPrivateKeyPath) - .thenReturn(tempDir.toString()); - keyPersistenceService = new KeyPersistenceService(applicationProperties, cacheManager); - keyPersistenceService.initializeKeystore(); + keyPersistenceService.reloadActiveKeyFromDb(); - JwtVerificationKey result = keyPersistenceService.getActiveKey(); - assertNotNull(result); - assertNotNull(result.getKeyId()); - assertNotNull(result.getVerifyingKey()); - } + // Converged: this node now signs with the peer's newer key. + assertEquals("jwt-key-peer-new", keyPersistenceService.getActiveKey().getKeyId()); + } + + @Test + void reloadDoesNothingOffCluster() { + KeyPersistenceService singleNode = + new KeyPersistenceService( + applicationProperties, cacheManager, keyRepository, false); + + singleNode.reloadActiveKeyFromDb(); + + // Off-cluster the DB is never consulted for convergence. + verify(keyRepository, org.mockito.Mockito.never()).findFirstByOrderByCreatedAtDesc(); + } + + @Test + void reloadIsANoOpWhenAlreadyHoldingTheNewestKey() { + when(keyRepository.count()).thenReturn(1L); + when(keyRepository.findAllByOrderByCreatedAtDesc()) + .thenReturn(List.of(entityFrom("jwt-key-current"))); + keyPersistenceService.initializeKeystore(); + + when(keyRepository.findFirstByOrderByCreatedAtDesc()) + .thenReturn(Optional.of(entityFrom("jwt-key-current"))); + + keyPersistenceService.reloadActiveKeyFromDb(); + + assertEquals("jwt-key-current", keyPersistenceService.getActiveKey().getKeyId()); } }