Add S3 Object Lock retention to policy outputs (#7094)

# Description of Changes Add S3 Object Lock retention to policy outputs (create file and cant be deleted untill after a set deadline passed) --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [ ] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details.
2026-09-02 21:03:34 +03:00 · 2026-08-11 14:04:32 +00:00
parent c146f7e877
commit 8b1bfb87f7
3 changed files with 252 additions and 2 deletions
@@ -34,6 +34,7 @@ import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectLockMode;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -176,6 +177,7 @@ public class S3OutputSink implements PolicyOutputSink {
            if (conditionalPuts) {
                put.ifNoneMatch("*");
            }
+            applyObjectLock(put, config);
            try {
                PutObjectResponse response =
                        client.putObject(put.build(), RequestBody.fromFile(staged));
@@ -267,4 +269,24 @@ public class S3OutputSink implements PolicyOutputSink {
            throw new IllegalStateException("MD5 unavailable", e);
        }
    }
+
+    /**
+     * Write the object under Object Lock retention when the connection asks for it.
+     *
+     * <p>The retain-until date is computed per object from "now", so a policy that runs daily gives
+     * each document its own full retention window rather than a shared deadline.
+     *
+     * <p>Requires the bucket to have Object Lock enabled; S3 rejects the PUT otherwise, which is
+     * the correct outcome - silently storing a deletable object while an operator believes it is
+     * locked would be worse than failing.
+     */
+    private static void applyObjectLock(PutObjectRequest.Builder put, S3Config config) {
+        if (config.objectLockMode() == null || config.retentionDays() == null) {
+            return;
+        }
+        put.objectLockMode(ObjectLockMode.fromValue(config.objectLockMode()))
+                .objectLockRetainUntilDate(
+                        java.time.Instant.now()
+                                .plus(config.retentionDays(), java.time.temporal.ChronoUnit.DAYS));
+    }
 }
@@ -19,7 +19,9 @@ public record S3Config(
        String endpoint,
        String accessKeyId,
        String secretAccessKey,
-        boolean snapshot) {
+        boolean snapshot,
+        String objectLockMode,
+        Integer retentionDays) {

    private static final String BUCKET_OPTION = "bucket";
    private static final String REGION_OPTION = "region";
@@ -28,6 +30,12 @@ public record S3Config(
    private static final String ACCESS_KEY_ID_OPTION = "accessKeyId";
    private static final String SECRET_ACCESS_KEY_OPTION = "secretAccessKey";
    private static final String MODE_OPTION = "mode";
+    private static final String OBJECT_LOCK_MODE_OPTION = "objectLockMode";
+    private static final String RETENTION_DAYS_OPTION = "retentionDays";
+
+    private static final String LOCK_GOVERNANCE = "GOVERNANCE";
+    private static final String LOCK_COMPLIANCE = "COMPLIANCE";
+    private static final int MAX_RETENTION_DAYS = 36525;
    private static final String MODE_CONSUME = "consume";
    private static final String MODE_SNAPSHOT = "snapshot";

@@ -52,6 +60,41 @@ public record S3Config(
        if (mode != null && !MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) {
            throw new IllegalArgumentException("s3 config 'mode' must be 'consume' or 'snapshot'");
        }
+        // Object Lock: write-once retention, for records that must survive an administrator.
+        // COMPLIANCE cannot be shortened or deleted by anyone (not even the account root) before
+        // the retain-until date; GOVERNANCE can be bypassed with a specific IAM permission, so
+        // only COMPLIANCE is the answer to SEC 17a-4(f) / FINRA. The bucket must already have
+        // Object Lock enabled - it cannot be turned on per-object - and that in turn requires
+        // versioning, which can then never be suspended.
+        String objectLockMode = trimmed(options.get(OBJECT_LOCK_MODE_OPTION));
+        if (objectLockMode != null) {
+            objectLockMode = objectLockMode.toUpperCase(java.util.Locale.ROOT);
+            if (!LOCK_GOVERNANCE.equals(objectLockMode)
+                    && !LOCK_COMPLIANCE.equals(objectLockMode)) {
+                throw new IllegalArgumentException(
+                        "s3 config 'objectLockMode' must be 'GOVERNANCE' or 'COMPLIANCE'");
+            }
+        }
+        Integer retentionDays = null;
+        Object rawRetention = options.get(RETENTION_DAYS_OPTION);
+        if (rawRetention != null && !rawRetention.toString().isBlank()) {
+            try {
+                retentionDays = Integer.valueOf(rawRetention.toString().trim());
+            } catch (NumberFormatException e) {
+                throw new IllegalArgumentException("s3 config 'retentionDays' must be a number");
+            }
+            if (retentionDays < 1 || retentionDays > MAX_RETENTION_DAYS) {
+                throw new IllegalArgumentException(
+                        "s3 config 'retentionDays' must be between 1 and " + MAX_RETENTION_DAYS);
+            }
+        }
+        // S3 rejects one without the other, so catch it here where the operator can still fix it
+        // rather than at upload time on a worker thread.
+        if ((objectLockMode == null) != (retentionDays == null)) {
+            throw new IllegalArgumentException(
+                    "s3 config 'objectLockMode' and 'retentionDays' must be set together");
+        }
+
        return new S3Config(
                bucket,
                region == null ? "us-east-1" : region,
@@ -59,7 +102,9 @@ public record S3Config(
                endpoint,
                accessKeyId,
                secretAccessKey,
-                MODE_SNAPSHOT.equals(mode));
+                MODE_SNAPSHOT.equals(mode),
+                objectLockMode,
+                retentionDays);
    }

    private static String validEndpoint(String endpoint) {
@@ -0,0 +1,183 @@
+package stirling.software.proprietary.policy.output;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ByteArrayResource;
+import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.job.ResultFile;
+import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger;
+import stirling.software.proprietary.policy.model.OutputSpec;
+import stirling.software.proprietary.policy.s3.S3ConnectionPool;
+import stirling.software.proprietary.policy.s3.S3TestConnections;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ObjectLockMode;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+/**
+ * Proves the Object Lock (WORM) retention path against a real S3 API.
+ *
+ * <p>The claim being tested is a compliance one - SEC 17a-4(f) and FINRA require records on
+ * non-rewritable, non-erasable storage - so asserting that we merely <em>send</em> the retention
+ * headers would be worthless. What matters is that the store then genuinely refuses to delete the
+ * object, which is what {@link #anObjectWrittenUnderComplianceRetentionCannotBeDeleted} checks.
+ */
+@Testcontainers(disabledWithoutDocker = true)
+class S3OutputSinkObjectLockMinioTest {
+
+    private static final String POLICY = "p1";
+    private static final String ACCESS_KEY = "minioadmin";
+    private static final String SECRET_KEY = "minioadmin";
+
+    @Container
+    static MinIOContainer minio =
+            new MinIOContainer("minio/minio:latest")
+                    .withUserName(ACCESS_KEY)
+                    .withPassword(SECRET_KEY);
+
+    private static S3Client adminClient;
+    private static int bucketCounter;
+
+    private String bucket;
+    private S3OutputSink sink;
+
+    @BeforeEach
+    void setUp() {
+        if (adminClient == null) {
+            adminClient =
+                    S3Client.builder()
+                            .endpointOverride(URI.create(minio.getS3URL()))
+                            .httpClient(UrlConnectionHttpClient.create())
+                            .region(Region.US_EAST_1)
+                            .credentialsProvider(
+                                    StaticCredentialsProvider.create(
+                                            AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
+                            .serviceConfiguration(
+                                    S3Configuration.builder().pathStyleAccessEnabled(true).build())
+                            .build();
+        }
+        bucket = "worm-archive-" + ++bucketCounter;
+        // Object Lock can only be enabled at bucket creation here, and implies versioning.
+        adminClient.createBucket(
+                CreateBucketRequest.builder()
+                        .bucket(bucket)
+                        .objectLockEnabledForBucket(true)
+                        .build());
+
+        ApplicationProperties properties = new ApplicationProperties();
+        properties.getPolicies().setAllowPrivateS3Endpoints(true);
+        sink =
+                new S3OutputSink(
+                        new S3ConnectionPool(properties),
+                        S3TestConnections.legacyResolver(),
+                        new InProcessProcessedLedger());
+    }
+
+    @Test
+    void anObjectWrittenUnderComplianceRetentionCannotBeDeleted() throws IOException {
+        List<ResultFile> results =
+                sink.deliver(
+                        new OutputDelivery("run-1", POLICY),
+                        List.of(output("statement.pdf", "regulated record")),
+                        lockedOutputSpec("COMPLIANCE", 7));
+
+        assertThat(results).hasSize(1);
+
+        HeadObjectResponse head =
+                adminClient.headObject(
+                        HeadObjectRequest.builder().bucket(bucket).key("statement.pdf").build());
+        assertThat(head.objectLockMode()).isEqualTo(ObjectLockMode.COMPLIANCE);
+        // Retention is computed per object from "now", so a daily policy gives each document its
+        // own full window rather than a shared deadline.
+        assertThat(head.objectLockRetainUntilDate())
+                .isBetween(
+                        Instant.now().plus(6, ChronoUnit.DAYS),
+                        Instant.now().plus(8, ChronoUnit.DAYS));
+
+        // The point of the feature: the store itself refuses, not us.
+        assertThatThrownBy(
+                        () ->
+                                adminClient.deleteObject(
+                                        DeleteObjectRequest.builder()
+                                                .bucket(bucket)
+                                                .key("statement.pdf")
+                                                .versionId(head.versionId())
+                                                .build()))
+                .isInstanceOf(S3Exception.class);
+
+        // And it is still readable - locked, not quarantined.
+        assertThat(
+                        adminClient
+                                .getObject(
+                                        GetObjectRequest.builder()
+                                                .bucket(bucket)
+                                                .key("statement.pdf")
+                                                .build())
+                                .response()
+                                .contentLength())
+                .isEqualTo("regulated record".length());
+    }
+
+    @Test
+    void withoutRetentionConfiguredObjectsAreWrittenUnlockedAsBefore() throws IOException {
+        sink.deliver(
+                new OutputDelivery("run-2", POLICY),
+                List.of(output("scratch.pdf", "ordinary output")),
+                lockedOutputSpec(null, null));
+
+        HeadObjectResponse head =
+                adminClient.headObject(
+                        HeadObjectRequest.builder().bucket(bucket).key("scratch.pdf").build());
+        // No accidental retention: an Object-Lock-enabled bucket must not silently lock everything.
+        assertThat(head.objectLockMode()).isNull();
+        assertThat(head.objectLockRetainUntilDate()).isNull();
+    }
+
+    private OutputSpec lockedOutputSpec(String lockMode, Integer retentionDays) {
+        Map<String, Object> options = new java.util.LinkedHashMap<>();
+        options.put("bucket", bucket);
+        options.put("prefix", "");
+        options.put("endpoint", minio.getS3URL());
+        options.put("accessKeyId", ACCESS_KEY);
+        options.put("secretAccessKey", SECRET_KEY);
+        if (lockMode != null) {
+            options.put("objectLockMode", lockMode);
+            options.put("retentionDays", String.valueOf(retentionDays));
+        }
+        return new OutputSpec("s3", options);
+    }
+
+    private static org.springframework.core.io.Resource output(String name, String content) {
+        return new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)) {
+            @Override
+            public String getFilename() {
+                return name;
+            }
+        };
+    }
+}