diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java index b0bd210e10..10bf80e278 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/S3OutputSink.java @@ -34,6 +34,7 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectLockMode; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; @@ -176,6 +177,7 @@ public class S3OutputSink implements PolicyOutputSink { if (conditionalPuts) { put.ifNoneMatch("*"); } + applyObjectLock(put, config); try { PutObjectResponse response = client.putObject(put.build(), RequestBody.fromFile(staged)); @@ -267,4 +269,24 @@ public class S3OutputSink implements PolicyOutputSink { throw new IllegalStateException("MD5 unavailable", e); } } + + /** + * Write the object under Object Lock retention when the connection asks for it. + * + *

The retain-until date is computed per object from "now", so a policy that runs daily gives + * each document its own full retention window rather than a shared deadline. + * + *

Requires the bucket to have Object Lock enabled; S3 rejects the PUT otherwise, which is + * the correct outcome - silently storing a deletable object while an operator believes it is + * locked would be worse than failing. + */ + private static void applyObjectLock(PutObjectRequest.Builder put, S3Config config) { + if (config.objectLockMode() == null || config.retentionDays() == null) { + return; + } + put.objectLockMode(ObjectLockMode.fromValue(config.objectLockMode())) + .objectLockRetainUntilDate( + java.time.Instant.now() + .plus(config.retentionDays(), java.time.temporal.ChronoUnit.DAYS)); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java index c9d3eabd83..faaa8fe76d 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/s3/S3Config.java @@ -19,7 +19,9 @@ public record S3Config( String endpoint, String accessKeyId, String secretAccessKey, - boolean snapshot) { + boolean snapshot, + String objectLockMode, + Integer retentionDays) { private static final String BUCKET_OPTION = "bucket"; private static final String REGION_OPTION = "region"; @@ -28,6 +30,12 @@ public record S3Config( private static final String ACCESS_KEY_ID_OPTION = "accessKeyId"; private static final String SECRET_ACCESS_KEY_OPTION = "secretAccessKey"; private static final String MODE_OPTION = "mode"; + private static final String OBJECT_LOCK_MODE_OPTION = "objectLockMode"; + private static final String RETENTION_DAYS_OPTION = "retentionDays"; + + private static final String LOCK_GOVERNANCE = "GOVERNANCE"; + private static final String LOCK_COMPLIANCE = "COMPLIANCE"; + private static final int MAX_RETENTION_DAYS = 36525; private static final String MODE_CONSUME = "consume"; private static final String MODE_SNAPSHOT = "snapshot"; @@ -52,6 +60,41 @@ public record S3Config( if (mode != null && !MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) { throw new IllegalArgumentException("s3 config 'mode' must be 'consume' or 'snapshot'"); } + // Object Lock: write-once retention, for records that must survive an administrator. + // COMPLIANCE cannot be shortened or deleted by anyone (not even the account root) before + // the retain-until date; GOVERNANCE can be bypassed with a specific IAM permission, so + // only COMPLIANCE is the answer to SEC 17a-4(f) / FINRA. The bucket must already have + // Object Lock enabled - it cannot be turned on per-object - and that in turn requires + // versioning, which can then never be suspended. + String objectLockMode = trimmed(options.get(OBJECT_LOCK_MODE_OPTION)); + if (objectLockMode != null) { + objectLockMode = objectLockMode.toUpperCase(java.util.Locale.ROOT); + if (!LOCK_GOVERNANCE.equals(objectLockMode) + && !LOCK_COMPLIANCE.equals(objectLockMode)) { + throw new IllegalArgumentException( + "s3 config 'objectLockMode' must be 'GOVERNANCE' or 'COMPLIANCE'"); + } + } + Integer retentionDays = null; + Object rawRetention = options.get(RETENTION_DAYS_OPTION); + if (rawRetention != null && !rawRetention.toString().isBlank()) { + try { + retentionDays = Integer.valueOf(rawRetention.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("s3 config 'retentionDays' must be a number"); + } + if (retentionDays < 1 || retentionDays > MAX_RETENTION_DAYS) { + throw new IllegalArgumentException( + "s3 config 'retentionDays' must be between 1 and " + MAX_RETENTION_DAYS); + } + } + // S3 rejects one without the other, so catch it here where the operator can still fix it + // rather than at upload time on a worker thread. + if ((objectLockMode == null) != (retentionDays == null)) { + throw new IllegalArgumentException( + "s3 config 'objectLockMode' and 'retentionDays' must be set together"); + } + return new S3Config( bucket, region == null ? "us-east-1" : region, @@ -59,7 +102,9 @@ public record S3Config( endpoint, accessKeyId, secretAccessKey, - MODE_SNAPSHOT.equals(mode)); + MODE_SNAPSHOT.equals(mode), + objectLockMode, + retentionDays); } private static String validEndpoint(String endpoint) { diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkObjectLockMinioTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkObjectLockMinioTest.java new file mode 100644 index 0000000000..89d2fe845d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/S3OutputSinkObjectLockMinioTest.java @@ -0,0 +1,183 @@ +package stirling.software.proprietary.policy.output; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ByteArrayResource; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.s3.S3ConnectionPool; +import stirling.software.proprietary.policy.s3.S3TestConnections; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ObjectLockMode; +import software.amazon.awssdk.services.s3.model.S3Exception; + +/** + * Proves the Object Lock (WORM) retention path against a real S3 API. + * + *

The claim being tested is a compliance one - SEC 17a-4(f) and FINRA require records on + * non-rewritable, non-erasable storage - so asserting that we merely send the retention + * headers would be worthless. What matters is that the store then genuinely refuses to delete the + * object, which is what {@link #anObjectWrittenUnderComplianceRetentionCannotBeDeleted} checks. + */ +@Testcontainers(disabledWithoutDocker = true) +class S3OutputSinkObjectLockMinioTest { + + private static final String POLICY = "p1"; + private static final String ACCESS_KEY = "minioadmin"; + private static final String SECRET_KEY = "minioadmin"; + + @Container + static MinIOContainer minio = + new MinIOContainer("minio/minio:latest") + .withUserName(ACCESS_KEY) + .withPassword(SECRET_KEY); + + private static S3Client adminClient; + private static int bucketCounter; + + private String bucket; + private S3OutputSink sink; + + @BeforeEach + void setUp() { + if (adminClient == null) { + adminClient = + S3Client.builder() + .endpointOverride(URI.create(minio.getS3URL())) + .httpClient(UrlConnectionHttpClient.create()) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))) + .serviceConfiguration( + S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + bucket = "worm-archive-" + ++bucketCounter; + // Object Lock can only be enabled at bucket creation here, and implies versioning. + adminClient.createBucket( + CreateBucketRequest.builder() + .bucket(bucket) + .objectLockEnabledForBucket(true) + .build()); + + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateS3Endpoints(true); + sink = + new S3OutputSink( + new S3ConnectionPool(properties), + S3TestConnections.legacyResolver(), + new InProcessProcessedLedger()); + } + + @Test + void anObjectWrittenUnderComplianceRetentionCannotBeDeleted() throws IOException { + List results = + sink.deliver( + new OutputDelivery("run-1", POLICY), + List.of(output("statement.pdf", "regulated record")), + lockedOutputSpec("COMPLIANCE", 7)); + + assertThat(results).hasSize(1); + + HeadObjectResponse head = + adminClient.headObject( + HeadObjectRequest.builder().bucket(bucket).key("statement.pdf").build()); + assertThat(head.objectLockMode()).isEqualTo(ObjectLockMode.COMPLIANCE); + // Retention is computed per object from "now", so a daily policy gives each document its + // own full window rather than a shared deadline. + assertThat(head.objectLockRetainUntilDate()) + .isBetween( + Instant.now().plus(6, ChronoUnit.DAYS), + Instant.now().plus(8, ChronoUnit.DAYS)); + + // The point of the feature: the store itself refuses, not us. + assertThatThrownBy( + () -> + adminClient.deleteObject( + DeleteObjectRequest.builder() + .bucket(bucket) + .key("statement.pdf") + .versionId(head.versionId()) + .build())) + .isInstanceOf(S3Exception.class); + + // And it is still readable - locked, not quarantined. + assertThat( + adminClient + .getObject( + GetObjectRequest.builder() + .bucket(bucket) + .key("statement.pdf") + .build()) + .response() + .contentLength()) + .isEqualTo("regulated record".length()); + } + + @Test + void withoutRetentionConfiguredObjectsAreWrittenUnlockedAsBefore() throws IOException { + sink.deliver( + new OutputDelivery("run-2", POLICY), + List.of(output("scratch.pdf", "ordinary output")), + lockedOutputSpec(null, null)); + + HeadObjectResponse head = + adminClient.headObject( + HeadObjectRequest.builder().bucket(bucket).key("scratch.pdf").build()); + // No accidental retention: an Object-Lock-enabled bucket must not silently lock everything. + assertThat(head.objectLockMode()).isNull(); + assertThat(head.objectLockRetainUntilDate()).isNull(); + } + + private OutputSpec lockedOutputSpec(String lockMode, Integer retentionDays) { + Map options = new java.util.LinkedHashMap<>(); + options.put("bucket", bucket); + options.put("prefix", ""); + options.put("endpoint", minio.getS3URL()); + options.put("accessKeyId", ACCESS_KEY); + options.put("secretAccessKey", SECRET_KEY); + if (lockMode != null) { + options.put("objectLockMode", lockMode); + options.put("retentionDays", String.valueOf(retentionDays)); + } + return new OutputSpec("s3", options); + } + + private static org.springframework.core.io.Resource output(String name, String content) { + return new ByteArrayResource(content.getBytes(StandardCharsets.UTF_8)) { + @Override + public String getFilename() { + return name; + } + }; + } +}