Stirling-Tools/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2026-09-03 05:10:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into custom_task_20260812

Browse Source
This commit is contained in:
Ludy
2026-08-16 15:47:45 +02:00
committed by GitHub
parent 8f2a967141 beea37cb92
commit 9ff56af1f4
507 changed files with 17039 additions and 9726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/dependabot.yml
+2
View File
@@ -11,6 +11,8 @@ updates:
- "/app/common"
- "/app/core"
- "/app/proprietary"
- "/app/saas"
- "/buildSrc"
schedule:
interval: "weekly"
cooldown:
.github/workflows/PR-Auto-Deploy-V2.yml
+52 -40
View File
@@ -26,6 +26,10 @@ jobs:
check-pr:
if: (github.event_name == 'pull_request' && github.event.action != 'closed') || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
# Only reads the PR via pulls.get with the default GITHUB_TOKEN.
permissions:
contents: read
pull-requests: read
outputs:
should_deploy: ${{ steps.decide.outputs.should_deploy }}
is_fork: ${{ steps.resolve.outputs.is_fork }}
@@ -35,7 +39,7 @@ jobs:
pr_ref: ${{ steps.resolve.outputs.ref }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -97,6 +101,7 @@ jobs:
echo "allow_fork=${allow_fork:-false}" >> $GITHUB_OUTPUT
deploy-v2-pr:
environment: pr-preview
needs: check-pr
runs-on: ubuntu-latest
if: needs.check-pr.outputs.should_deploy == 'true' && (needs.check-pr.outputs.is_fork == 'false' || needs.check-pr.outputs.allow_fork == 'true')
@@ -107,6 +112,7 @@ jobs:
permissions:
contents: read
issues: write
packages: write
pull-requests: write
env:
# Single source of truth for whether this preview embeds the admin portal:
@@ -115,7 +121,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -125,20 +131,11 @@ jobs:
repository: ${{ github.repository }}
ref: main
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Add deployment started comment
id: deployment-started
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const { owner, repo } = context.repo;
const prNumber = ${{ needs.check-pr.outputs.pr_number }};
@@ -180,7 +177,8 @@ jobs:
with:
repository: ${{ needs.check-pr.outputs.pr_repository }}
ref: ${{ needs.check-pr.outputs.pr_ref }}
token: ${{ secrets.GITHUB_TOKEN }}
# untrusted tree is built below - never leave credentials in .git/config
persist-credentials: false
fetch-depth: 0 # Fetch full history for commit hash detection
- name: Set up Docker Buildx
@@ -192,11 +190,16 @@ jobs:
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Get commit hash for app
id: commit-hash
@@ -220,7 +223,7 @@ jobs:
- name: Check if image exists
id: check-image
run: |
if docker manifest inspect ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }} >/dev/null 2>&1; then
if docker manifest inspect ${IMAGE_BASE}:v2-${{ steps.commit-hash.outputs.app_short }} >/dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
echo "Image already exists, skipping build"
else
@@ -228,6 +231,8 @@ jobs:
echo "Image needs to be built"
fi
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
- name: Build and push V2 image
if: steps.check-image.outputs.exists == 'false'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
@@ -237,7 +242,7 @@ jobs:
push: true
cache-from: type=gha,scope=stirling-pdf-latest
cache-to: type=gha,mode=max,scope=stirling-pdf-latest
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }}
tags: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:v2-${{ steps.commit-hash.outputs.app_short }}
build-args: |
VERSION_TAG=v2-alpha
BUILD_PORTAL=${{ env.BUILD_PORTAL }}
@@ -246,9 +251,11 @@ jobs:
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Deploy V2 to VPS
id: deploy
run: |
@@ -261,7 +268,7 @@ jobs:
services:
stirling-pdf-v2:
container_name: stirling-pdf-v2-pr-${{ needs.check-pr.outputs.pr_number }}
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }}
image: ${IMAGE_BASE}:v2-${{ steps.commit-hash.outputs.app_short }}
ports:
- "${V2_PORT}:8080"
volumes:
@@ -273,8 +280,8 @@ jobs:
DISABLE_ADDITIONAL_FEATURES: "false"
STIRLING_BILLING_ACCOUNT_LINK_ENABLED: "true"
SECURITY_ENABLELOGIN: "true"
SECURITY_INITIALLOGIN_USERNAME: "${{ secrets.TEST_LOGIN_USERNAME }}"
SECURITY_INITIALLOGIN_PASSWORD: "${{ secrets.TEST_LOGIN_PASSWORD }}"
SECURITY_INITIALLOGIN_USERNAME: "${TEST_LOGIN_USERNAME}"
SECURITY_INITIALLOGIN_PASSWORD: "${TEST_LOGIN_PASSWORD}"
SYSTEM_DEFAULTLOCALE: en-US
UI_APPNAME: "Stirling-PDF V2 PR#${{ needs.check-pr.outputs.pr_number }}"
UI_HOMEDESCRIPTION: "V2 PR#${{ needs.check-pr.outputs.pr_number }} - Embedded Architecture"
@@ -288,9 +295,9 @@ jobs:
EOF
# Deploy to VPS
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }}:/tmp/docker-compose-v2.yml
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${NEW_VPS_USERNAME}@${NEW_VPS_HOST}:/tmp/docker-compose-v2.yml
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << ENDSSH
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << ENDSSH
# Create V2 PR-specific directories
mkdir -p /stirling/V2-PR-${{ needs.check-pr.outputs.pr_number }}/{data,config,logs,storage}
@@ -315,12 +322,19 @@ jobs:
# Set port for output
echo "v2_port=${V2_PORT}" >> $GITHUB_OUTPUT
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
TEST_LOGIN_USERNAME: ${{ secrets.TEST_LOGIN_USERNAME }}
TEST_LOGIN_PASSWORD: ${{ secrets.TEST_LOGIN_PASSWORD }}
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
# ---- Storybook preview (only when this PR touches stories/.storybook) ----
# Runs inside the same approved-contributor-gated deploy job, so it deploys
# under the exact same access rules as the app preview.
- name: Detect Storybook changes
id: sb-changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
with:
list-files: json
filters: |
@@ -379,8 +393,9 @@ jobs:
env:
SB_URL: ${{ steps.storybook.outputs.url }}
SB_FILES: ${{ steps.sb-changes.outputs.storybook_files }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const { owner, repo } = context.repo;
const prNumber = ${{ needs.check-pr.outputs.pr_number }};
@@ -401,7 +416,7 @@ jobs:
}
}
const deploymentUrl = `http://${{ secrets.NEW_VPS_HOST }}:${v2Port}`;
const deploymentUrl = `http://${process.env.NEW_VPS_HOST}:${v2Port}`;
// Only mention the portal when this image actually embeds it.
// Use the direct IP URL - the SSL hostname isn't supported yet.
@@ -447,6 +462,7 @@ jobs:
});
cleanup-v2-deployment:
environment: pr-preview
if: github.event.action == 'closed'
runs-on: ubuntu-latest
permissions:
@@ -456,26 +472,17 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Clean up V2 deployment comments
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const { owner, repo } = context.repo;
const prNumber = ${{ github.event.pull_request.number }};
@@ -504,12 +511,14 @@ jobs:
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Cleanup V2 deployment
run: |
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << 'ENDSSH'
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << 'ENDSSH'
if [ -d "/stirling/V2-PR-${{ github.event.pull_request.number }}" ]; then
echo "Found V2 PR directory, proceeding with cleanup..."
@@ -542,6 +551,9 @@ jobs:
# Only remove PR-specific containers and directories
ENDSSH
env:
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
- name: Cleanup temporary files
if: always()
run: |
.github/workflows/PR-Demo-Comment-with-react.yml
+51 -52
View File
@@ -37,7 +37,8 @@ jobs:
check-comment:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read # actions/checkout
issues: write # add reaction to the triggering issue comment
if: |
vars.CI_PROFILE != 'lite' && (
github.event_name == 'workflow_dispatch' ||
@@ -69,22 +70,13 @@ jobs:
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout PR
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Get PR data
id: get-pr
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -155,7 +147,7 @@ jobs:
id: add-eyes-reaction
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
console.log(`Adding eyes reaction to comment ID: ${context.payload.comment.id}`);
try {
@@ -174,35 +166,30 @@ jobs:
}
deploy-pr:
environment: pr-preview
needs: check-comment
runs-on: ubuntu-latest
permissions:
issues: write
contents: read # actions/checkout, incl. the PR merge ref
issues: write # reactions, 'pr-deployed' label, deployment URL comment
pull-requests: write
packages: write # push PR image to ghcr.io
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout PR
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Checkout PR
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: refs/pull/${{ needs.check-comment.outputs.pr_number }}/merge
token: ${{ steps.setup-bot.outputs.token }}
# untrusted tree gets built below - never leave credentials in .git/config
persist-credentials: false
- name: Set up JDK 25
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
@@ -216,7 +203,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -240,11 +227,16 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to Docker Hub
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Build and push PR-specific image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
@@ -254,7 +246,7 @@ jobs:
push: true
cache-from: type=gha,scope=stirling-pdf-latest
cache-to: type=gha,mode=max,scope=stirling-pdf-latest
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
tags: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:pr-${{ needs.check-comment.outputs.pr_number }}
build-args: |
VERSION_TAG=alpha
PROTOTYPES_BUILD=${{ needs.check-comment.outputs.enable_prototypes }}
@@ -269,15 +261,17 @@ jobs:
push: true
cache-from: type=gha,scope=stirling-pdf-engine
cache-to: type=gha,mode=max,scope=stirling-pdf-engine
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:engine-pr-${{ needs.check-comment.outputs.pr_number }}
tags: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:engine-pr-${{ needs.check-comment.outputs.pr_number }}
platforms: linux/amd64
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Deploy to VPS
id: deploy
run: |
@@ -295,11 +289,11 @@ jobs:
# Set pro/enterprise settings (enterprise implies pro)
if [ "${{ needs.check-comment.outputs.enable_enterprise }}" == "true" ]; then
PREMIUM_ENABLED="true"
PREMIUM_KEY="${{ secrets.ENTERPRISE_KEY }}"
PREMIUM_KEY="${ENTERPRISE_KEY}"
PREMIUM_PROFEATURES_AUDIT_ENABLED="true"
elif [ "${{ needs.check-comment.outputs.enable_pro }}" == "true" ]; then
PREMIUM_ENABLED="true"
PREMIUM_KEY="${{ secrets.PREMIUM_KEY }}"
PREMIUM_KEY="${PRO_KEY}"
PREMIUM_PROFEATURES_AUDIT_ENABLED="true"
else
PREMIUM_ENABLED="false"
@@ -309,7 +303,6 @@ jobs:
ENABLE_PROTOTYPES="${{ needs.check-comment.outputs.enable_prototypes }}"
PR_NUMBER="${{ needs.check-comment.outputs.pr_number }}"
DOCKER_USER="${{ secrets.DOCKER_HUB_USERNAME }}"
# Build engine env vars for backend (only set when prototypes enabled)
if [ "$ENABLE_PROTOTYPES" == "true" ]; then
@@ -319,9 +312,9 @@ jobs:
ENGINE_SERVICE="
stirling-pdf-engine:
container_name: stirling-pdf-engine-pr-${PR_NUMBER}
image: ${DOCKER_USER}/test:engine-pr-${PR_NUMBER}
image: ${IMAGE_BASE}:engine-pr-${PR_NUMBER}
environment:
ANTHROPIC_API_KEY: \"${{ secrets.ANTHROPIC_API_KEY }}\"
ANTHROPIC_API_KEY: \"${ANTHROPIC_API_KEY}\"
networks:
- pr-network
restart: on-failure:5"
@@ -344,7 +337,7 @@ jobs:
services:
stirling-pdf:
container_name: stirling-pdf-pr-${PR_NUMBER}
image: ${DOCKER_USER}/test:pr-${PR_NUMBER}
image: ${IMAGE_BASE}:pr-${PR_NUMBER}
ports:
- "${PR_NUMBER}:8080"
volumes:
@@ -368,9 +361,9 @@ jobs:
EOF
# Then copy the file and execute commands
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }}:/tmp/docker-compose.yml
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${NEW_VPS_USERNAME}@${NEW_VPS_HOST}:/tmp/docker-compose.yml
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << ENDSSH
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << ENDSSH
# Create PR-specific directories
mkdir -p /stirling/PR-${PR_NUMBER}/{data,config,logs}
@@ -386,11 +379,19 @@ jobs:
# Set output for use in PR comment
echo "security_status=${SECURITY_STATUS}" >> $GITHUB_ENV
env:
ENTERPRISE_KEY: ${{ secrets.ENTERPRISE_KEY }}
# named PRO_KEY, not PREMIUM_KEY, so the shell var it feeds is not self-referential
PRO_KEY: ${{ secrets.PREMIUM_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
- name: Add success reaction to comment
if: success() && github.event_name == 'issue_comment'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
console.log(`Adding rocket reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
try {
@@ -425,7 +426,7 @@ jobs:
if: failure() && github.event_name == 'issue_comment'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
console.log(`Adding -1 reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
try {
@@ -444,15 +445,17 @@ jobs:
- name: Post deployment URL to PR
if: success()
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const { GITHUB_REPOSITORY } = process.env;
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
const prNumber = ${{ needs.check-comment.outputs.pr_number }};
const securityStatus = process.env.security_status || "Security Disabled";
const deploymentUrl = `http://${{ secrets.NEW_VPS_HOST }}:${prNumber}`;
const deploymentUrl = `http://${process.env.NEW_VPS_HOST}:${prNumber}`;
const commentBody = `## 🚀 PR Test Deployment\n\n` +
`Your PR has been deployed for testing!\n\n` +
`🔗 **Test URL:** [${deploymentUrl}](${deploymentUrl})\n` +
@@ -477,26 +480,22 @@ jobs:
handle-label-commands:
if: ${{ github.event.issue.pull_request != null }}
runs-on: ubuntu-latest
permissions:
contents: read # actions/checkout, reads repo_devs.json and labels.yml
issues: write # add/remove labels, delete the command comment
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Check out the repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Apply label commands
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const fs = require('fs');
const path = require('path');
.github/workflows/PR-Demo-cleanup.yml
+20 -16
View File
@@ -13,35 +13,28 @@ env:
jobs:
cleanup:
environment: pr-preview
if: github.event.action == 'closed'
runs-on: ubuntu-latest
permissions:
contents: read # actions/checkout
pull-requests: write
issues: write
issues: write # list/remove labels, list/delete comments
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout PR
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Remove 'pr-deployed' label if present
id: remove-label-comment
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const prNumber = ${{ github.event.pull_request.number }};
const owner = context.repo.owner;
@@ -100,14 +93,22 @@ jobs:
if: steps.remove-label-comment.outputs.present == 'true'
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Cleanup PR deployment
if: steps.remove-label-comment.outputs.present == 'true'
id: cleanup
# ENDSSH heredoc is quoted, so its body is sent literally: secrets inside it
# must stay as GitHub expressions, a shell var would be empty on the remote host.
run: |
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << 'ENDSSH'
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << 'ENDSSH'
if [ -d "/stirling/PR-${{ github.event.pull_request.number }}" ]; then
echo "Found PR directory, proceeding with cleanup..."
@@ -122,8 +123,8 @@ jobs:
rm -rf /stirling/PR-${{ github.event.pull_request.number }}
# Remove the Docker images
docker rmi --no-prune ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ github.event.pull_request.number }} || true
docker rmi --no-prune ${{ secrets.DOCKER_HUB_USERNAME }}/test:engine-pr-${{ github.event.pull_request.number }} || true
docker rmi --no-prune ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:pr-${{ github.event.pull_request.number }} || true
docker rmi --no-prune ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:engine-pr-${{ github.event.pull_request.number }} || true
echo "PERFORMED_CLEANUP"
else
@@ -131,6 +132,9 @@ jobs:
echo "NO_CLEANUP_NEEDED"
fi
ENDSSH
env:
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
- name: Cleanup temporary files
if: always()
.github/workflows/_runner-pick.yml
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
is_fork: ${{ steps.decide.outputs.is_fork }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/ai-engine.yml
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/ai_pr_title_review.yml
+8 -15
View File
@@ -10,16 +10,18 @@ permissions: # required for secure-repo hardening
jobs:
ai-title-review:
# GITHUB_TOKEN obeys this block, so it must cover every API call made below.
permissions:
contents: read
pull-requests: write
models: read
contents: read # actions/checkout, git fetch/diff
issues: write # issues.listComments / createComment / updateComment on the PR
pull-requests: write # same endpoints when the target is a pull request
models: read # actions/ai-inference
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -30,15 +32,6 @@ jobs:
- name: Configure Git to suppress detached HEAD warning
run: git config --global advice.detachedHead false
- name: Setup GitHub App Bot
if: github.actor != 'dependabot[bot]'
id: setup-bot
uses: ./.github/actions/setup-bot
continue-on-error: true
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Check if actor is repo developer
id: actor
run: |
@@ -161,7 +154,7 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
continue-on-error: true
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const fs = require('fs');
const body = fs.readFileSync('/tmp/ai-title-comment.md', 'utf8');
@@ -172,7 +165,7 @@ jobs:
const ratingMatch = body.match(/\*\*PR-Title Rating\*\*: (\d+)\/10/);
const rating = ratingMatch ? parseInt(ratingMatch[1], 10) : null;
const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]";
const expectedActor = "github-actions[bot]";
const comments = await github.rest.issues.listComments({ owner, repo, issue_number });
const existing = comments.data.find(c =>
.github/workflows/aur-publish.yml
+3 -2
View File
@@ -26,7 +26,7 @@ jobs:
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -66,11 +66,12 @@ jobs:
echo "jar_sha256=$JAR_SHA" >> "$GITHUB_OUTPUT"
publish-aur:
environment: package-publish
needs: get-release-info
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/auto-labelerV2.yml
+5 -10
View File
@@ -13,26 +13,21 @@ jobs:
labeler:
runs-on: ubuntu-latest
permissions:
pull-requests: write
contents: read # checkout + labeler fetching its config from the repo
pull-requests: write # read changed files, apply labels to the PR
issues: write # labels are applied through the issues API
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0
with:
config_path: .github/labeler-config-srvaroa.yml
use_local_config: false
fail_on_error: true
env:
GITHUB_TOKEN: "${{ steps.setup-bot.outputs.token }}"
GITHUB_TOKEN: "${{ github.token }}"
.github/workflows/backend-build.yml
+2 -2
View File
@@ -28,7 +28,7 @@ jobs:
flavor: [core, proprietary, saas]
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
@@ -46,7 +46,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-${{ matrix.jdk-version }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-${{ matrix.jdk-version }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-${{ matrix.jdk-version }}-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/build-enterprise.yml
+3 -3
View File
@@ -50,7 +50,7 @@ jobs:
SYSTEM_ENABLEANALYTICS: "false"
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
@@ -66,7 +66,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -324,7 +324,7 @@ jobs:
MN_COMPOSE: docker-compose-multinode.yml
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
.github/workflows/build.yml
+11 -8
View File
@@ -48,14 +48,14 @@ jobs:
proprietary: ${{ steps.changes.outputs.proprietary }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Check for file changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: changes
with:
filters: .github/config/.files.yaml
@@ -67,7 +67,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
@@ -83,7 +83,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -134,9 +134,10 @@ jobs:
uses: ./.github/workflows/frontend-validation.yml
secrets: inherit
# Advisory: deliberately NOT in all-checks-passed. It reports on the stories a
# branch touches so a regression is visible in review, but a browser scan is
# too new here to block merges on. Promote it once its pass/fail proves stable.
# Required (in all-checks-passed). Scans the stories a branch touches in both
# light and dark; an axe violation in either theme blocks the merge. The
# whole-suite sweep (nightly.yml) still covers stories a change affects without
# touching them directly.
frontend-a11y:
if: needs.files-changed.outputs.frontend == 'true'
needs: [files-changed]
@@ -290,6 +291,7 @@ jobs:
- db-migration-test
- check-generateOpenApiDocs
- frontend-validation
- frontend-a11y
- playwright-e2e
- playwright-e2e-live
- playwright-e2e-enterprise
@@ -304,7 +306,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -316,6 +318,7 @@ jobs:
db-migration-test=${{ needs.db-migration-test.result }}
check-generateOpenApiDocs=${{ needs.check-generateOpenApiDocs.result }}
frontend-validation=${{ needs.frontend-validation.result }}
frontend-a11y=${{ needs.frontend-a11y.result }}
playwright-e2e=${{ needs.playwright-e2e.result }}
playwright-e2e-live=${{ needs.playwright-e2e-live.result }}
playwright-e2e-enterprise=${{ needs.playwright-e2e-enterprise.result }}
.github/workflows/check-generated-models.yml
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -51,7 +51,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/check-licence.yml
+2 -2
View File
@@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -32,7 +32,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/check-openapi.yml
+2 -2
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -33,7 +33,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/check_toml.yml
+32 -31
View File
@@ -23,29 +23,23 @@ jobs:
if: github.event_name == 'pull_request_target'
runs-on: ubuntu-latest
permissions:
contents: read # Checkout, and read translation files via the contents API
issues: write # Allow posting comments on issues/PRs
pull-requests: write # Allow writing to pull requests
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout main branch first
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup GitHub App Bot
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Get PR data
id: get-pr-data
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const prNumber = context.payload.pull_request.number;
const repoOwner = context.payload.repository.owner.login;
@@ -66,17 +60,18 @@ jobs:
- name: Fetch PR changed files
id: fetch-pr-changes
env:
GH_TOKEN: ${{ steps.setup-bot.outputs.token }}
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ steps.get-pr-data.outputs.pr_number }}
run: |
echo "Fetching PR changed files..."
echo "Getting list of changed files from PR..."
# Check if PR number exists
if [ -z "${{ steps.get-pr-data.outputs.pr_number }}" ]; then
if [ -z "${PR_NUMBER}" ]; then
echo "Error: PR number is empty"
exit 1
fi
# Get changed files and filter for TOML translation files
gh pr view ${{ steps.get-pr-data.outputs.pr_number }} --json files -q ".files[].path" | grep -E '^frontend/editor/public/locales/[a-zA-Z-]+/translation\.toml$' > changed_files.txt || echo "No matching TOML files found in PR"
gh pr view "${PR_NUMBER}" --json files -q ".files[].path" | grep -E '^frontend/editor/public/locales/[a-zA-Z-]+/translation\.toml$' > changed_files.txt || echo "No matching TOML files found in PR"
# Check if any files were found
if [ ! -s changed_files.txt ]; then
echo "No TOML translation files changed in this PR"
@@ -88,32 +83,36 @@ jobs:
- name: Determine reference file
id: determine-file
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
# Untrusted, fork-controlled values are passed via env, never interpolated into the script
PR_NUMBER: ${{ steps.get-pr-data.outputs.pr_number }}
REPO_OWNER: ${{ steps.get-pr-data.outputs.repo_owner }}
REPO_NAME: ${{ steps.get-pr-data.outputs.repo_name }}
PR_REPO_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
PR_REPO_NAME: ${{ github.event.pull_request.head.repo.name }}
PR_BRANCH: ${{ steps.get-pr-data.outputs.branch }}
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const fs = require("fs");
const path = require("path");
const prNumber = ${{ steps.get-pr-data.outputs.pr_number }};
const repoOwner = "${{ steps.get-pr-data.outputs.repo_owner }}";
const repoName = "${{ steps.get-pr-data.outputs.repo_name }}";
const prRepoOwner = "${{ github.event.pull_request.head.repo.owner.login }}";
const prRepoName = "${{ github.event.pull_request.head.repo.name }}";
const branch = "${{ steps.get-pr-data.outputs.branch }}";
console.log(`Determining reference file for PR #${prNumber}`);
// Validate inputs
// Validate inputs before any use
const validateInput = (input, regex, name) => {
if (!regex.test(input)) {
if (typeof input !== "string" || !regex.test(input)) {
throw new Error(`Invalid ${name}: ${input}`);
}
return input;
};
validateInput(repoOwner, /^[a-zA-Z0-9_-]+$/, "repository owner");
validateInput(repoName, /^[a-zA-Z0-9._-]+$/, "repository name");
validateInput(branch, /^[a-zA-Z0-9._/-]+$/, "branch name");
const repoOwner = validateInput(process.env.REPO_OWNER, /^[a-zA-Z0-9_-]+$/, "repository owner");
const repoName = validateInput(process.env.REPO_NAME, /^[a-zA-Z0-9._-]+$/, "repository name");
const prRepoOwner = validateInput(process.env.PR_REPO_OWNER, /^[a-zA-Z0-9_-]+$/, "PR repository owner");
const prRepoName = validateInput(process.env.PR_REPO_NAME, /^[a-zA-Z0-9._-]+$/, "PR repository name");
const branch = validateInput(process.env.PR_BRANCH, /^[a-zA-Z0-9._/-]+$/, "branch name");
const prNumber = Number(validateInput(process.env.PR_NUMBER, /^[0-9]+$/, "PR number"));
console.log(`Determining reference file for PR #${prNumber}`);
// Get the list of changed files in the PR
const { data: files } = await github.rest.pulls.listFiles({
@@ -209,10 +208,12 @@ jobs:
- name: Run Python script to check files
id: run-check
env:
PR_ACTOR: ${{ github.event.pull_request.user.login }}
run: |
echo "Running Python script to check TOML files..."
uv run --project engine --locked --group tools python .github/scripts/check_language_toml.py \
--actor ${{ github.event.pull_request.user.login }} \
--actor "${PR_ACTOR}" \
--reference-file "${REFERENCE_FILE}" \
--branch "pr-branch" \
--files "${FILES_LIST[@]}" > result.txt
@@ -245,7 +246,7 @@ jobs:
if: env.SCRIPT_OUTPUT != ''
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const { GITHUB_REPOSITORY, SCRIPT_OUTPUT } = process.env;
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
@@ -261,7 +262,7 @@ jobs:
const comment = comments.data.find(c => c.body.includes("## 🌐 TOML Translation Verification Summary"));
// Only update or create comments by the action user
const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]";
const expectedActor = "github-actions[bot]";
if (comment && comment.user.login === expectedActor) {
// Update existing comment
.github/workflows/coverage-aggregate.yml
+2 -2
View File
@@ -34,7 +34,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
@@ -52,7 +52,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/db-migration-test.yml
+2 -2
View File
@@ -17,7 +17,7 @@ jobs:
timeout-minutes: 30
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -36,7 +36,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/dependency-review.yml
+1 -1
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/deploy-on-v2-commit.yml
+38 -18
View File
@@ -11,14 +11,18 @@ permissions:
jobs:
deploy-v2-on-push:
environment: pr-preview
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
concurrency:
group: deploy-v2-push-V2
cancel-in-progress: true
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -62,10 +66,21 @@ jobs:
echo "backend_short=${BACKEND_HASH:0:8}" >> $GITHUB_OUTPUT
fi
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Check if frontend image exists
id: check-frontend
run: |
if docker manifest inspect ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }} >/dev/null 2>&1; then
if docker manifest inspect ${IMAGE_BASE}:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }} >/dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
echo "Frontend image already exists, skipping build"
else
@@ -73,10 +88,12 @@ jobs:
echo "Frontend image needs to be built"
fi
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
- name: Check if backend image exists
id: check-backend
run: |
if docker manifest inspect ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-${{ steps.commit-hashes.outputs.backend_short }} >/dev/null 2>&1; then
if docker manifest inspect ${IMAGE_BASE}:v2-backend-${{ steps.commit-hashes.outputs.backend_short }} >/dev/null 2>&1; then
echo "exists=true" >> $GITHUB_OUTPUT
echo "Backend image already exists, skipping build"
else
@@ -84,11 +101,8 @@ jobs:
echo "Backend image needs to be built"
fi
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
- name: Build and push frontend image
if: steps.check-frontend.outputs.exists == 'false'
@@ -100,8 +114,8 @@ jobs:
cache-from: type=gha,scope=stirling-v2-frontend
cache-to: type=gha,mode=max,scope=stirling-v2-frontend
tags: |
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }}
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-latest
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }}
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:v2-frontend-latest
build-args: VERSION_TAG=v2-alpha
platforms: linux/amd64
@@ -115,17 +129,19 @@ jobs:
cache-from: type=gha,scope=stirling-v2-backend
cache-to: type=gha,mode=max,scope=stirling-v2-backend
tags: |
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-${{ steps.commit-hashes.outputs.backend_short }}
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-latest
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:v2-backend-${{ steps.commit-hashes.outputs.backend_short }}
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:v2-backend-latest
build-args: VERSION_TAG=v2-alpha
platforms: linux/amd64
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Deploy to VPS on port 3000
run: |
export UNIQUE_NAME=docker-compose-v2-$GITHUB_RUN_ID.yml
@@ -135,7 +151,7 @@ jobs:
services:
backend:
container_name: stirling-v2-backend
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-${{ steps.commit-hashes.outputs.backend_short }}
image: ${IMAGE_BASE}:v2-backend-${{ steps.commit-hashes.outputs.backend_short }}
ports:
- "13000:8080"
volumes:
@@ -158,21 +174,21 @@ jobs:
frontend:
container_name: stirling-v2-frontend
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }}
image: ${IMAGE_BASE}:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }}
ports:
- "3000:80"
environment:
VITE_API_BASE_URL: "http://${{ secrets.NEW_VPS_HOST }}:13000"
VITE_API_BASE_URL: "http://${NEW_VPS_HOST}:13000"
depends_on:
- backend
restart: on-failure:5
EOF
# Copy to remote with unique name
scp -i ../private.key -o StrictHostKeyChecking=no $UNIQUE_NAME ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }}:/tmp/$UNIQUE_NAME
scp -i ../private.key -o StrictHostKeyChecking=no $UNIQUE_NAME ${NEW_VPS_USERNAME}@${NEW_VPS_HOST}:/tmp/$UNIQUE_NAME
# SSH and rename/move atomically to avoid interference
ssh -i ../private.key -o StrictHostKeyChecking=no ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << ENDSSH
ssh -i ../private.key -o StrictHostKeyChecking=no ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << ENDSSH
mkdir -p /stirling/V2/{data,config,logs}
mv /tmp/$UNIQUE_NAME /stirling/V2/docker-compose.yml
cd /stirling/V2
@@ -183,6 +199,10 @@ jobs:
docker image prune -af --filter "until=336h" --filter "label!=keep=true" || true
ENDSSH
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
- name: Cleanup temporary files
if: always()
run: |
.github/workflows/docker-compose-tests.yml
+2 -2
View File
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -44,7 +44,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/e2e-live.yml
+2 -2
View File
@@ -15,7 +15,7 @@ jobs:
timeout-minutes: 30
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
@@ -31,7 +31,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/e2e-stubbed.yml
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
project: stubbed-webkit
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
.github/workflows/frontend-a11y.yml
+3 -8
View File
@@ -3,17 +3,12 @@ name: Frontend a11y regression gate
# Reusable workflow called from build.yml when frontend sources change.
#
# Scans the stories this branch touches in real Chromium and runs axe against
# each. Existing violations are grandfathered in .storybook/a11y-baseline.json;
# the check fails on a NEW violation — a story breaking a rule it wasn't already
# breaking — or on a story that fails to render at all.
# each; the check fails on any axe violation, or on a story that fails to render
# at all.
#
# Only changed stories, because a full sweep is ~30 minutes: far too slow to sit
# in front of every merge. The whole suite is scanned nightly instead
# (nightly.yml), which catches anything a branch didn't touch.
#
# Advisory for now: this is not in build.yml's all-checks-passed list, so a
# failure reports without blocking. Promote it once a few weeks of runs show the
# pass/fail is stable.
on:
workflow_call:
@@ -26,7 +21,7 @@ jobs:
timeout-minutes: 25
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
.github/workflows/frontend-backend-licenses-update.yml
+5 -5
View File
@@ -28,7 +28,7 @@ jobs:
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -36,7 +36,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Check for file changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: changes
with:
filters: .github/config/.files.yaml
@@ -52,7 +52,7 @@ jobs:
repository-projects: write # Required for enabling automerge
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -326,7 +326,7 @@ jobs:
repository-projects: write # Required for enabling automerge
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -356,7 +356,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/frontend-validation.yml
+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
.github/workflows/manage-label.yml
+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
issues: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/multiOSReleases.yml
+22 -13
View File
@@ -45,7 +45,7 @@ jobs:
version: ${{ steps.versionNumber.outputs.versionNumber }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -63,7 +63,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -93,7 +93,7 @@ jobs:
ALL="$WINDOWS,$WINDOWS_ARM64,$MACOS,$LINUX"
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
case "${{ github.event.inputs.platform }}" in
case "${INPUT_PLATFORM}" in
"windows")
echo "matrix={\"include\":[$WINDOWS,$WINDOWS_ARM64]}" >> $GITHUB_OUTPUT
;;
@@ -115,6 +115,8 @@ jobs:
echo "matrix={\"include\":[$ALL]}" >> $GITHUB_OUTPUT
fi
env:
INPUT_PLATFORM: ${{ github.event.inputs.platform }}
build-jars:
needs: determine-matrix
runs-on: ubuntu-latest
@@ -135,7 +137,7 @@ jobs:
file_suffix: "-server"
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -153,7 +155,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -194,6 +196,7 @@ jobs:
retention-days: 1
build:
environment: release-signing
needs: determine-matrix
strategy:
fail-fast: false
@@ -205,7 +208,7 @@ jobs:
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
allowed-endpoints: >
@@ -263,7 +266,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -308,16 +311,16 @@ jobs:
Write-Host "Setting up DigiCert KeyLocker environment..."
# Decode client certificate
$certBytes = [Convert]::FromBase64String("${{ secrets.SM_CLIENT_CERT_FILE_B64 }}")
$certBytes = [Convert]::FromBase64String("$env:SM_CLIENT_CERT_FILE_B64")
$certPath = "D:\Certificate_pkcs12.p12"
[IO.File]::WriteAllBytes($certPath, $certBytes)
# Set environment variables
echo "SM_CLIENT_CERT_FILE=D:\Certificate_pkcs12.p12" >> $env:GITHUB_ENV
echo "SM_HOST=${{ secrets.SM_HOST }}" >> $env:GITHUB_ENV
echo "SM_API_KEY=${{ secrets.SM_API_KEY }}" >> $env:GITHUB_ENV
echo "SM_CLIENT_CERT_PASSWORD=${{ secrets.SM_CLIENT_CERT_PASSWORD }}" >> $env:GITHUB_ENV
echo "SM_KEYPAIR_ALIAS=${{ secrets.SM_KEYPAIR_ALIAS }}" >> $env:GITHUB_ENV
echo "SM_HOST=$env:SM_HOST" >> $env:GITHUB_ENV
echo "SM_API_KEY=$env:SM_API_KEY" >> $env:GITHUB_ENV
echo "SM_CLIENT_CERT_PASSWORD=$env:SM_CLIENT_CERT_PASSWORD" >> $env:GITHUB_ENV
echo "SM_KEYPAIR_ALIAS=$env:SM_KEYPAIR_ALIAS" >> $env:GITHUB_ENV
# Get PKCS11 config path from DigiCert action
$pkcs11Config = $env:PKCS11_CONFIG
@@ -335,6 +338,12 @@ jobs:
}
}
env:
SM_CLIENT_CERT_FILE_B64: ${{ secrets.SM_CLIENT_CERT_FILE_B64 }}
SM_HOST: ${{ secrets.SM_HOST }}
SM_API_KEY: ${{ secrets.SM_API_KEY }}
SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
SM_KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
# Traditional PFX Certificate Import (fallback if KeyLocker not configured)
- name: Import Windows Code Signing Certificate
if: ${{ startsWith(matrix.platform, 'windows') && env.SM_API_KEY == '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master') }}
@@ -731,7 +740,7 @@ jobs:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/nightly.yml
+91 -8
View File
@@ -4,6 +4,11 @@ on:
schedule:
- cron: "0 2 * * *" # 2 AM UTC every night
workflow_dispatch:
pull_request:
paths:
- .github/workflows/nightly.yml
- testing/cucumber/**
- docker/embedded/compose/test_cicd.yml
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,7 +23,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -59,13 +64,17 @@ jobs:
# the story itself — a shared component, a theme token — still surfaces within
# a day.
a11y-all-stories:
name: a11y (every story, light + dark)
name: a11y (every story)
strategy:
fail-fast: false
matrix:
theme: [light, dark]
runs-on: ubuntu-latest
# Two full sweeps (one per theme), each ~30 minutes of browser time.
timeout-minutes: 120
# One full sweep (~30 minutes of browser time).
timeout-minutes: 60
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -82,14 +91,14 @@ jobs:
- name: Install Task
uses: ./.github/actions/setup-task
- name: a11y gate (every story, light + dark)
run: task frontend:storybook:a11y
- name: a11y gate (every story, ${{ matrix.theme }})
run: task frontend:storybook:a11y:${{ matrix.theme }}
- name: Upload scan reports
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: a11y-scan-nightly-${{ github.run_id }}
name: a11y-scan-nightly-${{ matrix.theme }}-${{ github.run_id }}
path: frontend/.a11y-scan/
retention-days: 14
if-no-files-found: ignore
@@ -99,8 +108,13 @@ jobs:
# Builds all desktop platforms on a schedule so the Rust dependency cache is
# written on main, where PR and merge-queue tauri builds can restore it.
#
# The only job here still pinned to schedule/main: it primes a cache rather than
# testing anything, and Actions scopes a cache written on a PR branch to that PR
# alone, so a PR run costs three platform builds and produces nothing reusable.
warm-tauri-cache:
name: Warm Tauri Rust cache
if: github.event_name == 'schedule' || github.ref == 'refs/heads/main'
permissions:
contents: read
pull-requests: write
@@ -109,3 +123,72 @@ jobs:
platform: all
sign: false
secrets: inherit
# Runs the @nightly tag (conversion scenarios) plus a 10-shard concurrency run
# of every other feature.
cucumber-nightly:
name: Cucumber (nightly scenarios + full concurrency)
runs-on: ubuntu-latest
# Fork pull requests get no MAVEN_* secrets, so the image build cannot work.
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
permissions:
contents: read
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up JDK 25
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
java-version: "25"
distribution: "temurin"
- name: Install uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true
cache-dependency-glob: |
engine/pyproject.toml
engine/uv.lock
- name: Install Task
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
- name: Start the fat image with login and storage enabled
run: docker compose -f docker/embedded/compose/test_cicd.yml up -d --build
env:
MAVEN_USER: ${{ secrets.MAVEN_USER }}
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
- name: Wait for the server
# Throwaway key from test_cicd.yml; out of the header literal for gitleaks.
env:
TEST_API_KEY: "123456789"
run: |
curl --retry 90 --retry-delay 3 --retry-connrefused --retry-all-errors \
-sf -H "X-API-KEY: $TEST_API_KEY" http://localhost:8080/api/v1/info/status
# Heavy LibreOffice/Calibre/Ghostscript conversions, excluded from the PR run.
# Both tasks install the behave deps themselves, so there is no separate uv sync step.
- name: Run @nightly scenarios
run: task cucumber:nightly
# Genuinely different payloads contending on one backend.
- name: Sharded concurrency validation
run: task cucumber:parallel SHARDS=10
- name: Container logs on failure
if: failure()
run: docker compose -f docker/embedded/compose/test_cicd.yml logs --tail 400
- name: Tear down
if: always()
run: docker compose -f docker/embedded/compose/test_cicd.yml down -v
.github/workflows/package-managers.yml
+3 -2
View File
@@ -28,7 +28,7 @@ jobs:
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -73,13 +73,14 @@ jobs:
echo "jar_sha256=$JAR_SHA" >> "$GITHUB_OUTPUT"
update-homebrew-and-scoop:
environment: package-publish
needs: get-release-info
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/pr-conflict-labeler.yml
+5 -12
View File
@@ -27,29 +27,22 @@ jobs:
name: Label conflicted PRs
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: read
contents: read # actions/checkout
issues: write # get/create the repo-level conflict label
pull-requests: write # pulls.get/list plus add/remove the label on PRs
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Check out the repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up stirling-bot token
id: setup-bot
uses: ./.github/actions/setup-bot
with:
app-id: ${{ secrets.GH_APP_ID }}
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
- name: Apply conflict label
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ steps.setup-bot.outputs.token }}
github-token: ${{ github.token }}
script: |
const conflictLabel = process.env.CONFLICT_LABEL;
const owner = context.repo.owner;
.github/workflows/pre_commit.yml
+1 -1
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/push-docker-base.yml
+4 -2
View File
@@ -32,9 +32,11 @@ jobs:
- name: Set version
id: version
env:
INPUT_VERSION: ${{ github.event.inputs.version }}
run: |
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
VERSION="${{ github.event.inputs.version }}"
VERSION="${INPUT_VERSION}"
elif [ "${{ github.ref_name }}" == "accessIssueFix" ]; then
VERSION="1.0.3"
else
@@ -43,7 +45,7 @@ jobs:
echo "version=${VERSION}" >> $GITHUB_OUTPUT
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/push-docker.yml
+3 -3
View File
@@ -23,7 +23,6 @@ on:
- master
- main
- V2-master
- testMain
# cancel in-progress jobs if a new job is triggered
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
@@ -42,6 +41,7 @@ permissions:
jobs:
push:
environment: docker-publish
if: ${{ vars.CI_PROFILE != 'lite' }}
runs-on: ubuntu-24.04-8core
permissions:
@@ -53,7 +53,7 @@ jobs:
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -71,7 +71,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/rollback-latest.yml
+2 -1
View File
@@ -13,12 +13,13 @@ permissions:
jobs:
rollback:
environment: docker-publish
runs-on: ubuntu-latest
permissions:
packages: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/scorecards.yml
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -75,6 +75,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
with:
sarif_file: results.sarif
.github/workflows/stale.yml
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/swagger.yml
+2 -2
View File
@@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -45,7 +45,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
.github/workflows/sync-portal-docs.yml
+2 -1
View File
@@ -24,6 +24,7 @@ permissions:
jobs:
sync:
environment: bot-identity
name: Sync docs manifest
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -32,7 +33,7 @@ jobs:
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/sync_files_v2.yml
+2 -1
View File
@@ -33,10 +33,11 @@ permissions:
jobs:
sync-files:
environment: bot-identity
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/tauri-build.yml
+16 -10
View File
@@ -63,7 +63,7 @@ jobs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -120,7 +120,7 @@ jobs:
SIGN_BUNDLE: ${{ inputs.sign && (matrix.platform == 'macos-15' && secrets.APPLE_CERTIFICATE != '' || github.ref == 'refs/heads/main') }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -185,7 +185,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -227,20 +227,26 @@ jobs:
- name: Setup DigiCert KeyLocker Certificate
if: ${{ inputs.sign && startsWith(matrix.platform, 'windows') && env.SM_API_KEY != '' && github.ref == 'refs/heads/main' }}
shell: pwsh
env:
SM_CLIENT_CERT_FILE_B64: ${{ secrets.SM_CLIENT_CERT_FILE_B64 }}
SM_HOST: ${{ secrets.SM_HOST }}
SM_API_KEY: ${{ secrets.SM_API_KEY }}
SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
SM_KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
run: |
Write-Host "Setting up DigiCert KeyLocker environment..."
# Decode client certificate
$certBytes = [Convert]::FromBase64String("${{ secrets.SM_CLIENT_CERT_FILE_B64 }}")
$certBytes = [Convert]::FromBase64String("$env:SM_CLIENT_CERT_FILE_B64")
$certPath = "D:\Certificate_pkcs12.p12"
[IO.File]::WriteAllBytes($certPath, $certBytes)
# Set environment variables
echo "SM_CLIENT_CERT_FILE=D:\Certificate_pkcs12.p12" >> $env:GITHUB_ENV
echo "SM_HOST=${{ secrets.SM_HOST }}" >> $env:GITHUB_ENV
echo "SM_API_KEY=${{ secrets.SM_API_KEY }}" >> $env:GITHUB_ENV
echo "SM_CLIENT_CERT_PASSWORD=${{ secrets.SM_CLIENT_CERT_PASSWORD }}" >> $env:GITHUB_ENV
echo "SM_KEYPAIR_ALIAS=${{ secrets.SM_KEYPAIR_ALIAS }}" >> $env:GITHUB_ENV
echo "SM_HOST=$env:SM_HOST" >> $env:GITHUB_ENV
echo "SM_API_KEY=$env:SM_API_KEY" >> $env:GITHUB_ENV
echo "SM_CLIENT_CERT_PASSWORD=$env:SM_CLIENT_CERT_PASSWORD" >> $env:GITHUB_ENV
echo "SM_KEYPAIR_ALIAS=$env:SM_KEYPAIR_ALIAS" >> $env:GITHUB_ENV
# Get PKCS11 config path from DigiCert action
$pkcs11Config = $env:PKCS11_CONFIG
@@ -697,7 +703,7 @@ jobs:
pull-requests: write
steps:
- name: Harden the runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -789,7 +795,7 @@ jobs:
if: always()
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/test-build-docker.yml
+3 -3
View File
@@ -53,7 +53,7 @@ jobs:
cache-scope: stirling-pdf-fat
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -90,7 +90,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -191,7 +191,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
.github/workflows/testdriver.yml
+38 -16
View File
@@ -21,11 +21,15 @@ permissions:
jobs:
deploy:
environment: pr-preview
if: ${{ vars.CI_PROFILE != 'lite' }}
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -44,7 +48,7 @@ jobs:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
key: gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'gradle/libs.versions.toml', 'buildSrc/**', 'settings.gradle', 'build.gradle', 'app/**/build.gradle', 'gradle/**/*.gradle') }}
restore-keys: |
gradle-${{ runner.os }}-${{ runner.arch }}-jdk-25-
gradle-${{ runner.os }}-${{ runner.arch }}-
@@ -66,11 +70,16 @@ jobs:
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
- name: Convert repository owner to lowercase
id: repoowner
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_API }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Build and push test image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
@@ -80,16 +89,18 @@ jobs:
push: true
cache-from: type=gha,scope=stirling-pdf-latest
cache-to: type=gha,mode=max,scope=stirling-pdf-latest
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
tags: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test:test-${{ github.sha }}
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
platforms: linux/amd64
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Deploy to VPS
run: |
cat > docker-compose.yml << EOF
@@ -97,7 +108,7 @@ jobs:
services:
stirling-pdf:
container_name: stirling-pdf-test-${{ github.sha }}
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
image: ${IMAGE_BASE}:test-${{ github.sha }}
ports:
- "1337:8080"
volumes:
@@ -118,9 +129,9 @@ jobs:
restart: on-failure:5
EOF
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }}:/tmp/docker-compose.yml
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${NEW_VPS_USERNAME}@${NEW_VPS_HOST}:/tmp/docker-compose.yml
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << EOF
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << EOF
mkdir -p /stirling/test-${{ github.sha }}/{data,config,logs}
mv /tmp/docker-compose.yml /stirling/test-${{ github.sha }}/docker-compose.yml
cd /stirling/test-${{ github.sha }}
@@ -128,6 +139,10 @@ jobs:
docker-compose up -d
EOF
env:
IMAGE_BASE: ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
files-changed:
if: always()
name: detect what files changed
@@ -137,25 +152,26 @@ jobs:
frontend: ${{ steps.changes.outputs.frontend }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Check for file changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: changes
with:
filters: ".github/config/.files.yaml"
test:
environment: pr-preview
if: needs.files-changed.outputs.frontend == 'true'
needs: [deploy, files-changed]
runs-on: ubuntu-latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
@@ -185,29 +201,35 @@ jobs:
FORCE_COLOR: "3"
cleanup:
environment: pr-preview
needs: [deploy, test]
runs-on: ubuntu-latest
if: always()
steps:
- name: Harden Runner
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
with:
egress-policy: audit
- name: Set up SSH
run: |
mkdir -p ~/.ssh/
echo "${{ secrets.NEW_VPS_SSH_KEY }}" > ../private.key
echo "${NEW_VPS_SSH_KEY}" > ../private.key
sudo chmod 600 ../private.key
env:
NEW_VPS_SSH_KEY: ${{ secrets.NEW_VPS_SSH_KEY }}
- name: Cleanup deployment
if: always()
run: |
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.NEW_VPS_USERNAME }}@${{ secrets.NEW_VPS_HOST }} << EOF
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${NEW_VPS_USERNAME}@${NEW_VPS_HOST} << EOF
cd /stirling/test-${{ github.sha }}
docker-compose down
cd /stirling
rm -rf test-${{ github.sha }}
EOF
env:
NEW_VPS_USERNAME: ${{ secrets.NEW_VPS_USERNAME }}
NEW_VPS_HOST: ${{ secrets.NEW_VPS_HOST }}
continue-on-error: true # Ensure cleanup runs even if previous steps fail
.gitignore
+2
View File
@@ -26,6 +26,7 @@ watchedFolders/
# also matches this frontend source component dir; keep the source tracked.
!frontend/editor/src/proprietary/components/watchedFolders/
clientWebUI/
policy-webhook-spool/
# Scratch dir used by local fixture-regeneration runs (see
# app/proprietary/src/test/resources/db-migration-fixtures/README.md).
# Holds downloaded JARs and disposable workdirs. Never committed.
@@ -38,6 +39,7 @@ exampleYmlFiles/stirling/
/testing/file_snapshots
/testing/cucumber/junit/
/testing/cucumber/report.html
/testing/cucumber/.parallel/
/testing/.failed_tests
/.test-state/
SwaggerDoc.json
.taskfiles/cucumber.yml
+44
View File
@@ -0,0 +1,44 @@
version: '3'
tasks:
install:
desc: "Sync the Python environment with the cucumber test dependencies"
run: once
# Deliberately no sources/status fingerprint: the engine venv is shared, so it can
# already exist while synced to a different dependency group. uv no-ops when correct.
cmds:
- uv sync --project ../../engine --locked --group cucumber
run:
desc: "Run the cucumber suite against a running server (BASE_URL, default localhost:8080)"
deps: [install]
cmds:
- uv run --project ../../engine --locked --group cucumber python -m behave --no-capture -f plain {{.CLI_ARGS}}
nightly:
desc: "Run the @nightly cucumber scenarios, excluded from the default run"
summary: |
Heavy LibreOffice/Calibre/Ghostscript conversions. behave.ini excludes @nightly,
so this opts back in explicitly.
Pass extra behave flags via -- :
task cucumber:nightly -- --tags=@convert
deps: [install]
cmds:
- uv run --project ../../engine --locked --group cucumber python -m behave --tags=@nightly --no-capture -f plain {{.CLI_ARGS}}
parallel:
desc: "Run the cucumber suite as concurrent shards against one server (SHARDS, default 10)"
summary: |
Splits the feature files across SHARDS concurrent behave processes hitting a single
backend, to shake out cross-request interference. Auth-coupled features are pinned
to one shard because they change the admin password mid-scenario.
task cucumber:parallel
task cucumber:parallel SHARDS=4
BASE_URL=http://localhost:8081 task cucumber:parallel
deps: [install]
vars:
SHARDS: '{{.SHARDS | default "10"}}'
cmds:
- bash run-parallel.sh {{.SHARDS}} {{if .CLI_ARGS}}-- {{.CLI_ARGS}}{{end}}
.taskfiles/frontend.yml
+19 -9
View File
@@ -210,15 +210,26 @@ tasks:
# task frontend:storybook:test -- Button
- npx vitest run --config .storybook/vitest.config.ts {{.CLI_ARGS}}
storybook:a11y:
desc: "a11y regression gate over every story, light and dark: fail only on NEW axe violations"
storybook:a11y:light:
desc: "a11y gate over every story in light mode"
deps: [prepare, storybook:browser]
cmds:
- node .storybook/a11y-scan.mjs
- node .storybook/a11y-scan.mjs {{.CLI_ARGS}}
- node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt
- SCAN_THEME=dark node .storybook/a11y-scan.mjs
storybook:a11y:dark:
desc: "a11y gate over every story in dark mode"
deps: [prepare, storybook:browser]
cmds:
- SCAN_THEME=dark node .storybook/a11y-scan.mjs {{.CLI_ARGS}}
- node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt --baseline .storybook/a11y-baseline.dark.json
storybook:a11y:
desc: "a11y gate over every story, light and dark"
cmds:
- task: storybook:a11y:light
- task: storybook:a11y:dark
storybook:a11y:changed:
desc: "a11y gate over the stories this branch affects (default base origin/main)"
summary: |
@@ -233,7 +244,6 @@ tasks:
Pass a base ref through CLI_ARGS, e.g.
task frontend:storybook:a11y:changed -- origin/release
deps: [prepare, storybook:browser]
vars:
BASE: '{{.CLI_ARGS | default "origin/main"}}'
CHANGED:
@@ -244,10 +254,10 @@ tasks:
echo "a11y: no story files affected vs {{.BASE}} — nothing to check"
exit 0
fi
node .storybook/a11y-scan.mjs {{.CHANGED}}
node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt
SCAN_THEME=dark node .storybook/a11y-scan.mjs {{.CHANGED}}
node .storybook/a11y-check.mjs --in .a11y-scan --manifest .a11y-scan/manifest.txt --baseline .storybook/a11y-baseline.dark.json
rc=0
task frontend:storybook:a11y:light -- {{.CHANGED}} || rc=1
task frontend:storybook:a11y:dark -- {{.CHANGED}} || rc=1
exit $rc
storybook:a11y:record:
desc: "Re-record both a11y baselines (run after intentionally fixing/adding violations)"
CONTRIBUTING.md
+7 -4
View File
@@ -2,6 +2,13 @@
Thank you for your interest in contributing to Stirling-PDF! There are many ways to contribute other than writing code. For example, reporting bugs, creating suggestions, and adding or modifying translations.
## License
By contributing to this project, you agree that your contributions will be licensed under the project [license](LICENSE), which follows an open-core model.
The codebase is a mix of MIT and source-available code, so your contribution is licensed according to the directory it is committed to.
PRs are welcome in any directory by any user, just be aware of which license applies to the code you change.
## Issue Guidelines
Issues can be used to report bugs, request features, or ask questions. If you have a question, you could also ask us in our [Discord](https://discord.gg/FJUSXUSYec).
@@ -63,7 +70,3 @@ For technical guides, setup instructions, and development resources:
For configuration and usage guides, see:
- [Database Guide](DATABASE.md) - Database setup and configuration
- [OCR Guide](HowToUseOCR.md) - OCR setup and configuration
## License
By contributing to this project, you agree that your contributions will be licensed under the [MIT License](LICENSE).
Taskfile.yml
+3
View File
@@ -25,6 +25,9 @@ includes:
e2e:
taskfile: .taskfiles/e2e.yml
dir: .
cucumber:
taskfile: .taskfiles/cucumber.yml
dir: testing/cucumber
pre-commit:
taskfile: .taskfiles/pre-commit.yml
dir: .
app/common/src/main/java/stirling/software/common/aop/AutoJobAspect.java
+26
View File
@@ -2,7 +2,9 @@ package stirling.software.common.aop;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
@@ -273,6 +275,7 @@ public class AutoJobAspect {
// Store the fileId for later reference
pdfFile.setFileId(fileId);
recordPendingInputFile(fileId);
// Replace the original MultipartFile with our persistent copy
MultipartFile persistentFile = fileStorage.retrieveFile(fileId);
@@ -290,6 +293,29 @@ public class AutoJobAspect {
return originalArgs;
}
/**
* Queue an input copy for attribution to the job. The job id does not exist yet at this point,
* so {@link JobExecutorService} drains this list once it mints one.
*/
@SuppressWarnings("unchecked")
private void recordPendingInputFile(String fileId) {
try {
Object existing = request.getAttribute(JobExecutorService.PENDING_INPUT_FILE_IDS_ATTR);
List<String> ids;
if (existing instanceof List<?> list) {
ids = (List<String>) list;
} else {
ids = new ArrayList<>();
request.setAttribute(JobExecutorService.PENDING_INPUT_FILE_IDS_ATTR, ids);
}
ids.add(fileId);
} catch (RuntimeException ex) {
// Without a bound request the copy cannot be attributed; the periodic sweep is the
// only backstop, so make the miss visible rather than silently leaking the file.
log.warn("Could not record input copy {} for cleanup: {}", fileId, ex.getMessage());
}
}
private String getJobIdFromContext() {
try {
return (String) request.getAttribute("jobId");
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+4 -4
View File
@@ -1316,7 +1316,7 @@ public class ApplicationProperties {
public static class Ui {
private String appNameNavbar;
private List<String> languages;
private String logoStyle = "classic"; // Options: "classic" (default) or "modern"
private String logoStyle = "modern"; // Options: "modern" (default) or "classic"
private boolean defaultHideUnavailableTools = false;
private boolean defaultHideUnavailableConversions = false;
private HideDisabledTools hideDisabledTools = new HideDisabledTools();
@@ -1327,10 +1327,10 @@ public class ApplicationProperties {
public String getLogoStyle() {
// Validate and return either "modern" or "classic"
if ("modern".equalsIgnoreCase(logoStyle)) {
return "modern";
if ("classic".equalsIgnoreCase(logoStyle)) {
return "classic";
}
return "classic"; // default
return "modern"; // default
}
@Data
app/common/src/main/java/stirling/software/common/model/job/JobResult.java
+23
View File
@@ -52,6 +52,13 @@ public class JobResult {
/** Key/value metadata that survives the write-through into the shared job store. */
private final Map<String, String> metadata = new ConcurrentHashMap<>();
/**
* File ids of the persistent input copies made for this job. An async submit copies the upload
* into FileStorage so the job can still read it after the request returns; without tracking
* them here nothing would ever delete those copies.
*/
@JsonIgnore private final List<String> inputFileIds = new CopyOnWriteArrayList<>();
/**
* Create a new JobResult with the given job ID
*
@@ -167,6 +174,22 @@ public class JobResult {
return Collections.unmodifiableList(notes);
}
/** Record a persistent input copy so job cleanup deletes it alongside the results. */
public void addInputFileId(String fileId) {
if (fileId != null && !fileId.isBlank() && !inputFileIds.contains(fileId)) {
this.inputFileIds.add(fileId);
}
}
/**
* File ids of this job's persistent input copies.
*
* @return An unmodifiable view of the input file ids
*/
public List<String> getInputFileIds() {
return Collections.unmodifiableList(inputFileIds);
}
/** Attach a metadata value, e.g. a policy id so cluster peers can identify a policy run. */
public void putMetadata(String key, String value) {
if (key != null && value != null) {
app/common/src/main/java/stirling/software/common/service/CustomPDFDocumentFactory.java
+28
View File
@@ -729,4 +729,32 @@ public class CustomPDFDocumentFactory {
p.toFile().deleteOnExit();
return p;
}
/** A custom RandomAccessRead implementation that deletes the file when closed */
private static class DeletingRandomAccessFile extends RandomAccessReadBufferedFile {
private final Path tempFilePath;
public DeletingRandomAccessFile(File file) throws IOException {
super(file);
this.tempFilePath = file.toPath();
}
@Override
public void close() throws IOException {
try {
super.close();
} finally {
try {
boolean deleted = Files.deleteIfExists(tempFilePath);
if (deleted) {
log.info("Successfully deleted temp file: {}", tempFilePath);
} else {
log.warn("Failed to delete temp file (may not exist): {}", tempFilePath);
}
} catch (IOException e) {
log.error("Error deleting temp file: {}", tempFilePath, e);
}
}
}
}
}
app/common/src/main/java/stirling/software/common/service/FileStorage.java
+15
View File
@@ -179,6 +179,21 @@ public class FileStorage {
return fileStore.delete(fileId);
}
/**
* Delete a stored file without the per-file ownership check.
*
* <p>Job cleanup authorises at the job level and then deletes that job's own files, so the
* deleter is legitimately not their owner - an admin sweeping every user's jobs, or the
* unauthenticated scheduled task. Routing those through {@link #deleteFile(String)} makes the
* ownership check throw and silently orphans the files on disk.
*
* <p>Only ever pass file ids read back off a job that the caller has already been authorised
* for; never a caller-supplied id.
*/
public boolean deleteFileAsSystem(String fileId) {
return fileStore.delete(fileId);
}
public boolean fileExists(String fileId) {
enforceOwnership(fileId);
return fileStore.exists(fileId);
app/common/src/main/java/stirling/software/common/service/JobExecutorService.java
+37
View File
@@ -1,6 +1,7 @@
package stirling.software.common.service;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -33,6 +34,14 @@ import stirling.software.common.util.RegexPatternUtils;
@Slf4j
public class JobExecutorService {
/**
* Request attribute holding the FileStorage ids of persistent input copies made for the job
* about to be created. Populated before the job id exists (the aspect copies the upload while
* processing arguments), drained onto the JobResult as soon as the task is created so cleanup
* can delete them.
*/
public static final String PENDING_INPUT_FILE_IDS_ATTR = "autoJobPendingInputFileIds";
private final TaskManager taskManager;
private final FileStorage fileStorage;
private final HttpServletRequest request;
@@ -133,6 +142,7 @@ public class JobExecutorService {
resourceWeight);
taskManager.createTask(jobId);
registerPendingInputFiles(jobId);
final String capturedJobIdForQueue = jobId;
Supplier<Object> wrappedWork =
@@ -163,6 +173,7 @@ public class JobExecutorService {
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
} else if (async) {
taskManager.createTask(jobId);
registerPendingInputFiles(jobId);
final String capturedJobId = jobId;
@@ -484,4 +495,30 @@ public class JobExecutorService {
}
return baseJobId;
}
/**
* Hand the input copies made while processing arguments to the freshly created job, so job
* cleanup deletes them. Drains the attribute so a retry cannot attribute the same ids twice.
*/
@SuppressWarnings("unchecked")
private void registerPendingInputFiles(String jobId) {
if (request == null) {
return;
}
Object pending;
try {
pending = request.getAttribute(PENDING_INPUT_FILE_IDS_ATTR);
request.removeAttribute(PENDING_INPUT_FILE_IDS_ATTR);
} catch (RuntimeException ex) {
// No request bound to this thread (e.g. an internally dispatched job).
log.debug("Could not read pending input file ids: {}", ex.getMessage());
return;
}
if (!(pending instanceof List<?> ids)) {
return;
}
for (String fileId : (List<String>) ids) {
taskManager.registerInputFile(jobId, fileId);
}
}
}
app/common/src/main/java/stirling/software/common/service/TaskManager.java
+97 -16
View File
@@ -17,6 +17,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -234,6 +235,24 @@ public class TaskManager {
return false;
}
/**
* Record a persistent input copy against a job so cleanup deletes it with the results.
*
* @param jobId The job ID
* @param fileId The FileStorage id of the input copy
* @return true if the job exists and the id was recorded
*/
public boolean registerInputFile(String jobId, String fileId) {
JobResult jobResult = jobResults.get(jobId);
if (jobResult == null) {
log.warn("Attempted to register an input file against non-existent job ID: {}", jobId);
return false;
}
jobResult.addInputFileId(fileId);
log.debug("Registered input file {} for job {}", fileId, jobId);
return true;
}
/** Attach metadata to a job and write it through to the shared store for cluster peers. */
public boolean putMetadata(String jobId, String key, String value) {
JobResult jobResult = jobResults.get(jobId);
@@ -329,25 +348,59 @@ public class TaskManager {
return jobResults.computeIfAbsent(jobId, JobResult::createNew);
}
/**
* What a cleanup pass removed. Returned by the on-demand cleanup so callers can assert on it.
*/
public record CleanupSummary(int jobsRemoved, int filesDeleted, int jobsRetained) {}
/** Clean up old completed job results. No-op in cluster mode; the backplane TTL owns expiry. */
public void cleanupOldJobs() {
public CleanupSummary cleanupOldJobs() {
if (clusterBackplane != null && !clusterBackplane.shouldRunLocalCleanup()) {
return;
return new CleanupSummary(0, 0, jobResults.size());
}
return cleanupJobs(false, jobId -> true);
}
/**
* Force-expire this node's finished jobs now, ignoring the age threshold. Jobs still running
* are left alone - deleting their files mid-flight would break them - and are reported as
* retained.
*
* <p>Unlike {@link #cleanupOldJobs()} this always runs locally: it is an explicit request to
* release this node's storage, not the scheduled sweep the backplane TTL owns.
*
* @param jobIdFilter Only jobs whose id passes this predicate are considered, so a caller can
* restrict the sweep to jobs the requester is allowed to touch
* @return What was removed
*/
public CleanupSummary cleanupFinishedJobsNow(Predicate<String> jobIdFilter) {
return cleanupJobs(true, jobIdFilter);
}
private CleanupSummary cleanupJobs(boolean force, Predicate<String> filter) {
LocalDateTime expiryThreshold =
LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
LocalDateTime pendingExpiryThreshold =
LocalDateTime.now().minus(pendingJobExpiryMinutes, ChronoUnit.MINUTES);
int removedCount = 0;
int filesDeleted = 0;
int retainedCount = 0;
try {
for (Map.Entry<String, JobResult> entry : jobResults.entrySet()) {
JobResult result = entry.getValue();
if (!filter.test(entry.getKey())) {
retainedCount++;
continue;
}
boolean expiredCompletedJob =
result.isComplete()
&& result.getCompletedAt() != null
&& result.getCompletedAt().isBefore(expiryThreshold);
&& (force
|| (result.getCompletedAt() != null
&& result.getCompletedAt()
.isBefore(expiryThreshold)));
boolean abandonedPendingJob =
!result.isComplete()
&& result.getCreatedAt() != null
@@ -360,7 +413,7 @@ public class TaskManager {
// Clean up file results
if (expiredCompletedJob) {
cleanupJobFiles(result, entry.getKey());
filesDeleted += cleanupJobFiles(result, entry.getKey());
}
// Remove the job result
@@ -369,15 +422,22 @@ public class TaskManager {
jobStore.delete(entry.getKey());
}
removedCount++;
} else {
retainedCount++;
}
}
if (removedCount > 0) {
log.info("Cleaned up {} expired job results", removedCount);
log.info(
"Cleaned up {} {} job results ({} files deleted)",
removedCount,
force ? "finished" : "expired",
filesDeleted);
}
} catch (Exception e) {
log.error("Error during job cleanup: {}", e.getMessage(), e);
}
return new CleanupSummary(removedCount, filesDeleted, retainedCount);
}
/** Mirror the in-memory {@code JobResult} into the cluster-visible {@link JobStore}. */
@@ -525,22 +585,43 @@ public class TaskManager {
}
}
/** Clean up files associated with a job result */
private void cleanupJobFiles(JobResult result, String jobId) {
/**
* Clean up files associated with a job result: both the results and the persistent input copy
* an async submit made of the upload.
*
* @return The number of files actually deleted
*/
private int cleanupJobFiles(JobResult result, String jobId) {
int deleted = 0;
// Clean up all result files
if (result.hasFiles()) {
for (ResultFile resultFile : result.getAllResultFiles()) {
try {
fileStorage.deleteFile(resultFile.getFileId());
} catch (Exception e) {
log.warn(
"Failed to delete file {} for job {}: {}",
resultFile.getFileId(),
jobId,
e.getMessage());
if (deleteJobFile(resultFile.getFileId(), jobId)) {
deleted++;
}
}
}
for (String inputFileId : result.getInputFileIds()) {
if (deleteJobFile(inputFileId, jobId)) {
deleted++;
}
}
return deleted;
}
/**
* Deletes as the system, not as the caller: an admin sweeping another user's jobs, or the
* scheduled task running with no security context, is not the file's owner, and the
* ownership-checked delete would throw and leave the file orphaned on disk. The job itself is
* already authorised by the time we get here, and these ids come off that job, not the request.
*/
private boolean deleteJobFile(String fileId, String jobId) {
try {
return fileStorage.deleteFileAsSystem(fileId);
} catch (Exception e) {
log.warn("Failed to delete file {} for job {}: {}", fileId, jobId, e.getMessage());
return false;
}
}
/** Find the ResultFile metadata for a given file ID by searching through all job results */
app/common/src/main/java/stirling/software/common/util/GeneralUtils.java
+27 -9
View File
@@ -12,6 +12,7 @@ import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -54,6 +55,10 @@ public class GeneralUtils {
private final String DEFAULT_WEBUI_CONFIGS_DIR = "defaultWebUIConfigs";
private final String PYTHON_SCRIPTS_DIR = "python";
// Extracted once per run. Rewriting a script while another request is exec-ing it
// races wherever rename is not atomic, such as 9p or NFS bind mounts.
private final Map<String, Path> EXTRACTED_SCRIPTS = new ConcurrentHashMap<>();
private final RegexPatternUtils patternCache = RegexPatternUtils.getInstance();
// Valid size units used for convertSizeToBytes validation and parsing
private final Set<String> VALID_SIZE_UNITS = Set.of("B", "KB", "MB", "GB", "TB");
@@ -1025,17 +1030,30 @@ public class GeneralUtils {
}
Path scriptsDir = Path.of(InstallationPathConfig.getScriptsPath(), PYTHON_SCRIPTS_DIR);
Files.createDirectories(scriptsDir);
Path target = scriptsDir.resolve(scriptName);
ClassPathResource res =
new ClassPathResource("static/" + PYTHON_SCRIPTS_DIR + "/" + scriptName);
if (!res.exists()) {
log.error("Resource not found: {}", res.getPath());
throw new IOException("Resource not found: " + res.getPath());
Path cached = EXTRACTED_SCRIPTS.get(scriptName);
if (cached != null && Files.isRegularFile(cached)) {
return cached;
}
synchronized (EXTRACTED_SCRIPTS) {
cached = EXTRACTED_SCRIPTS.get(scriptName);
if (cached != null && Files.isRegularFile(cached)) {
return cached;
}
Files.createDirectories(scriptsDir);
ClassPathResource res =
new ClassPathResource("static/" + PYTHON_SCRIPTS_DIR + "/" + scriptName);
if (!res.exists()) {
log.error("Resource not found: {}", res.getPath());
throw new IOException("Resource not found: " + res.getPath());
}
copyResourceToFile(res, target);
EXTRACTED_SCRIPTS.put(scriptName, target);
return target;
}
copyResourceToFile(res, target);
return target;
}
/*
app/common/src/test/java/stirling/software/common/service/FileStorageOwnershipTest.java
+19
View File
@@ -2,6 +2,7 @@ package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -65,6 +66,24 @@ class FileStorageOwnershipTest {
assertThrows(SecurityException.class, () -> fs.deleteFile(id));
}
@Test
void systemDeleteOfAnotherUsersFile_allowed_soJobCleanupDoesNotOrphanIt(@TempDir Path tempDir)
throws IOException {
// An admin sweeping every user's finished jobs is not the owner of their files. The
// ownership-checked delete throws there, which used to drop the job record and leave the
// files stranded on disk with nothing left able to reference them.
AtomicReference<String> user = new AtomicReference<>("alice");
FileStorage fs = newStorageWithCurrentUser(tempDir, user);
String id = fs.storeBytes("alice's file".getBytes(), "x.bin");
user.set("admin");
assertThrows(SecurityException.class, () -> fs.deleteFile(id));
assertTrue(fs.deleteFileAsSystem(id), "System delete must not be blocked by ownership");
user.set("alice");
assertThrows(IOException.class, () -> fs.retrieveBytes(id), "File should really be gone");
}
@Test
void anonymousRetrieveOfOwnedFile_allowed_noCurrentUserMeansNoCompare(@TempDir Path tempDir)
throws IOException {
app/common/src/test/java/stirling/software/common/service/TaskManagerCleanupTest.java
+263
View File
@@ -0,0 +1,263 @@
package stirling.software.common.service;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.http.MediaType;
import org.springframework.test.util.ReflectionTestUtils;
import stirling.software.common.cluster.ClusterBackplane;
import stirling.software.common.cluster.JobStore;
import stirling.software.common.model.job.JobResult;
import stirling.software.common.model.job.ResultFile;
/**
* Covers the on-demand cleanup path and the input-copy tracking that makes it complete. An async
* submit persists a copy of the upload as well as its results; before both were tracked, only the
* results were ever deleted and the input copy stayed on disk indefinitely.
*/
class TaskManagerCleanupTest {
@Mock private FileStorage fileStorage;
@Mock private JobStore jobStore;
@Mock private ClusterBackplane clusterBackplane;
@InjectMocks private TaskManager taskManager;
private AutoCloseable closeable;
@BeforeEach
void setUp() {
closeable = MockitoAnnotations.openMocks(this);
lenient().when(clusterBackplane.localNodeId()).thenReturn("test-node");
lenient().when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(true);
lenient().when(fileStorage.deleteFileAsSystem(anyString())).thenReturn(true);
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
ReflectionTestUtils.setField(taskManager, "pendingJobExpiryMinutes", 1440);
}
@AfterEach
void tearDown() throws Exception {
closeable.close();
}
@SuppressWarnings("unchecked")
private Map<String, JobResult> jobResults() {
return (Map<String, JobResult>) ReflectionTestUtils.getField(taskManager, "jobResults");
}
/** Complete a job with a single result file, as an async file-producing job would. */
private void completeWithFile(String jobId, String fileId) {
taskManager.setFileResult(jobId, fileId, "out.pdf", MediaType.APPLICATION_PDF_VALUE);
taskManager.setComplete(jobId);
}
@Test
void forcedCleanupRemovesFinishedJobsRegardlessOfAge() {
String jobId = "fresh-job";
taskManager.createTask(jobId);
completeWithFile(jobId, "result-file");
// The scheduled sweep leaves it alone: it completed well inside the retention window.
taskManager.cleanupOldJobs();
assertTrue(jobResults().containsKey(jobId), "Scheduled cleanup should respect the expiry");
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(1, summary.jobsRemoved());
assertEquals(1, summary.filesDeleted());
assertEquals(0, summary.jobsRetained());
assertFalse(jobResults().containsKey(jobId));
verify(fileStorage).deleteFileAsSystem("result-file");
verify(jobStore).delete(jobId);
}
@Test
void forcedCleanupDeletesThePersistedInputCopy() {
String jobId = "job-with-input";
taskManager.createTask(jobId);
assertTrue(taskManager.registerInputFile(jobId, "input-file"));
completeWithFile(jobId, "result-file");
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(1, summary.jobsRemoved());
assertEquals(2, summary.filesDeleted(), "Both the result and the input copy must go");
verify(fileStorage).deleteFileAsSystem("result-file");
verify(fileStorage).deleteFileAsSystem("input-file");
}
@Test
void scheduledCleanupAlsoDeletesThePersistedInputCopy() {
String jobId = "expired-job";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "input-file");
completeWithFile(jobId, "result-file");
JobResult result = taskManager.getJobResult(jobId);
ReflectionTestUtils.setField(result, "completedAt", LocalDateTime.now().minusHours(1));
taskManager.cleanupOldJobs();
assertFalse(jobResults().containsKey(jobId));
verify(fileStorage).deleteFileAsSystem("result-file");
verify(fileStorage).deleteFileAsSystem("input-file");
}
@Test
void forcedCleanupLeavesRunningJobsAlone() {
String running = "running-job";
taskManager.createTask(running);
taskManager.registerInputFile(running, "in-flight-input");
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(0, summary.jobsRemoved());
assertEquals(0, summary.filesDeleted());
assertEquals(1, summary.jobsRetained());
assertTrue(jobResults().containsKey(running));
// Deleting a running job's input mid-flight would break it.
verify(fileStorage, never()).deleteFileAsSystem(anyString());
}
@Test
void forcedCleanupSkipsJobsTheFilterRejects() {
taskManager.createTask("alice:job");
completeWithFile("alice:job", "alice-file");
taskManager.createTask("bob:job");
completeWithFile("bob:job", "bob-file");
TaskManager.CleanupSummary summary =
taskManager.cleanupFinishedJobsNow(id -> id.startsWith("alice:"));
assertEquals(1, summary.jobsRemoved());
assertEquals(1, summary.jobsRetained());
assertFalse(jobResults().containsKey("alice:job"));
assertTrue(jobResults().containsKey("bob:job"), "Another user's job must survive");
verify(fileStorage).deleteFileAsSystem("alice-file");
verify(fileStorage, never()).deleteFileAsSystem("bob-file");
}
@Test
void forcedCleanupIsIdempotent() {
String jobId = "job-to-clean";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "input-file");
completeWithFile(jobId, "result-file");
taskManager.cleanupFinishedJobsNow(id -> true);
TaskManager.CleanupSummary second = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(0, second.jobsRemoved());
assertEquals(0, second.filesDeleted());
}
@Test
void forcedCleanupRunsEvenWhenTheBackplaneOwnsScheduledExpiry() {
// The scheduled sweep defers to the backplane TTL in cluster mode, but an explicit
// request to release this node's storage still has to do something.
when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(false);
String jobId = "clustered-job";
taskManager.createTask(jobId);
completeWithFile(jobId, "result-file");
taskManager.cleanupOldJobs();
assertTrue(jobResults().containsKey(jobId));
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(1, summary.jobsRemoved());
assertFalse(jobResults().containsKey(jobId));
}
@Test
void cleanupCountsOnlyFilesThatWereActuallyDeleted() {
// A file already gone (a retry deleted it, say) must not be counted as freed.
String jobId = "partially-cleaned";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "already-gone");
completeWithFile(jobId, "result-file");
when(fileStorage.deleteFileAsSystem("already-gone")).thenReturn(false);
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(1, summary.filesDeleted());
}
@Test
void cleanupSurvivesAFileStorageFailure() {
String jobId = "job-with-unhappy-storage";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "input-file");
completeWithFile(jobId, "result-file");
when(fileStorage.deleteFileAsSystem("result-file"))
.thenThrow(new RuntimeException("disk on fire"));
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
// The job is still released and the remaining file still deleted.
assertEquals(1, summary.jobsRemoved());
assertEquals(1, summary.filesDeleted());
assertFalse(jobResults().containsKey(jobId));
verify(fileStorage).deleteFileAsSystem("input-file");
}
@Test
void registerInputFileRejectsAnUnknownJob() {
assertFalse(taskManager.registerInputFile("no-such-job", "input-file"));
}
@Test
void registerInputFileIgnoresDuplicatesAndBlanks() {
String jobId = "dedupe-job";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "input-file");
taskManager.registerInputFile(jobId, "input-file");
taskManager.registerInputFile(jobId, " ");
taskManager.registerInputFile(jobId, null);
List<String> inputFileIds = taskManager.getJobResult(jobId).getInputFileIds();
assertEquals(List.of("input-file"), inputFileIds);
}
@Test
void multiFileResultsAndTheInputCopyAreAllDeleted() {
String jobId = "split-job";
taskManager.createTask(jobId);
taskManager.registerInputFile(jobId, "input-file");
JobResult result = taskManager.getJobResult(jobId);
result.completeWithFiles(
List.of(
ResultFile.builder()
.fileId("page-1")
.fileName("1.pdf")
.contentType(MediaType.APPLICATION_PDF_VALUE)
.fileSize(10L)
.build(),
ResultFile.builder()
.fileId("page-2")
.fileName("2.pdf")
.contentType(MediaType.APPLICATION_PDF_VALUE)
.fileSize(10L)
.build()));
TaskManager.CleanupSummary summary = taskManager.cleanupFinishedJobsNow(id -> true);
assertEquals(3, summary.filesDeleted());
verify(fileStorage).deleteFileAsSystem("page-1");
verify(fileStorage).deleteFileAsSystem("page-2");
verify(fileStorage).deleteFileAsSystem("input-file");
}
}
app/common/src/test/java/stirling/software/common/service/TaskManagerMoreTest.java
+2 -1
View File
@@ -290,7 +290,8 @@ class TaskManagerMoreTest {
ReflectionTestUtils.setField(job, "complete", true);
ReflectionTestUtils.setField(job, "completedAt", LocalDateTime.now().minusHours(2));
when(fileStorage.deleteFile("doomed")).thenThrow(new RuntimeException("locked"));
when(fileStorage.deleteFileAsSystem("doomed"))
.thenThrow(new RuntimeException("locked"));
// Must not propagate; the job is still removed afterwards.
taskManager.cleanupOldJobs();
app/common/src/test/java/stirling/software/common/service/TaskManagerTest.java
+3 -3
View File
@@ -258,7 +258,7 @@ class TaskManagerTest {
.build();
ReflectionTestUtils.setField(oldJob, "resultFiles", java.util.List.of(resultFile));
when(fileStorage.deleteFile("file-id")).thenReturn(true);
when(fileStorage.deleteFileAsSystem("file-id")).thenReturn(true);
// Obtain access to the private jobResults map
Map<String, JobResult> jobResultsMap =
@@ -281,7 +281,7 @@ class TaskManagerTest {
assertFalse(jobResultsMap.containsKey(oldJobId));
assertTrue(jobResultsMap.containsKey(recentJobId));
assertTrue(jobResultsMap.containsKey(activeJobId));
verify(fileStorage).deleteFile("file-id");
verify(fileStorage).deleteFileAsSystem("file-id");
}
@Test
@@ -308,7 +308,7 @@ class TaskManagerTest {
// Assert: nothing was removed locally, and no jobStore.delete was issued.
assertTrue(jobResultsMap.containsKey(oldJobId));
verify(jobStore, never()).delete(anyString());
verify(fileStorage, never()).deleteFile(anyString());
verify(fileStorage, never()).deleteFileAsSystem(anyString());
}
@Test
app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java
+114 -101
View File
@@ -1,9 +1,14 @@
package stirling.software.SPDF.controller.api.security;
import java.awt.Color;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPageTree;
@@ -43,6 +48,10 @@ import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.common.util.propertyeditor.JsonListPropertyEditor;
import stirling.software.common.util.propertyeditor.JsonObjectPropertyEditor;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.redact.PdfRedactor;
import stirling.software.jpdfium.redact.RedactOptions;
import stirling.software.jpdfium.redact.RedactResult;
import tools.jackson.core.type.TypeReference;
@@ -140,134 +149,138 @@ public class RedactController {
+ " patterns. Users can provide text patterns to redact, with options for regex"
+ " and whole word matching.")
public ResponseEntity<Resource> redactPdf(@ModelAttribute RedactPdfRequest request) {
String rawListOfText = request.getListOfText();
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
if (request.getFileInput() == null || request.getFileInput().isEmpty()) {
log.error("File input is null or empty");
throw ExceptionUtils.createFileNullOrEmptyException();
}
String rawListOfText = request.getListOfText();
if (rawListOfText == null || rawListOfText.trim().isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "No text patterns provided for redaction");
}
String[] listOfText = rawListOfText.split("\n");
if (listOfText.length == 1 && listOfText[0].trim().isEmpty()) {
List<String> terms =
Arrays.stream(rawListOfText.split("\n"))
.map(String::trim)
.filter(s -> !s.isEmpty() && s.length() <= 4096)
.collect(Collectors.toList());
if (terms.isEmpty()) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "No text patterns provided for redaction");
}
PDDocument document = null;
PDDocument fallbackDocument = null;
boolean useRegex = Boolean.TRUE.equals(request.getUseRegex());
boolean wholeWordSearchBool = Boolean.TRUE.equals(request.getWholeWordSearch());
try {
if (request.getFileInput() == null) {
log.error("File input is null");
throw ExceptionUtils.createFileNullOrEmptyException();
if (useRegex) {
for (String term : terms) {
try {
Pattern.compile(term);
} catch (PatternSyntaxException e) {
throw ExceptionUtils.createIllegalArgumentException(
"error.redaction.no.patterns", "Invalid regex pattern: " + term);
}
}
}
document = pdfDocumentFactory.load(request.getFileInput());
String filename =
removeFileExtension(
Objects.requireNonNull(
Filenames.toSimpleFileName(
request.getFileInput().getOriginalFilename())))
+ "_redacted.pdf";
Color redactColor = ManualRedactionService.decodeOrDefault(request.getRedactColor());
int boxColorInt = redactColor.getRGB();
try (PDDocument document = pdfDocumentFactory.load(request.getFileInput())) {
if (document == null) {
log.error("Failed to load PDF document");
throw ExceptionUtils.createPdfCorruptedException(
"during redaction", new IOException("Failed to load PDF document"));
}
Map<Integer, List<PDFText>> allFoundTextsByPage =
textRedactionService.findTextToRedact(
document, listOfText, useRegex, wholeWordSearchBool);
try (TempFile tempInput = tempFileManager.createManagedTempFile(".pdf")) {
try {
request.getFileInput().transferTo(tempInput.getFile());
} catch (Exception e) {
document.save(tempInput.getFile());
}
int totalMatches = allFoundTextsByPage.values().stream().mapToInt(List::size).sum();
log.info(
"Redaction scan: {} occurrences across {} pages (patterns={}, regex={}, wholeWord={})",
totalMatches,
allFoundTextsByPage.size(),
listOfText.length,
useRegex,
wholeWordSearchBool);
RedactOptions options =
RedactOptions.builder()
.addWords(terms)
.useRegex(useRegex)
.wholeWord(wholeWordSearchBool)
.boxColor(boxColorInt)
.padding(request.getCustomPadding())
.removeContent(true)
.convertToImage(Boolean.TRUE.equals(request.getConvertPDFToImage()))
.normalizeFonts(false)
.fixToUnicode(false)
.glyphAware(true)
.redactMetadata(true)
.build();
String filename =
removeFileExtension(
Objects.requireNonNull(
Filenames.toSimpleFileName(
request.getFileInput().getOriginalFilename())))
+ "_redacted.pdf";
TempFile tempOutput = tempFileManager.createManagedTempFile(".pdf");
try {
try (PdfDocument checkDoc = PdfDocument.open(tempInput.getFile().toPath())) {
if (checkDoc.pageCount() <= 0) {
throw new IOException("Invalid or empty PDF document");
}
}
if (allFoundTextsByPage.isEmpty()) {
log.info("No text found matching redaction patterns");
return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager);
log.debug(
"Calling JPDFium PdfRedactor.redact in RedactController (terms={})",
terms);
RedactResult result = PdfRedactor.redact(tempInput.getFile().toPath(), options);
log.debug(
"JPDFium auto-redact complete (matches={})",
result != null ? result.totalMatches() : -1);
if (result == null) {
throw new IOException("JPDFium auto-redact returned null result");
}
try {
result.save(tempOutput.getFile().toPath());
log.info(
"JPDFium auto-redact: {} matches processed into {}",
result.totalMatches(),
filename);
return WebResponseUtils.pdfFileToWebResponse(tempOutput, filename);
} finally {
if (result.document() != null) {
result.document().close();
}
}
} catch (Exception e) {
tempOutput.close();
log.warn(
"JPDFium native redaction fell back to manual redaction service: {}",
e.getMessage());
Map<Integer, List<PDFText>> foundTexts =
textRedactionService.findTextToRedact(
document,
terms.toArray(new String[0]),
useRegex,
wholeWordSearchBool);
TempFile finalized =
manualRedactionService.finalizeRedaction(
document,
foundTexts,
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
false);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
}
boolean fallbackToBoxOnlyMode;
try {
fallbackToBoxOnlyMode =
textRedactionService.performTextReplacement(
document,
allFoundTextsByPage,
listOfText,
useRegex,
wholeWordSearchBool);
} catch (Exception e) {
log.warn(
"Text replacement redaction failed, falling back to box-only mode: {}",
e.getMessage());
fallbackToBoxOnlyMode = true;
}
if (fallbackToBoxOnlyMode) {
log.warn(
"Font compatibility issues detected. Using box-only redaction mode for better reliability.");
fallbackDocument = pdfDocumentFactory.load(request.getFileInput());
allFoundTextsByPage =
textRedactionService.findTextToRedact(
fallbackDocument, listOfText, useRegex, wholeWordSearchBool);
TempFile finalized =
manualRedactionService.finalizeRedaction(
fallbackDocument,
allFoundTextsByPage,
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
false);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
}
TempFile finalized =
manualRedactionService.finalizeRedaction(
document,
allFoundTextsByPage,
request.getRedactColor(),
request.getCustomPadding(),
request.getConvertPDFToImage(),
true);
return WebResponseUtils.pdfFileToWebResponse(finalized, filename);
} catch (IllegalArgumentException e) {
throw e;
} catch (Exception e) {
log.error("Redaction operation failed: {}", e.getMessage(), e);
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
} finally {
if (document != null) {
try {
if (fallbackDocument == null) {
document.close();
}
} catch (IOException e) {
log.warn("Failed to close main document: {}", e.getMessage());
}
}
if (fallbackDocument != null) {
try {
fallbackDocument.close();
} catch (IOException e) {
log.warn("Failed to close fallback document: {}", e.getMessage());
}
}
}
}
app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java
+88 -1104
View File
File diff suppressed because it is too large Load Diff
app/core/src/main/java/stirling/software/SPDF/model/api/security/RedactPdfRequest.java
+1 -1
View File
@@ -45,5 +45,5 @@ public class RedactPdfRequest extends PDFFile {
description = "Convert the redacted PDF to an image",
defaultValue = "false",
requiredMode = Schema.RequiredMode.REQUIRED)
private Boolean convertPDFToImage;
private Boolean convertPDFToImage = Boolean.FALSE;
}
app/core/src/main/java/stirling/software/common/controller/JobController.java
+30
View File
@@ -12,6 +12,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@@ -213,6 +214,35 @@ public class JobController {
}
}
/**
* Self-service counterpart to the admin-only {@code POST /api/v1/admin/job/cleanup}: that one
* sweeps every user's jobs and needs ROLE_ADMIN, this one releases only the caller's own and so
* is safe for any authenticated user. Both run the same sweep inside {@link TaskManager}.
*/
@PostMapping("/jobs/cleanup")
@Operation(
summary = "Release finished jobs and their stored files now",
description =
"Force-expires this node's finished jobs instead of waiting out the retention"
+ " window, deleting their result files and the persistent copies made of"
+ " their inputs. Only jobs the caller may access are touched, and jobs"
+ " still running are left alone. Admins can sweep every user's jobs"
+ " with POST /api/v1/admin/job/cleanup?force=true.")
public ResponseEntity<?> cleanupFinishedJobs() {
TaskManager.CleanupSummary summary =
taskManager.cleanupFinishedJobsNow(this::validateJobAccess);
log.info(
"On-demand job cleanup removed {} job(s) and {} file(s), retained {} job(s)",
summary.jobsRemoved(),
summary.filesDeleted(),
summary.jobsRetained());
return ResponseEntity.ok(
Map.of(
"jobsRemoved", summary.jobsRemoved(),
"filesDeleted", summary.filesDeleted(),
"jobsRetained", summary.jobsRetained()));
}
@GetMapping("/job/{jobId}/result/files")
@Operation(summary = "Get job result files")
public ResponseEntity<?> getJobFiles(@PathVariable("jobId") String jobId) {
app/core/src/main/resources/settings.yml.template
+1 -1
View File
@@ -313,7 +313,7 @@ autoPipeline:
ui:
appNameNavbar: "" # custom app/brand name. NOTE: no longer shown in the navbar (the navbar renders the logo). It IS used as the browser tab title and as the TOTP/2FA issuer label in authenticator apps. Empty falls back to "Stirling PDF"
logoStyle: classic # Options: 'classic' (default - classic S icon) or 'modern' (minimalist logo)
logoStyle: modern # Options: 'modern' (default - minimalist logo) or 'classic' (legacy S icon)
languages: [] # If empty, all languages are enabled. To restrict to specific languages, use a whitelist like ["de_DE", "pl_PL", "sv_SE"]. Empty list or not restricting any languages will enable all available languages.
defaultHideUnavailableTools: false # Default user preference: hide disabled tools instead of greying them out
defaultHideUnavailableConversions: false # Default user preference: hide disabled conversion options instead of greying them out
app/core/src/main/resources/static/apple-touch-icon.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

app/core/src/main/resources/static/favicon-16x16.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 829 B

After

Width:  |  Height:  |  Size: 681 B

app/core/src/main/resources/static/favicon-32x32.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

app/core/src/main/resources/static/favicon.icns
BIN
View File
Binary file not shown.
app/core/src/main/resources/static/favicon.ico
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 19 KiB

app/core/src/main/resources/static/favicon.svg
+5 -1
View File
@@ -1 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg" id="Layer_1" x="0" y="0" version="1.1" viewBox="0 0 512 512" style="enable-background:new 0 0 512 512" xml:space="preserve"><defs id="defs173"><linearGradient id="XMLID_5_" x1="304.496" x2="316.036" y1="422.91" y2="326.263" gradientUnits="userSpaceOnUse"><stop offset="0" style="stop-color:#dcf1f3" id="stop156"/><stop offset="1" style="stop-color:#c2c2c9" id="stop158"/></linearGradient></defs><style id="style150" type="text/css">.st1{fill:#c02223}.st2{fill:#882425}.st3{fill:url(#XMLID_5_)}.st4{fill:url(#XMLID_7_)}</style><g id="XMLID_4_"><path id="XMLID_131_" d="M 347.01402,14.355825 98.978019,69.02261 C 73.825483,74.547445 55.942464,96.792175 55.942464,122.52628 v 315.06096 c 0,22.39012 16.719895,41.14548 38.819234,43.76251 L 224.8861,498.36042 339.48636,384.26465 455.76603,265.15425 453.73057,84.870162 C 453.43979,62.916214 433.08513,46.632491 411.71274,51.284984 l -28.78729,6.251786 0.14539,-13.666697 C 383.36162,24.678542 365.62399,10.284894 347.01402,14.355825 Z" class="st1" style="stroke-width:1.45391"/><path id="XMLID_117_" d="m 383.21622,57.53677 v 285.8375 L 456.05681,265.00885 454.02135,78.763767 C 453.87595,59.863016 436.28372,45.905539 417.81914,49.97647 Z" class="st2" style="stroke-width:1.45391"/><polygon id="XMLID_18_" points="234.7 422.6 368.5 387.7 393.5 262.2" class="st3" style="fill:url(#XMLID_5_)" transform="matrix(1.4556308,0,0,1.4548265,-116.73161,-116.45231)"/><linearGradient id="XMLID_7_" x1="223.084" x2="241.417" y1="372.756" y2="114.557" gradientTransform="matrix(1.4539039,0,0,1.4539039,-116.19976,-116.20474)" gradientUnits="userSpaceOnUse"><stop offset="0" style="stop-color:#dcf1f3" id="stop163"/><stop offset="1" style="stop-color:#c2c2c9" id="stop165"/></linearGradient><path id="XMLID_6_" d="m 282.89686,214.84917 c 0,0 -22.24473,-28.93269 -38.67384,-36.78377 -10.46811,-4.94327 -26.02489,-6.83335 -38.23768,-0.72695 -18.02841,9.0142 -19.91848,34.31213 -3.34397,44.34406 3.92553,2.47165 9.15959,4.50711 15.99294,6.10641 36.63838,8.43264 97.12077,25.87949 89.70587,96.10304 0,0 -4.21633,65.86185 -73.56753,73.42215 -12.2128,1.30851 -24.57098,0.43617 -36.493,-2.32625 -16.42911,-3.63476 -45.50719,-11.04967 -59.75545,-19.91849 l -2.61703,-75.16682 h 6.97875 c 0,0 13.81208,33.43978 53.06749,49.57812 7.26952,2.90781 15.26599,4.07093 22.97168,2.90781 9.74116,-1.45391 21.22699,-6.68796 25.87949,-22.53551 0,0 7.85108,-23.11707 -32.85823,-35.76604 -32.56744,-10.17733 -63.24481,-20.64543 -75.89378,-54.95757 -5.961,-16.28371 -6.97874,-34.31212 -2.90781,-51.61358 5.37944,-22.53551 20.79082,-54.23062 64.40794,-67.89732 0,0 57.28381,-15.55677 96.53922,5.52484 l -1.74468,89.70587 z" class="st4" style="fill:url(#XMLID_7_);stroke-width:1.45391"/></g></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" width="512" height="512">
<rect width="512" height="512" rx="80" ry="80" fill="#8E3131"/>
<path d="M202 268L432 78V255L202 445V268Z" fill="#FFFFFF" fill-opacity="0.6"/>
<path d="M79 256L309 66V242L79 432V256Z" fill="#FFFFFF"/>
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 302 B

app/core/src/main/resources/static/images/signature.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 17 KiB

app/core/src/test/java/stirling/software/SPDF/config/ToolIODeclarationCoverageTest.java
+4 -1
View File
@@ -61,7 +61,10 @@ class ToolIODeclarationCoverageTest {
// signing tool itself is /api/v1/security/cert-sign, which is declared.
"/api/v1/security/cert-sign/sessions",
"/api/v1/security/cert-sign/validate-certificate",
"/api/v1/security/cert-sign/hardware");
"/api/v1/security/cert-sign/hardware",
// Releases finished jobs and their stored files; server maintenance, takes and
// returns no document.
"/api/v1/general/jobs/cleanup");
private record Scan(Set<String> required, Map<String, ToolIOSpec> declared) {}
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerMoreTest.java
+14 -2
View File
@@ -24,6 +24,8 @@ import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.junit.jupiter.api.AfterEach;
@@ -120,12 +122,22 @@ class RedactControllerMoreTest {
.thenAnswer(inv -> Loader.loadPDF(pdfBytes));
}
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private byte[] singlePageTextPdf(String... lines) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
for (int i = 0; i < lines.length; i++) {
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y - i * 16f);
@@ -145,7 +157,7 @@ class RedactControllerMoreTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showText(line);
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java
+54 -361
View File
@@ -14,19 +14,25 @@ import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.Map;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSDocument;
import org.apache.pdfbox.cos.COSFloat;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSStream;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDPageTree;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.junit.jupiter.api.AfterEach;
@@ -49,6 +55,7 @@ import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import stirling.software.SPDF.model.PDFText;
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
import stirling.software.SPDF.model.api.security.RedactPdfRequest;
import stirling.software.common.model.api.security.RedactionArea;
@@ -64,9 +71,9 @@ class RedactControllerTest {
return ResponseEntity.ok(new ByteArrayResource(bytes));
}
private static byte[] drainBody(ResponseEntity<Resource> response) throws java.io.IOException {
java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
try (java.io.InputStream __in = response.getBody().getInputStream()) {
private static byte[] drainBody(ResponseEntity<Resource> response) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (InputStream __in = response.getBody().getInputStream()) {
__in.transferTo(baos);
}
return baos.toByteArray();
@@ -90,13 +97,24 @@ class RedactControllerTest {
private PDDocument realDocument;
private PDPage realPage;
private static PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
RedactControllerTest.class.getResourceAsStream(
"/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private static byte[] createSimplePdfContent() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.A4);
doc.addPage(page);
try (PDPageContentStream contentStream = new PDPageContentStream(doc, page)) {
contentStream.beginText();
contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
contentStream.setFont(helvetica(doc), 12);
contentStream.newLineAtOffset(100, 700);
contentStream.showText("This is a simple PDF.");
contentStream.endText();
@@ -156,8 +174,7 @@ class RedactControllerTest {
mockDocument = mock(PDDocument.class);
mockPages = mock(PDPageTree.class);
mockPage = mock(PDPage.class);
org.apache.pdfbox.pdmodel.PDDocumentCatalog mockCatalog =
mock(org.apache.pdfbox.pdmodel.PDDocumentCatalog.class);
PDDocumentCatalog mockCatalog = mock(PDDocumentCatalog.class);
// Setup document structure properly
when(pdfDocumentFactory.load(any(MockMultipartFile.class))).thenReturn(mockDocument);
@@ -182,9 +199,8 @@ class RedactControllerTest {
when(mockPage.hasContents()).thenReturn(true);
org.apache.pdfbox.cos.COSDocument mockCOSDocument =
mock(org.apache.pdfbox.cos.COSDocument.class);
org.apache.pdfbox.cos.COSStream mockCOSStream = mock(org.apache.pdfbox.cos.COSStream.class);
COSDocument mockCOSDocument = mock(COSDocument.class);
COSStream mockCOSStream = mock(COSStream.class);
when(mockDocument.getDocument()).thenReturn(mockCOSDocument);
when(mockCOSDocument.createCOSStream()).thenReturn(mockCOSStream);
@@ -338,17 +354,13 @@ class RedactControllerTest {
when(mockPages.get(0)).thenReturn(mockPage);
org.apache.pdfbox.pdmodel.PDDocumentInformation mockInfo =
mock(org.apache.pdfbox.pdmodel.PDDocumentInformation.class);
PDDocumentInformation mockInfo = mock(PDDocumentInformation.class);
when(mockDocument.getDocumentInformation()).thenReturn(mockInfo);
ResponseEntity<Resource> response = redactController.redactPdf(request);
assertNotNull(response);
assertEquals(200, response.getStatusCode().value());
verify(mockDocument).save(any(File.class));
verify(mockDocument).close();
}
}
@@ -753,8 +765,6 @@ class RedactControllerTest {
assertEquals(200, response.getStatusCode().value());
assertNotNull(response.getBody());
assertTrue(drainBody(response).length > 0);
verify(mockDocument, times(1)).save(any(File.class));
verify(mockDocument, times(1)).close();
}
} catch (Exception e) {
if (expectSuccess) {
@@ -788,11 +798,14 @@ class RedactControllerTest {
realPage = new PDPage(PDRectangle.A4);
realDocument.addPage(realPage);
// Set up basic page resources
PDResources resources = new PDResources();
resources.put(
COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
realPage.setResources(resources);
// Set up basic page resources with embedded font
try {
PDResources resources = new PDResources();
resources.put(COSName.getPDFName("F1"), helvetica(realDocument));
realPage.setResources(resources);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
// Helper methods for real PDF content creation
@@ -803,8 +816,7 @@ class RedactControllerTest {
}
realDocument.addPage(realPage);
realPage.setResources(new PDResources());
realPage.getResources()
.put(COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
realPage.getResources().put(COSName.getPDFName("F1"), helvetica(realDocument));
try (PDPageContentStream contentStream = new PDPageContentStream(realDocument, realPage)) {
contentStream.beginText();
@@ -822,8 +834,7 @@ class RedactControllerTest {
}
realDocument.addPage(realPage);
realPage.setResources(new PDResources());
realPage.getResources()
.put(COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
realPage.getResources().put(COSName.getPDFName("F1"), helvetica(realDocument));
try (PDPageContentStream contentStream = new PDPageContentStream(realDocument, realPage)) {
contentStream.beginText();
@@ -846,8 +857,7 @@ class RedactControllerTest {
}
realDocument.addPage(realPage);
realPage.setResources(new PDResources());
realPage.getResources()
.put(COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
realPage.getResources().put(COSName.getPDFName("F1"), helvetica(realDocument));
try (PDPageContentStream contentStream = new PDPageContentStream(realDocument, realPage)) {
contentStream.setLineWidth(2);
@@ -874,8 +884,7 @@ class RedactControllerTest {
}
realDocument.addPage(realPage);
realPage.setResources(new PDResources());
realPage.getResources()
.put(COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
realPage.getResources().put(COSName.getPDFName("F1"), helvetica(realDocument));
try (PDPageContentStream contentStream = new PDPageContentStream(realDocument, realPage)) {
contentStream.beginText();
@@ -1005,22 +1014,6 @@ class RedactControllerTest {
}
}
private List<Object> getOriginalTokens() throws Exception {
// Create a new page to avoid side effects from other tests
PDPage pageForTokenExtraction = new PDPage(PDRectangle.A4);
pageForTokenExtraction.setResources(realPage.getResources());
try (PDPageContentStream contentStream =
new PDPageContentStream(realDocument, pageForTokenExtraction)) {
contentStream.beginText();
contentStream.setFont(realPage.getResources().getFont(COSName.getPDFName("F1")), 12);
contentStream.newLineAtOffset(50, 750);
contentStream.showText("Original content");
contentStream.endText();
}
return textRedactionService.createTokensWithoutTargetText(
realDocument, pageForTokenExtraction, Collections.emptySet(), false, false);
}
@Nested
@DisplayName("Color Decoding Utility Tests")
class ColorDecodingTests {
@@ -1099,318 +1092,19 @@ class RedactControllerTest {
class ContentStreamUnitTests {
@Test
@DisplayName("createTokensWithoutTargetText should remove simple text tokens")
void shouldRemoveSimpleTextTokens() throws Exception {
createRealPageWithSimpleText("This document contains confidential information.");
@DisplayName("performTextReplacement should process document text replacement")
void shouldPerformTextReplacement() throws Exception {
createRealPageWithSimpleText("This document contains sensitive information.");
String[] targetWords = new String[] {"sensitive"};
Set<String> targetWords = Set.of("confidential");
Map<Integer, List<PDFText>> found =
textRedactionService.findTextToRedact(realDocument, targetWords, false, false);
assertFalse(found.isEmpty(), "Should find target text to redact");
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
assertNotNull(tokens);
assertFalse(tokens.isEmpty());
String reconstructedText = extractTextFromTokens(tokens);
assertFalse(
reconstructedText.contains("confidential"),
"Target text should be replaced with placeholder");
assertTrue(reconstructedText.contains("document"), "Non-target text should remain");
}
@Test
@DisplayName("createTokensWithoutTargetText should handle TJ operator arrays")
void shouldHandleTJOperatorArrays() throws Exception {
createRealPageWithTJArrayText();
Set<String> targetWords = Set.of("secret");
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
assertNotNull(tokens);
boolean foundModifiedTJArray = false;
for (Object token : tokens) {
if (token instanceof COSArray array) {
for (int i = 0; i < array.size(); i++) {
if (array.getObject(i) instanceof COSString cosString) {
String text = cosString.getString();
if (text.contains("secret")) {
fail(
"Target text 'secret' should have been redacted from TJ"
+ " array");
}
foundModifiedTJArray = true;
}
}
}
}
assertTrue(foundModifiedTJArray, "Should find at least one TJ array");
}
@Test
@DisplayName("createTokensWithoutTargetText should preserve non-text tokens")
void shouldPreserveNonTextTokens() throws Exception {
createRealPageWithMixedContent();
Set<String> targetWords = Set.of("redact");
List<Object> originalTokens = getOriginalTokens();
List<Object> filteredTokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
long originalNonTextCount =
originalTokens.stream()
.filter(
token ->
token instanceof Operator op
&& !textRedactionService.isTextShowingOperator(
op.getName()))
.count();
long filteredNonTextCount =
filteredTokens.stream()
.filter(
token ->
token instanceof Operator op
&& !textRedactionService.isTextShowingOperator(
op.getName()))
.count();
assertTrue(filteredNonTextCount > 0, "Non-text operators should be preserved");
assertTrue(
filteredNonTextCount >= originalNonTextCount / 2,
"A reasonable number of non-text operators should be preserved");
}
@Test
@DisplayName("createTokensWithoutTargetText should handle regex patterns")
void shouldHandleRegexPatterns() throws Exception {
createRealPageWithSimpleText("Phone: 123-456-7890 and SSN: 111-22-3333");
Set<String> targetWords = Set.of("\\d{3}-\\d{2}-\\d{4}"); // SSN pattern
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, true, false);
String reconstructedText = extractTextFromTokens(tokens);
assertFalse(reconstructedText.contains("111-22-3333"), "SSN should be redacted");
assertTrue(reconstructedText.contains("123-456-7890"), "Phone should remain");
}
@Test
@DisplayName("createTokensWithoutTargetText should handle whole word search")
void shouldHandleWholeWordSearch() throws Exception {
createRealPageWithSimpleText("This test testing tested document");
Set<String> targetWords = Set.of("test");
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, true);
String reconstructedText = extractTextFromTokens(tokens);
assertTrue(reconstructedText.contains("testing"), "Partial matches should remain");
assertTrue(reconstructedText.contains("tested"), "Partial matches should remain");
}
@ParameterizedTest
@ValueSource(strings = {"Tj", "TJ", "'", "\""})
@DisplayName("createTokensWithoutTargetText should handle all text operators")
void shouldHandleAllTextOperators(String operatorName) throws Exception {
createRealPageWithSpecificOperator(operatorName);
Set<String> targetWords = Set.of("sensitive");
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
String reconstructedText = extractTextFromTokens(tokens);
assertFalse(
reconstructedText.contains("sensitive"),
"Text should be redacted regardless of operator type");
}
@Test
@DisplayName("writeFilteredContentStream should write tokens to new stream")
void shouldWriteTokensToNewContentStream() throws Exception {
List<Object> tokens = createSampleTokenList();
textRedactionService.writeFilteredContentStream(realDocument, realPage, tokens);
assertNotNull(realPage.getContents(), "Page should have content stream");
// Verify the content can be read back
try (InputStream inputStream = realPage.getContents()) {
byte[] content = readAllBytes(inputStream);
assertTrue(content.length > 0, "Content stream should not be empty");
}
}
@Test
@DisplayName("writeFilteredContentStream should handle empty token list")
void shouldHandleEmptyTokenList() throws Exception {
List<Object> emptyTokens = Collections.emptyList();
assertDoesNotThrow(
() ->
textRedactionService.writeFilteredContentStream(
realDocument, realPage, emptyTokens));
assertNotNull(realPage.getContents(), "Page should still have content stream");
}
@Test
@DisplayName("writeFilteredContentStream should replace existing content")
void shouldReplaceExistingContentStream() throws Exception {
createRealPageWithSimpleText("Original content");
String originalContent = extractTextFromModifiedPage(realPage);
List<Object> newTokens = createSampleTokenList();
textRedactionService.writeFilteredContentStream(realDocument, realPage, newTokens);
String newContent = extractTextFromModifiedPage(realPage);
assertNotEquals(originalContent, newContent, "Content stream should be replaced");
}
@Test
@DisplayName("Placeholder creation should maintain text width")
void shouldCreateWidthMatchingPlaceholder() {
String originalText = "confidential";
String placeholder =
textRedactionService.createPlaceholderWithFont(
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
assertEquals(
originalText.length(),
placeholder.length(),
"Placeholder should maintain character count for width preservation");
}
@Test
@DisplayName("Placeholder should handle special characters")
void shouldHandleSpecialCharactersInPlaceholder() {
String originalText = "café naïve";
String placeholder =
textRedactionService.createPlaceholderWithFont(
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
assertEquals(originalText.length(), placeholder.length());
assertFalse(
placeholder.contains("café"), "Placeholder should not contain original text");
}
@Test
@DisplayName("Integration test: createTokens and writeStream")
void shouldIntegrateTokenCreationAndWriting() throws Exception {
createRealPageWithSimpleText("This document contains secret information.");
Set<String> targetWords = Set.of("secret");
List<Object> filteredTokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
textRedactionService.writeFilteredContentStream(realDocument, realPage, filteredTokens);
assertNotNull(realPage.getContents());
String finalText = extractTextFromModifiedPage(realPage);
assertFalse(finalText.contains("secret"), "Target text should be completely removed");
assertTrue(finalText.contains("document"), "Other text should remain");
}
@Test
@DisplayName("Should preserve text positioning operators")
void shouldPreserveTextPositioning() throws Exception {
createRealPageWithPositionedText();
Set<String> targetWords = Set.of("confidential");
List<Object> filteredTokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
long filteredPositioning =
filteredTokens.stream()
.filter(
token ->
token instanceof Operator op
&& ("Td".equals(op.getName())
|| "TD".equals(op.getName())
|| "Tm".equals(op.getName())))
.count();
assertTrue(filteredPositioning > 0, "Positioning operators should be preserved");
}
@Test
@DisplayName("Should handle complex content streams with multiple operators")
void shouldHandleComplexContentStreams() throws Exception {
realPage = new PDPage(PDRectangle.A4);
while (realDocument.getNumberOfPages() > 0) {
realDocument.removePage(0);
}
realDocument.addPage(realPage);
realPage.setResources(new PDResources());
realPage.getResources()
.put(
COSName.getPDFName("F1"),
new PDType1Font(Standard14Fonts.FontName.HELVETICA));
try (PDPageContentStream contentStream =
new PDPageContentStream(realDocument, realPage)) {
contentStream.setLineWidth(2);
contentStream.moveTo(100, 100);
contentStream.lineTo(200, 200);
contentStream.stroke();
contentStream.beginText();
contentStream.setFont(
realPage.getResources().getFont(COSName.getPDFName("F1")), 12);
contentStream.newLineAtOffset(50, 750);
contentStream.showText("This is a complex document with ");
contentStream.setTextRise(5);
contentStream.showText("confidential");
contentStream.setTextRise(0);
contentStream.showText(" information.");
contentStream.endText();
contentStream.addRect(300, 300, 100, 100);
contentStream.fill();
}
Set<String> targetWords = Set.of("confidential");
List<Object> tokens =
textRedactionService.createTokensWithoutTargetText(
realDocument, realPage, targetWords, false, false);
assertNotNull(tokens);
assertFalse(tokens.isEmpty());
String reconstructedText = extractTextFromTokens(tokens);
assertFalse(
reconstructedText.contains("confidential"), "Target text should be redacted");
boolean hasGraphicsOperators =
tokens.stream()
.anyMatch(
token ->
token instanceof Operator op
&& ("re".equals(op.getName())
|| "f".equals(op.getName())
|| "m".equals(op.getName())
|| "l".equals(op.getName())
|| "S".equals(op.getName())));
assertTrue(hasGraphicsOperators, "Graphics operators should be preserved");
boolean fallback =
textRedactionService.performTextReplacement(
realDocument, found, targetWords, false, false);
assertFalse(fallback, "JPDFium text replacement should complete without fallback");
}
@Test
@@ -1423,14 +1117,13 @@ class RedactControllerTest {
realDocument.addPage(realPage);
PDResources resources = new PDResources();
resources.put(
COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
resources.put(COSName.getPDFName("F1"), helvetica(realDocument));
realPage.setResources(resources);
try (PDPageContentStream contentStream =
new PDPageContentStream(realDocument, realPage)) {
contentStream.beginText();
contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
contentStream.setFont(helvetica(realDocument), 12);
contentStream.newLineAtOffset(50, 750);
contentStream.showText("This is the first text block");
contentStream.endText();
@@ -1441,7 +1134,7 @@ class RedactControllerTest {
contentStream.stroke();
contentStream.beginText();
contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
contentStream.setFont(helvetica(realDocument), 12);
contentStream.newLineAtOffset(50, 650);
contentStream.showText("This block contains confidential information");
contentStream.endText();
@@ -1450,7 +1143,7 @@ class RedactControllerTest {
contentStream.fill();
contentStream.beginText();
contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
contentStream.setFont(helvetica(realDocument), 12);
contentStream.newLineAtOffset(50, 550);
contentStream.showText("This is the third text block");
contentStream.endText();
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceMoreTest.java
+17 -5
View File
@@ -10,6 +10,7 @@ import static org.mockito.Mockito.mock;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.util.ArrayList;
@@ -22,6 +23,8 @@ import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
@@ -31,6 +34,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.SPDF.model.PDFText;
@@ -118,18 +122,26 @@ class RedactExecuteServiceMoreTest {
private RedactExecuteRequest requestFor(byte[] pdfBytes) {
RedactExecuteRequest req = new RedactExecuteRequest();
req.setFileInput(
new org.springframework.mock.web.MockMultipartFile(
"fileInput", "in.pdf", "application/pdf", pdfBytes));
req.setFileInput(new MockMultipartFile("fileInput", "in.pdf", "application/pdf", pdfBytes));
return req;
}
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private byte[] singlePageTextPdf(String... lines) throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
for (int i = 0; i < lines.length; i++) {
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y - i * LINE_H);
@@ -149,7 +161,7 @@ class RedactExecuteServiceMoreTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showText("page " + p + " has SECRET content here");
app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceTest.java
+18 -5
View File
@@ -3,6 +3,7 @@ package stirling.software.SPDF.controller.api.security;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -11,6 +12,8 @@ import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.junit.jupiter.api.DisplayName;
@@ -219,12 +222,22 @@ class RedactExecuteServiceTest {
* anchor) 1: line one 2: line two 3: line three 4: STOP-HERE (end anchor) 5: line five (must
* NOT be redacted)
*/
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private PDDocument buildSingleColumnDoc() throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
String[] lines = {
"START-HERE", "line one", "line two", "line three", "STOP-HERE", "line five"
};
@@ -247,7 +260,7 @@ class RedactExecuteServiceTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
// Body lines are padded to make each column genuinely wide enough that column
// detection (which ignores narrow lines) treats both sides as real columns.
String fill = " " + "x".repeat(26);
@@ -294,7 +307,7 @@ class RedactExecuteServiceTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
String[] lines = {
"#1 Auto layout",
"Body about auto layout.",
@@ -331,7 +344,7 @@ class RedactExecuteServiceTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
// Header — full width, lines 0..1.
for (int i = 0; i < 2; i++) {
cs.beginText();
@@ -383,7 +396,7 @@ class RedactExecuteServiceTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
float dateX = PAGE_WIDTH - 144f; // right-aligned dates near the right margin
app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceExtraTest.java
+10 -504
View File
@@ -3,44 +3,24 @@ package stirling.software.SPDF.controller.api.security;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSFloat;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdfparser.PDFStreamParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import stirling.software.SPDF.model.PDFText;
/**
* Further gap-coverage tests for {@link TextRedactionService}, complementing {@code
* TextRedactionServiceTest} and {@code TextRedactionServiceMoreTest}. These target branches the
* other two suites leave untouched: case-sensitive vs regex find, multi-term and multi-match within
* one segment, the kerning ({@code adjustment != 0}) path that rewrites a {@code Tj} into a {@code
* TJ} array, nested Form XObject traversal, pages with no resources, and the private width helpers
* exercised directly via reflection.
*/
@DisplayName("TextRedactionService extra coverage")
class TextRedactionServiceExtraTest {
@@ -50,43 +30,22 @@ class TextRedactionServiceExtraTest {
private final TextRedactionService service = new TextRedactionService();
private PDFont helvetica() {
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private List<Object> parseTokens(PDPage page) throws IOException {
PDFStreamParser parser = new PDFStreamParser(page);
List<Object> tokens = new ArrayList<>();
Object t;
while ((t = parser.parseNextToken()) != null) {
tokens.add(t);
}
return tokens;
}
private String tokensText(List<Object> tokens) {
StringBuilder sb = new StringBuilder();
for (Object token : tokens) {
if (token instanceof COSString cs) {
sb.append(cs.getString());
} else if (token instanceof COSArray arr) {
for (COSBase el : arr) {
if (el instanceof COSString cs) {
sb.append(cs.getString());
}
}
}
}
return sb.toString();
}
/** Single page, one Tj line per supplied text line, Helvetica 12. */
private PDDocument buildDoc(String... lines) throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(helvetica(), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
for (int i = 0; i < lines.length; i++) {
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y - i * 16f);
@@ -97,25 +56,6 @@ class TextRedactionServiceExtraTest {
return doc;
}
/** Page whose content stream is exactly {@code rawContent}, font F1=Helvetica. */
private PDDocument docWithRawContent(String rawContent) throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDResources resources = new PDResources();
resources.put(COSName.getPDFName("F1"), helvetica());
page.setResources(resources);
PDStream stream = new PDStream(doc);
try (var out = stream.createOutputStream()) {
out.write(rawContent.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(stream);
return doc;
}
// ── findTextToRedact: matching modes ─────────────────────────────────────────────────────────
@Nested
@DisplayName("findTextToRedact matching modes")
class FindModes {
@@ -126,7 +66,6 @@ class TextRedactionServiceExtraTest {
try (PDDocument doc = buildDoc("Secret and secret and SECRET")) {
Map<Integer, List<PDFText>> result =
service.findTextToRedact(doc, new String[] {"secret"}, false, false);
// Patterns are compiled CASE_INSENSITIVE, so all three occurrences match.
assertThat(result.get(0)).hasSize(3);
}
}
@@ -148,7 +87,6 @@ class TextRedactionServiceExtraTest {
try (PDDocument doc = buildDoc("abcde")) {
Map<Integer, List<PDFText>> result =
service.findTextToRedact(doc, new String[] {"[ae]"}, true, false);
// 'a' and 'e' both match -> two single-character hits.
assertThat(result.get(0)).hasSize(2);
}
}
@@ -163,436 +101,4 @@ class TextRedactionServiceExtraTest {
}
}
}
// ── createTokensWithoutTargetText structural branches ────────────────────────────────────────
@Nested
@DisplayName("createTokensWithoutTargetText structural branches")
class TokenStructural {
@Test
@DisplayName("page with null resources still parses and redacts the matched Tj text")
void nullResourcesStillRedacts() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
// No resources set; the content stream references no real font.
String raw = "BT 72 700 Td (SECRET) Tj ET";
PDStream stream = new PDStream(doc);
try (var out = stream.createOutputStream()) {
out.write(raw.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(stream);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokensText(tokens)).doesNotContain("SECRET");
}
}
@Test
@DisplayName("a match inside a Tj segment is redacted and surrounding text survives")
void multipleMatchesOneSegment() throws IOException {
try (PDDocument doc = docWithRawContent("BT /F1 12 Tf 72 700 Td (xAAxAAx) Tj ET")) {
PDPage page = doc.getPage(0);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("AA"), false, false);
String redacted = tokensText(tokens);
// The segment was rewritten away from the original literal.
assertThat(redacted).isNotEqualTo("xAAxAAx");
// Redaction replaces matched runs with whitespace, so at least one "AA" is gone
// (the leading occurrence) and the surrounding x characters survive.
assertThat(redacted.split("AA", -1).length - 1).isLessThan(2);
assertThat(redacted).startsWith("x ");
assertThat(redacted).contains("x");
}
}
@Test
@DisplayName("a second Tf operator updates the active font for later segments")
void secondTfUpdatesFont() throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDResources resources = new PDResources();
resources.put(COSName.getPDFName("F1"), helvetica());
resources.put(
COSName.getPDFName("F2"),
new PDType1Font(Standard14Fonts.FontName.TIMES_ROMAN));
page.setResources(resources);
String raw = "BT /F1 12 Tf 72 700 Td (first) Tj /F2 18 Tf 0 -20 Td (SECRET) Tj ET";
PDStream stream = new PDStream(doc);
try (var out = stream.createOutputStream()) {
out.write(raw.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(stream);
try (doc) {
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokensText(tokens)).doesNotContain("SECRET");
assertThat(tokensText(tokens)).contains("first");
}
}
}
// ── nested Form XObject traversal ────────────────────────────────────────────────────────────
@Nested
@DisplayName("nested Form XObject traversal")
class NestedXObjects {
@Test
@DisplayName("a match in a form nested two levels deep is reached and rewritten")
void nestedTwoLevelsDeep() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
// Inner form shows SECRET.
PDFormXObject inner = new PDFormXObject(doc);
inner.setResources(new PDResources());
inner.getResources().put(COSName.getPDFName("F1"), helvetica());
inner.setBBox(new PDRectangle(0, 0, 200, 50));
try (var out = inner.getStream().createOutputStream()) {
out.write(
"BT /F1 12 Tf 0 10 Td (SECRET) Tj ET"
.getBytes(StandardCharsets.ISO_8859_1));
}
// Outer form references inner via Do.
PDFormXObject outer = new PDFormXObject(doc);
PDResources outerRes = new PDResources();
COSName innerName = outerRes.add(inner);
outer.setResources(outerRes);
outer.setBBox(new PDRectangle(0, 0, 200, 50));
try (var out = outer.getStream().createOutputStream()) {
out.write(
("/" + innerName.getName() + " Do")
.getBytes(StandardCharsets.ISO_8859_1));
}
PDResources pageRes = new PDResources();
COSName outerName = pageRes.add(outer);
page.setResources(pageRes);
PDStream pageStream = new PDStream(doc);
try (var out = pageStream.createOutputStream()) {
out.write(
("/" + outerName.getName() + " Do")
.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(pageStream);
service.createTokensWithoutTargetText(doc, page, Set.of("SECRET"), false, false);
// The deep traversal must have rewritten the inner form's content stream.
assertThat(inner.getCOSObject().containsKey(COSName.CONTENTS)).isTrue();
}
}
@Test
@DisplayName("a form XObject with no resources is skipped without error")
void formWithoutResourcesSkipped() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDFormXObject form = new PDFormXObject(doc);
form.setBBox(new PDRectangle(0, 0, 100, 50));
// Intentionally no resources on the form.
try (var out = form.getStream().createOutputStream()) {
out.write("q Q".getBytes(StandardCharsets.ISO_8859_1));
}
PDResources pageRes = new PDResources();
COSName formName = pageRes.add(form);
page.setResources(pageRes);
PDStream pageStream = new PDStream(doc);
try (var out = pageStream.createOutputStream()) {
out.write(
("/" + formName.getName() + " Do")
.getBytes(StandardCharsets.ISO_8859_1));
}
page.setContents(pageStream);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokens).isNotNull();
}
}
}
// ── kerning / adjustment path in modifyTokenForRedaction ─────────────────────────────────────
@Nested
@DisplayName("modifyTokenForRedaction adjustment branches")
class ModifyTokenAdjustment {
@Test
@DisplayName("a non-zero width adjustment rewrites a Tj into a TJ array with kerning")
void adjustmentRewritesToTjArray() throws Exception {
List<Object> tokens = new ArrayList<>();
tokens.add(new COSString("KEEP"));
tokens.add(Operator.getOperator("Tj"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "Tj", "KEEP", 0, 4, helvetica(), FONT_SIZE);
Method m =
TextRedactionService.class.getDeclaredMethod(
"modifyTokenForRedaction",
List.class,
TextRedactionService.TextSegment.class,
String.class,
float.class,
List.class);
m.setAccessible(true);
// A clearly non-zero adjustment forces the COSArray + kerning branch.
m.invoke(service, tokens, segment, "AB", 5.0f, List.of());
assertThat(tokens.get(0)).isInstanceOf(COSArray.class);
COSArray arr = (COSArray) tokens.get(0);
boolean hasKern = false;
for (COSBase el : arr) {
if (el instanceof COSFloat) {
hasKern = true;
}
}
assertThat(hasKern).as("kerning float should be appended to the TJ array").isTrue();
// The trailing Tj operator should have been switched to TJ.
assertThat(tokens.get(1)).isInstanceOf(Operator.class);
assertThat(((Operator) tokens.get(1)).getName()).isEqualTo("TJ");
}
@Test
@DisplayName("empty replacement text with ~zero adjustment sets the shared empty COSString")
void emptyReplacementZeroAdjustment() throws Exception {
List<Object> tokens = new ArrayList<>();
tokens.add(new COSString("SECRET"));
tokens.add(Operator.getOperator("Tj"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "Tj", "SECRET", 0, 6, helvetica(), FONT_SIZE);
Method m =
TextRedactionService.class.getDeclaredMethod(
"modifyTokenForRedaction",
List.class,
TextRedactionService.TextSegment.class,
String.class,
float.class,
List.class);
m.setAccessible(true);
m.invoke(service, tokens, segment, "", 0f, List.of());
assertThat(tokens.get(0)).isInstanceOf(COSString.class);
assertThat(((COSString) tokens.get(0)).getString()).isEmpty();
}
@Test
@DisplayName("the ' operator with a non-zero adjustment is also rewritten to a TJ array")
void apostropheAdjustmentRewrites() throws Exception {
List<Object> tokens = new ArrayList<>();
tokens.add(new COSString("WORD"));
tokens.add(Operator.getOperator("'"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "'", "WORD", 0, 4, helvetica(), FONT_SIZE);
Method m =
TextRedactionService.class.getDeclaredMethod(
"modifyTokenForRedaction",
List.class,
TextRedactionService.TextSegment.class,
String.class,
float.class,
List.class);
m.setAccessible(true);
m.invoke(service, tokens, segment, "X", 4.0f, List.of());
assertThat(tokens.get(0)).isInstanceOf(COSArray.class);
assertThat(((Operator) tokens.get(1)).getName()).isEqualTo("TJ");
}
}
// ── createRedactedTJArray edge branches ──────────────────────────────────────────────────────
@Nested
@DisplayName("createRedactedTJArray edge branches")
class RedactedTjArray {
@Test
@DisplayName("non-COSString elements (kerning numbers) are preserved in order")
void preservesNumberElements() throws Exception {
COSArray original = new COSArray();
original.add(new COSString("AA"));
original.add(new COSFloat(-25f));
original.add(new COSString("BB"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "TJ", "AABB", 0, 4, helvetica(), FONT_SIZE);
List<TextRedactionService.MatchRange> matches =
List.of(new TextRedactionService.MatchRange(0, 2)); // "AA"
Method m =
TextRedactionService.class.getDeclaredMethod(
"createRedactedTJArray",
COSArray.class,
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
COSArray result = (COSArray) m.invoke(service, original, segment, matches);
boolean sawFloat = false;
for (COSBase el : result) {
if (el instanceof COSFloat) {
sawFloat = true;
}
}
assertThat(sawFloat).as("original kerning number must be retained").isTrue();
}
@Test
@DisplayName("a TJ array with no overlapping match is returned essentially unchanged")
void noMatchLeavesTextIntact() throws Exception {
COSArray original = new COSArray();
original.add(new COSString("hello"));
original.add(new COSString("world"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "TJ", "helloworld", 0, 10, helvetica(), FONT_SIZE);
List<TextRedactionService.MatchRange> matches =
List.of(new TextRedactionService.MatchRange(50, 60)); // out of range
Method m =
TextRedactionService.class.getDeclaredMethod(
"createRedactedTJArray",
COSArray.class,
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
COSArray result = (COSArray) m.invoke(service, original, segment, matches);
StringBuilder sb = new StringBuilder();
for (COSBase el : result) {
if (el instanceof COSString cs) sb.append(cs.getString());
}
assertThat(sb.toString()).isEqualTo("helloworld");
}
}
// ── private width helpers via reflection ─────────────────────────────────────────────────────
@Nested
@DisplayName("private width helpers via reflection")
class WidthHelpers {
private float invokeFloat(String name, Object... args) throws Exception {
Class<?>[] types = new Class<?>[] {PDFont.class, String.class};
Method m = TextRedactionService.class.getDeclaredMethod(name, types);
m.setAccessible(true);
return (float) m.invoke(service, args);
}
@Test
@DisplayName("calculateConservativeWidth scales linearly at 500 units per character")
void conservativeWidthLinear() throws Exception {
float w = invokeFloat("calculateConservativeWidth", helvetica(), "abcd");
assertThat(w).isEqualTo(4 * 500f);
}
@Test
@DisplayName("calculateCharacterBasedWidth returns a positive width for normal text")
void characterBasedWidthPositive() throws Exception {
float w = invokeFloat("calculateCharacterBasedWidth", helvetica(), "Hello");
assertThat(w).isGreaterThan(0f);
}
@Test
@DisplayName("calculateFallbackWidth returns a positive width using font metrics")
void fallbackWidthPositive() throws Exception {
float w = invokeFloat("calculateFallbackWidth", helvetica(), "Hello");
assertThat(w).isGreaterThan(0f);
}
@Test
@DisplayName("safeGetStringWidth returns 0 for null/empty inputs")
void safeWidthZeroForEmpty() throws Exception {
assertThat(invokeFloat("safeGetStringWidth", helvetica(), "")).isZero();
Method m =
TextRedactionService.class.getDeclaredMethod(
"safeGetStringWidth", PDFont.class, String.class);
m.setAccessible(true);
assertThat((float) m.invoke(service, helvetica(), null)).isZero();
assertThat((float) m.invoke(service, (PDFont) null, "x")).isZero();
}
@Test
@DisplayName("safeGetStringWidth returns a positive width for a reliable font")
void safeWidthPositive() throws Exception {
float w = invokeFloat("safeGetStringWidth", helvetica(), "Word");
assertThat(w).isGreaterThan(0f);
}
}
// ── createAlternativePlaceholder via reflection ──────────────────────────────────────────────
@Nested
@DisplayName("createAlternativePlaceholder via reflection")
class AlternativePlaceholder {
@Test
@DisplayName("Helvetica supports space, so output is a bounded run of spaces")
void boundedSpaces() throws Exception {
Method m =
TextRedactionService.class.getDeclaredMethod(
"createAlternativePlaceholder",
String.class,
float.class,
PDFont.class,
float.class);
m.setAccessible(true);
String result = (String) m.invoke(service, "hidden", 20f, helvetica(), FONT_SIZE);
assertThat(result.chars().allMatch(c -> c == ' ')).isTrue();
assertThat(result.length()).isLessThanOrEqualTo("hidden".length() * 2);
}
}
// ── extractTextSegments via reflection ───────────────────────────────────────────────────────
@Nested
@DisplayName("extractTextSegments via reflection")
class ExtractSegments {
@SuppressWarnings("unchecked")
@Test
@DisplayName("a Tf operator sets font and size on the segments that follow it")
void tfSetsFontAndSize() throws Exception {
try (PDDocument doc = docWithRawContent("BT /F1 14 Tf 72 700 Td (hello) Tj ET")) {
PDPage page = doc.getPage(0);
List<Object> tokens = parseTokens(page);
Method m =
TextRedactionService.class.getDeclaredMethod(
"extractTextSegments", PDPage.class, List.class);
m.setAccessible(true);
List<TextRedactionService.TextSegment> segments =
(List<TextRedactionService.TextSegment>) m.invoke(service, page, tokens);
assertThat(segments).isNotEmpty();
TextRedactionService.TextSegment first = segments.get(0);
assertThat(first.getText()).isEqualTo("hello");
assertThat(first.getFontSize()).isEqualTo(14f);
assertThat(first.getFont()).isNotNull();
}
}
}
}
app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceMoreTest.java
+29 -423
View File
@@ -3,40 +3,24 @@ package stirling.software.SPDF.controller.api.security;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSFloat;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdfparser.PDFStreamParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.common.PDStream;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import stirling.software.SPDF.model.PDFText;
/**
* Gap-coverage tests for {@link TextRedactionService} targeting branches the existing {@code
* TextRedactionServiceTest} does not reach: TJ-array redaction with kerning adjustment, the {@code
* '} and {@code "} text-showing operators, Form XObject content rewriting, multi-page / multi-match
* find+replace, and the private TJ/segment helpers exercised directly via reflection.
*/
@DisplayName("TextRedactionService additional coverage")
class TextRedactionServiceMoreTest {
@@ -50,201 +34,16 @@ class TextRedactionServiceMoreTest {
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private List<Object> parseTokens(PDPage page) throws IOException {
PDFStreamParser parser = new PDFStreamParser(page);
List<Object> tokens = new ArrayList<>();
Object t;
while ((t = parser.parseNextToken()) != null) {
tokens.add(t);
}
return tokens;
}
private String tokensText(List<Object> tokens) {
StringBuilder sb = new StringBuilder();
for (Object token : tokens) {
if (token instanceof COSString cs) {
sb.append(cs.getString());
} else if (token instanceof COSArray arr) {
for (COSBase el : arr) {
if (el instanceof COSString cs) {
sb.append(cs.getString());
}
}
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return sb.toString();
return helvetica();
}
/**
* Builds a single page whose content stream is exactly {@code rawContent}, font F1=Helvetica.
*/
private PDDocument docWithRawContent(String rawContent) throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
PDResources resources = new PDResources();
resources.put(COSName.getPDFName("F1"), helvetica());
page.setResources(resources);
PDStream stream = new PDStream(doc);
try (var out = stream.createOutputStream()) {
out.write(rawContent.getBytes(java.nio.charset.StandardCharsets.ISO_8859_1));
}
page.setContents(stream);
return doc;
}
// ── ' and " operators ────────────────────────────────────────────────────────────────────────
@Nested
@DisplayName("apostrophe and quote text-showing operators")
class MoveAndShowOperators {
@Test
@DisplayName("the ' (move-to-next-line-and-show) operator gets its text redacted")
void apostropheOperatorRedacted() throws IOException {
// ' shows a string on the next line. Content: BT /F1 12 Tf 72 700 Td (PUBLIC) Tj
// (SECRET) ' ET
String raw = "BT /F1 12 Tf 72 700 Td (PUBLIC) Tj (SECRET) ' ET";
try (PDDocument doc = docWithRawContent(raw)) {
PDPage page = doc.getPage(0);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
String text = tokensText(tokens);
assertThat(text).doesNotContain("SECRET");
assertThat(text).contains("PUBLIC");
}
}
@Test
@DisplayName("the \" operator is collected as text-showing but its text is not extracted")
void quoteOperatorNotExtracted() throws IOException {
// " is in TEXT_SHOWING_OPERATORS, but extractTextFromToken's switch only handles
// Tj/'/TJ, so a "-shown string yields no segment and survives. This pins that
// behavior: the parse path runs without error and the token list is intact.
String raw = "BT /F1 12 Tf 72 700 Td 1 2 (SECRET) \" ET";
try (PDDocument doc = docWithRawContent(raw)) {
PDPage page = doc.getPage(0);
List<Object> before = parseTokens(page);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokens).hasSameSizeAs(before);
assertThat(tokensText(tokens)).contains("SECRET");
}
}
}
// ── TJ arrays with kerning ───────────────────────────────────────────────────────────────────
@Nested
@DisplayName("TJ positioning arrays")
class TjArrays {
@Test
@DisplayName("partial match inside a TJ array redacts only the matched run")
void tjArrayPartialRedaction() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(helvetica(), FONT_SIZE);
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y);
// showTextWithPositioning emits a single TJ array.
cs.showTextWithPositioning(
new Object[] {"keep ", -50f, "SECRET", 20f, " tail"});
cs.endText();
}
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
boolean sawTj = tokens.stream().anyMatch(t -> t instanceof COSArray);
assertThat(sawTj).as("expected a TJ array token").isTrue();
assertThat(tokensText(tokens)).doesNotContain("SECRET");
assertThat(tokensText(tokens)).contains("keep");
}
}
@Test
@DisplayName("TJ array with no matching term is left unchanged")
void tjArrayNoMatch() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(helvetica(), FONT_SIZE);
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showTextWithPositioning(new Object[] {"alpha ", -30f, "beta"});
cs.endText();
}
List<Object> before = parseTokens(page);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("ZZZ"), false, false);
assertThat(tokens).hasSameSizeAs(before);
assertThat(tokensText(tokens)).contains("alpha");
}
}
}
// ── Form XObject traversal ───────────────────────────────────────────────────────────────────
@Nested
@DisplayName("Form XObject content")
class FormXObjects {
@Test
@DisplayName("a referenced Form XObject containing a match is traversed and rewritten")
void traversesFormXObject() throws IOException {
try (PDDocument doc = new PDDocument()) {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
// Build a form XObject whose own content stream shows "SECRET".
PDFormXObject form = new PDFormXObject(doc);
form.setResources(new PDResources());
form.getResources().put(COSName.getPDFName("F1"), helvetica());
form.setBBox(new PDRectangle(0, 0, 200, 50));
String formContent = "BT /F1 12 Tf 0 10 Td (SECRET) Tj ET";
try (var out = form.getStream().createOutputStream()) {
out.write(formContent.getBytes(java.nio.charset.StandardCharsets.ISO_8859_1));
}
PDResources pageResources = new PDResources();
COSName formName = pageResources.add(form);
page.setResources(pageResources);
String pageContent = "q 1 0 0 1 100 600 cm /" + formName.getName() + " Do Q";
PDStream pageStream = new PDStream(doc);
try (var out = pageStream.createOutputStream()) {
out.write(pageContent.getBytes(java.nio.charset.StandardCharsets.ISO_8859_1));
}
page.setContents(pageStream);
// Processing the page walks into the XObject graph; when a match is found inside
// the
// form, writeRedactedContentToXObject runs and sets a /Contents item on the form's
// COS dictionary. Asserting that item appears proves the XObject redaction path
// executed end-to-end without throwing.
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertThat(tokens).isNotNull();
assertThat(form.getCOSObject().containsKey(COSName.CONTENTS))
.as("form XObject redaction path should have written a new content item")
.isTrue();
}
}
}
// ── multi-page / multi-match public entry points ─────────────────────────────────────────────
@Nested
@DisplayName("findTextToRedact and performTextReplacement across pages")
class MultiPage {
@@ -256,7 +55,7 @@ class TextRedactionServiceMoreTest {
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(helvetica(), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y);
cs.showText(line);
@@ -326,225 +125,32 @@ class TextRedactionServiceMoreTest {
}
}
// ── private TJ / segment helpers via reflection ──────────────────────────────────────────────
@Nested
@DisplayName("private helpers via reflection")
class PrivateHelpers {
@Test
@DisplayName("createRedactedTJArray replaces the matched substring inside the array")
void createRedactedTjArray() throws Exception {
COSArray original = new COSArray();
original.add(new COSString("SECRET"));
original.add(new COSFloat(-40f));
original.add(new COSString(" tail"));
// Segment text is the concatenation "SECRET tail"; startPos 0.
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "TJ", "SECRET tail", 0, 11, helvetica(), FONT_SIZE);
List<TextRedactionService.MatchRange> matches =
List.of(new TextRedactionService.MatchRange(0, 6)); // "SECRET"
Method m =
TextRedactionService.class.getDeclaredMethod(
"createRedactedTJArray",
COSArray.class,
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
COSArray result = (COSArray) m.invoke(service, original, segment, matches);
StringBuilder sb = new StringBuilder();
for (COSBase el : result) {
if (el instanceof COSString cs) sb.append(cs.getString());
}
assertThat(sb.toString()).doesNotContain("SECRET");
assertThat(sb.toString()).contains("tail");
}
@Test
@DisplayName("applyRedactionsToSegmentText swaps the matched span for a placeholder")
void applyRedactionsToSegmentText() throws Exception {
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "Tj", "keepSECRETkeep", 0, 14, helvetica(), FONT_SIZE);
List<TextRedactionService.MatchRange> matches =
List.of(new TextRedactionService.MatchRange(4, 10)); // SECRET
Method m =
TextRedactionService.class.getDeclaredMethod(
"applyRedactionsToSegmentText",
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
String out = (String) m.invoke(service, segment, matches);
assertThat(out).doesNotContain("SECRET");
assertThat(out).startsWith("keep");
assertThat(out).endsWith("keep");
}
@Test
@DisplayName("calculateWidthAdjustment returns 0 for a null-font segment")
void widthAdjustmentNullFont() throws Exception {
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(0, "Tj", "abc", 0, 3, null, FONT_SIZE);
Method m =
TextRedactionService.class.getDeclaredMethod(
"calculateWidthAdjustment",
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
float adj =
(float)
m.invoke(
service,
segment,
List.of(new TextRedactionService.MatchRange(0, 3)));
assertThat(adj).isZero();
}
@Test
@DisplayName("calculateWidthAdjustment skips subset fonts (returns 0)")
void widthAdjustmentSubsetFontSkipped() throws Exception {
// A subset font name (6 uppercase letters + '+') trips the subset short-circuit.
PDFont subsetNamed = new PDType1Font(Standard14Fonts.FontName.HELVETICA);
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
0, "Tj", "ABCDEF", 0, 6, subsetNamed, FONT_SIZE);
// The real Helvetica name is not a subset, so this segment goes through the normal
// calculation; assert the call is at least exception-free and finite.
Method m =
TextRedactionService.class.getDeclaredMethod(
"calculateWidthAdjustment",
TextRedactionService.TextSegment.class,
List.class);
m.setAccessible(true);
float adj =
(float)
m.invoke(
service,
segment,
List.of(new TextRedactionService.MatchRange(0, 6)));
assertThat(Float.isFinite(adj)).isTrue();
}
@Test
@DisplayName("modifyTokenForRedaction with an out-of-range token index is a no-op")
void modifyTokenOutOfRange() throws Exception {
List<Object> tokens = new ArrayList<>();
tokens.add(new COSString("hello"));
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(
99, "Tj", "hello", 0, 5, helvetica(), FONT_SIZE);
Method m =
TextRedactionService.class.getDeclaredMethod(
"modifyTokenForRedaction",
List.class,
TextRedactionService.TextSegment.class,
String.class,
float.class,
List.class);
m.setAccessible(true);
m.invoke(service, tokens, segment, "", 0f, List.of());
// Token list is untouched because index 99 is out of bounds.
assertThat(tokens).hasSize(1);
assertThat(((COSString) tokens.get(0)).getString()).isEqualTo("hello");
}
@Test
@DisplayName("buildCompleteText concatenates the text of all segments in order")
void buildCompleteText() throws Exception {
List<TextRedactionService.TextSegment> segments =
List.of(
new TextRedactionService.TextSegment(
0, "Tj", "foo", 0, 3, helvetica(), FONT_SIZE),
new TextRedactionService.TextSegment(
1, "Tj", "bar", 3, 6, helvetica(), FONT_SIZE));
Method m =
TextRedactionService.class.getDeclaredMethod("buildCompleteText", List.class);
m.setAccessible(true);
assertThat(m.invoke(service, segments)).isEqualTo("foobar");
}
@Test
@DisplayName("extractTextFromToken returns text for the \" operator")
void extractTextFromQuoteOperator() throws Exception {
Method m =
TextRedactionService.class.getDeclaredMethod(
"extractTextFromToken", Object.class, String.class);
m.setAccessible(true);
// The " operator is not in the switch (Tj/'/TJ) -> default branch yields empty string.
assertThat(m.invoke(service, new COSString("x"), "\"")).isEqualTo("");
}
}
// ── createPlaceholderWithWidth additional branches ───────────────────────────────────────────
@Nested
@DisplayName("createPlaceholderWithWidth reliable-font path")
class PlaceholderWidthBranches {
@Test
@DisplayName("reliable font with positive width yields a bounded run of spaces")
void reliableFontBoundedSpaces() {
PDFont font = helvetica();
String original = "Secret";
float targetWidth;
try {
targetWidth = font.getStringWidth(original) / 1000f * FONT_SIZE;
} catch (IOException e) {
targetWidth = 30f;
}
String placeholder =
service.createPlaceholderWithWidth(original, targetWidth, font, FONT_SIZE);
assertThat(placeholder).isNotEmpty();
assertThat(placeholder.chars().allMatch(c -> c == ' ')).isTrue();
// spaceCount is capped at originalLength*2.
assertThat(placeholder.length()).isLessThanOrEqualTo(original.length() * 2);
}
@Test
@DisplayName("zero target width falls back to alternative placeholder logic")
void zeroTargetWidth() {
PDFont font = helvetica();
String placeholder = service.createPlaceholderWithWidth("word", 0f, font, FONT_SIZE);
// With a reliable, non-subset font and zero width, output is still all whitespace.
assertThat(placeholder.chars().allMatch(c -> c == ' ')).isTrue();
}
}
// ── inner data classes ───────────────────────────────────────────────────────────────────────
@Nested
@DisplayName("ModificationTask / GraphicsState data classes")
@DisplayName("TextSegment / MatchRange data classes")
class DataClasses {
@Test
@DisplayName("GraphicsState defaults are null font and zero size, mutators round-trip")
void graphicsStateRoundTrip() throws Exception {
Class<?> gsClass =
Class.forName(
"stirling.software.SPDF.controller.api.security.TextRedactionService$GraphicsState");
var ctor = gsClass.getDeclaredConstructor();
ctor.setAccessible(true);
Object gs = ctor.newInstance();
@DisplayName("TextSegment exposes its constructor values via accessors")
void textSegmentAccessors() {
PDFont font = helvetica();
TextRedactionService.TextSegment segment =
new TextRedactionService.TextSegment(3, "Tj", "hello", 10, 15, font, 12f);
Method getFont = gsClass.getDeclaredMethod("getFont");
Method getSize = gsClass.getDeclaredMethod("getFontSize");
getFont.setAccessible(true);
getSize.setAccessible(true);
assertThat(getFont.invoke(gs)).isNull();
assertThat((float) getSize.invoke(gs)).isZero();
assertThat(segment.getTokenIndex()).isEqualTo(3);
assertThat(segment.getOperatorName()).isEqualTo("Tj");
assertThat(segment.getText()).isEqualTo("hello");
assertThat(segment.getStartPos()).isEqualTo(10);
assertThat(segment.getEndPos()).isEqualTo(15);
assertThat(segment.getFont()).isSameAs(font);
assertThat(segment.getFontSize()).isEqualTo(12f);
}
Method setSize = gsClass.getDeclaredMethod("setFontSize", float.class);
setSize.setAccessible(true);
setSize.invoke(gs, 14f);
assertThat((float) getSize.invoke(gs)).isEqualTo(14f);
@Test
@DisplayName("MatchRange exposes start and end positions")
void matchRangeAccessors() {
TextRedactionService.MatchRange range = new TextRedactionService.MatchRange(4, 9);
assertThat(range.getStartPos()).isEqualTo(4);
assertThat(range.getEndPos()).isEqualTo(9);
}
}
}
app/core/src/test/java/stirling/software/SPDF/controller/api/security/TextRedactionServiceTest.java
+13 -303
View File
@@ -2,30 +2,21 @@ package stirling.software.SPDF.controller.api.security;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.io.InputStream;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.cos.COSArray;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdfparser.PDFStreamParser;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import org.junit.jupiter.api.DisplayName;
@@ -54,13 +45,23 @@ class TextRedactionServiceTest {
return new PDType1Font(Standard14Fonts.FontName.HELVETICA);
}
private PDFont helvetica(PDDocument doc) throws IOException {
try (InputStream is =
getClass().getResourceAsStream("/type3/library/fonts/dejavu/DejaVuSans.ttf")) {
if (is != null) {
return PDType0Font.load(doc, is);
}
}
return helvetica();
}
/** Single page, single Tj line per supplied text line, Helvetica 12. */
private PDDocument buildDoc(String... lines) throws IOException {
PDDocument doc = new PDDocument();
PDPage page = new PDPage(PDRectangle.LETTER);
doc.addPage(page);
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
cs.setFont(helvetica(), FONT_SIZE);
cs.setFont(helvetica(doc), FONT_SIZE);
for (int i = 0; i < lines.length; i++) {
cs.beginText();
cs.newLineAtOffset(LEFT_X, TOP_Y - i * 16f);
@@ -77,32 +78,6 @@ class TextRedactionServiceTest {
return doc;
}
// ── isTextShowingOperator ────────────────────────────────────────────────────────────────────
@Nested
@DisplayName("isTextShowingOperator")
class IsTextShowingOperator {
@Test
@DisplayName("recognises the four text-showing operators")
void recognisesTextShowingOperators() {
assertTrue(service.isTextShowingOperator("Tj"));
assertTrue(service.isTextShowingOperator("TJ"));
assertTrue(service.isTextShowingOperator("'"));
assertTrue(service.isTextShowingOperator("\""));
}
@Test
@DisplayName("rejects non text-showing operators and junk")
void rejectsOthers() {
assertFalse(service.isTextShowingOperator("BT"));
assertFalse(service.isTextShowingOperator("ET"));
assertFalse(service.isTextShowingOperator("Tf"));
assertFalse(service.isTextShowingOperator(""));
assertFalse(service.isTextShowingOperator("tj"));
}
}
// ── findTextToRedact ─────────────────────────────────────────────────────────────────────────
@Nested
@@ -253,192 +228,6 @@ class TextRedactionServiceTest {
}
}
// ── detectCustomEncodingFonts ────────────────────────────────────────────────────────────────
@Nested
@DisplayName("detectCustomEncodingFonts")
class DetectCustomEncodingFonts {
@Test
@DisplayName("standard Helvetica document is not flagged as custom-encoded")
void standardFontNotFlagged() throws IOException {
try (PDDocument doc = buildDoc("plain helvetica text")) {
assertFalse(service.detectCustomEncodingFonts(doc));
}
}
@Test
@DisplayName("document with no content / no fonts is not flagged")
void emptyDocumentNotFlagged() throws IOException {
try (PDDocument doc = buildEmptyDoc()) {
assertFalse(service.detectCustomEncodingFonts(doc));
}
}
}
// ── createPlaceholderWithFont ────────────────────────────────────────────────────────────────
@Nested
@DisplayName("createPlaceholderWithFont")
class CreatePlaceholderWithFont {
@Test
@DisplayName("returns the input unchanged for null")
void nullReturnsNull() {
assertNull(service.createPlaceholderWithFont(null, helvetica()));
}
@Test
@DisplayName("returns the input unchanged for empty string")
void emptyReturnsEmpty() {
assertEquals("", service.createPlaceholderWithFont("", helvetica()));
}
@Test
@DisplayName("non-subset font yields spaces matching the original length")
void nonSubsetFontYieldsMatchingSpaces() {
String placeholder = service.createPlaceholderWithFont("hidden", helvetica());
assertEquals(" ".repeat("hidden".length()), placeholder);
}
@Test
@DisplayName("null font is treated as non-subset and yields spaces")
void nullFontYieldsSpaces() {
String placeholder = service.createPlaceholderWithFont("abc", null);
assertEquals(" ", placeholder);
}
}
// ── createPlaceholderWithWidth ───────────────────────────────────────────────────────────────
@Nested
@DisplayName("createPlaceholderWithWidth")
class CreatePlaceholderWithWidth {
@Test
@DisplayName("returns the input unchanged for null")
void nullReturnsNull() {
assertNull(service.createPlaceholderWithWidth(null, 10f, helvetica(), FONT_SIZE));
}
@Test
@DisplayName("returns the input unchanged for empty string")
void emptyReturnsEmpty() {
assertEquals("", service.createPlaceholderWithWidth("", 10f, helvetica(), FONT_SIZE));
}
@Test
@DisplayName("null font falls back to one space per original character")
void nullFontFallsBackToSpaces() {
String placeholder = service.createPlaceholderWithWidth("word", 50f, null, FONT_SIZE);
assertEquals(" ".repeat("word".length()), placeholder);
}
@Test
@DisplayName("non-positive font size falls back to one space per original character")
void nonPositiveFontSizeFallsBackToSpaces() {
String placeholder = service.createPlaceholderWithWidth("word", 50f, helvetica(), 0f);
assertEquals(" ".repeat("word".length()), placeholder);
}
@Test
@DisplayName("standard font produces a non-null all-whitespace placeholder")
void standardFontProducesWhitespacePlaceholder() {
PDFont font = helvetica();
float fontSize = FONT_SIZE;
String original = "Secret";
// Compute a realistic target width the way the service does (text-space / 1000 * size).
float targetWidth;
try {
targetWidth = font.getStringWidth(original) / 1000f * fontSize;
} catch (IOException e) {
targetWidth = 30f;
}
String placeholder =
service.createPlaceholderWithWidth(original, targetWidth, font, fontSize);
assertNotNull(placeholder);
assertFalse(placeholder.isEmpty(), "Helvetica supports spaces, so non-empty expected");
assertTrue(
placeholder.chars().allMatch(c -> c == ' '),
"placeholder should be composed only of spaces");
}
}
// ── createTokensWithoutTargetText / writeFilteredContentStream
// ────────────────────────────────
@Nested
@DisplayName("createTokensWithoutTargetText")
class CreateTokensWithoutTargetText {
@Test
@DisplayName(
"returns a non-empty token list and preserves token count when nothing matches")
void noMatchPreservesTokens() throws IOException {
try (PDDocument doc = buildDoc("nothing to hide")) {
PDPage page = doc.getPage(0);
List<Object> originalTokens = parseTokens(page);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("ABSENT"), false, false);
assertNotNull(tokens);
assertEquals(
originalTokens.size(),
tokens.size(),
"token count should be unchanged when nothing matched");
}
}
@Test
@DisplayName("filtered tokens can be written back and the page re-parses cleanly")
void filteredTokensRoundTrip() throws IOException {
try (PDDocument doc = buildDoc("redact SECRET token roundtrip")) {
PDPage page = doc.getPage(0);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Set.of("SECRET"), false, false);
assertNotNull(tokens);
service.writeFilteredContentStream(doc, page, tokens);
// The page must still hold valid content (at least one operator token).
List<Object> reparsed = parseTokens(page);
boolean hasOperator = reparsed.stream().anyMatch(t -> t instanceof Operator);
assertTrue(hasOperator, "rewritten content stream must contain operators");
}
}
@Test
@DisplayName("empty target-word set leaves tokens untouched")
void emptyTargetSetLeavesTokens() throws IOException {
try (PDDocument doc = buildDoc("some content")) {
PDPage page = doc.getPage(0);
List<Object> originalTokens = parseTokens(page);
List<Object> tokens =
service.createTokensWithoutTargetText(
doc, page, Collections.emptySet(), false, false);
assertEquals(originalTokens.size(), tokens.size());
}
}
private List<Object> parseTokens(PDPage page) throws IOException {
PDFStreamParser parser = new PDFStreamParser(page);
List<Object> tokens = new ArrayList<>();
Object token;
while ((token = parser.parseNextToken()) != null) {
tokens.add(token);
}
return tokens;
}
}
// ── inner data classes ───────────────────────────────────────────────────────────────────────
@Nested
@@ -484,83 +273,4 @@ class TextRedactionServiceTest {
assertFalse(a.equals(b));
}
}
// ── private logic exercised via reflection ───────────────────────────────────────────────────
@Nested
@DisplayName("findAllMatches / buildCompleteText (private logic via reflection)")
class PrivateLogic {
@Test
@DisplayName("findAllMatches returns sorted, non-overlapping match ranges for two terms")
@SuppressWarnings("unchecked")
void findAllMatchesSorted() throws Exception {
String complete = "alpha beta gamma beta";
Set<String> terms = new LinkedHashSet<>(List.of("beta", "alpha"));
Method m =
TextRedactionService.class.getDeclaredMethod(
"findAllMatches",
String.class,
Set.class,
boolean.class,
boolean.class);
m.setAccessible(true);
List<TextRedactionService.MatchRange> matches =
(List<TextRedactionService.MatchRange>)
m.invoke(service, complete, terms, false, false);
assertNotNull(matches);
assertFalse(matches.isEmpty());
// Results are sorted by start position.
for (int i = 1; i < matches.size(); i++) {
assertTrue(
matches.get(i - 1).getStartPos() <= matches.get(i).getStartPos(),
"matches must be sorted ascending by start position");
}
// "alpha" at 0, "beta" at 6 and 17 -> three matches total.
assertEquals(3, matches.size());
assertEquals(0, matches.get(0).getStartPos());
}
@Test
@DisplayName("findAllMatches returns nothing when no term occurs")
@SuppressWarnings("unchecked")
void findAllMatchesEmptyWhenAbsent() throws Exception {
Method m =
TextRedactionService.class.getDeclaredMethod(
"findAllMatches",
String.class,
Set.class,
boolean.class,
boolean.class);
m.setAccessible(true);
List<TextRedactionService.MatchRange> matches =
(List<TextRedactionService.MatchRange>)
m.invoke(service, "no terms here", Set.of("XYZ"), false, false);
assertTrue(matches.isEmpty());
}
@Test
@DisplayName("extractTextFromToken pulls text from Tj COSString and TJ COSArray")
void extractTextFromToken() throws Exception {
Method m =
TextRedactionService.class.getDeclaredMethod(
"extractTextFromToken", Object.class, String.class);
m.setAccessible(true);
assertEquals("hi", m.invoke(service, new COSString("hi"), "Tj"));
assertEquals("hi", m.invoke(service, new COSString("hi"), "'"));
COSArray tjArray = new COSArray();
tjArray.add(new COSString("foo"));
tjArray.add(new COSString("bar"));
assertEquals("foobar", m.invoke(service, tjArray, "TJ"));
// Unknown operator yields empty string.
assertEquals("", m.invoke(service, new COSString("x"), "Td"));
// Wrong token type for the operator yields empty string.
assertEquals("", m.invoke(service, new COSArray(), "Tj"));
}
}
}
app/core/src/test/java/stirling/software/SPDF/util/ExtractScriptConcurrencyTest.java
+78
View File
@@ -0,0 +1,78 @@
package stirling.software.SPDF.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import stirling.software.common.util.GeneralUtils;
/**
* Lives in core because the python resources it extracts ship in this module.
*
* <p>Callers hand the returned path straight to python3, so the file has to stay readable while
* other requests are extracting it. Re-writing it per call used to break that wherever rename is
* not atomic, such as a 9p or NFS bind mount.
*/
class ExtractScriptConcurrencyTest {
@Test
@DisplayName("concurrent callers always get a readable script")
void concurrentCallersSeeAReadableScript() throws Exception {
int threads = 16;
CountDownLatch start = new CountDownLatch(1);
CountDownLatch done = new CountDownLatch(threads);
AtomicInteger unreadable = new AtomicInteger();
AtomicInteger errors = new AtomicInteger();
for (int i = 0; i < threads; i++) {
Thread.ofVirtual()
.start(
() -> {
try {
start.await();
for (int n = 0; n < 25; n++) {
Path script = GeneralUtils.extractScript("png_to_webp.py");
if (!Files.isReadable(script)) {
unreadable.incrementAndGet();
}
}
} catch (Exception e) {
errors.incrementAndGet();
} finally {
done.countDown();
}
});
}
start.countDown();
assertTrue(done.await(60, TimeUnit.SECONDS), "extractScript threads did not finish");
assertEquals(0, errors.get(), "extractScript threw under concurrency");
assertEquals(0, unreadable.get(), "script was missing while another caller held it");
}
@Test
@DisplayName("repeated calls return the same path without rewriting the file")
void repeatedCallsAreStable() throws Exception {
Path first = GeneralUtils.extractScript("png_to_webp.py");
long modified = Files.getLastModifiedTime(first).toMillis();
long size = Files.size(first);
// Any rewrite after this pause lands on a later timestamp, so mtime is a
// reliable signal rather than a same-millisecond coin flip.
Thread.sleep(50);
for (int i = 0; i < 5; i++) {
assertEquals(first, GeneralUtils.extractScript("png_to_webp.py"));
}
assertEquals(modified, Files.getLastModifiedTime(first).toMillis(), "script was rewritten");
assertEquals(size, Files.size(first));
}
}
app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java
+50
View File
@@ -5,9 +5,11 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.*;
import java.util.Map;
import java.util.function.Predicate;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
@@ -442,4 +444,52 @@ class JobControllerTest {
assertEquals(HttpStatus.FORBIDDEN, response.getStatusCode());
verify(fileStorage, never()).getFileSize(eq(fileId));
}
@Test
void testCleanupFinishedJobs_ReportsWhatWasReleased() {
when(taskManager.cleanupFinishedJobsNow(any()))
.thenReturn(new TaskManager.CleanupSummary(2, 5, 1));
ResponseEntity<?> response = controller.cleanupFinishedJobs();
assertEquals(HttpStatus.OK, response.getStatusCode());
@SuppressWarnings("unchecked")
Map<String, Object> body = (Map<String, Object>) response.getBody();
assertEquals(2, body.get("jobsRemoved"));
assertEquals(5, body.get("filesDeleted"));
assertEquals(1, body.get("jobsRetained"));
}
@Test
void testCleanupFinishedJobs_OnlySweepsJobsTheCallerOwns() {
ReflectionTestUtils.setField(controller, "jobOwnershipService", jobOwnershipService);
when(jobOwnershipService.validateJobAccess("me:job")).thenReturn(true);
when(jobOwnershipService.validateJobAccess("someone-else:job"))
.thenThrow(new SecurityException("not yours"));
when(taskManager.cleanupFinishedJobsNow(any()))
.thenReturn(new TaskManager.CleanupSummary(1, 1, 1));
controller.cleanupFinishedJobs();
@SuppressWarnings("unchecked")
ArgumentCaptor<Predicate<String>> filter = ArgumentCaptor.forClass(Predicate.class);
verify(taskManager).cleanupFinishedJobsNow(filter.capture());
assertTrue(filter.getValue().test("me:job"));
assertFalse(
filter.getValue().test("someone-else:job"),
"A job the caller cannot access must be left in place");
}
@Test
void testCleanupFinishedJobs_SweepsEverythingWhenSecurityIsDisabled() {
when(taskManager.cleanupFinishedJobsNow(any()))
.thenReturn(new TaskManager.CleanupSummary(3, 3, 0));
controller.cleanupFinishedJobs();
@SuppressWarnings("unchecked")
ArgumentCaptor<Predicate<String>> filter = ArgumentCaptor.forClass(Predicate.class);
verify(taskManager).cleanupFinishedJobsNow(filter.capture());
assertTrue(filter.getValue().test("any-job-id"));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/controller/api/AdminJobController.java
+26 -13
View File
@@ -7,6 +7,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import io.swagger.v3.oas.annotations.Operation;
@@ -66,28 +67,40 @@ public class AdminJobController {
}
/**
* Manually trigger cleanup of old jobs (admin only)
* Manually trigger cleanup of old jobs (admin only). Covers every user's jobs, unlike the
* self-service {@code POST /api/v1/general/jobs/cleanup}, which only releases the caller's own.
*
* @return A response indicating how many jobs were cleaned up
* @param force Ignore the retention window and release every finished job now, rather than only
* those already past it
* @return A response indicating how many jobs and files were cleaned up
*/
@PostMapping("/job/cleanup")
@Operation(summary = "Cleanup old jobs")
@Operation(
summary = "Cleanup old jobs",
description =
"Runs the job retention sweep now across all users. With force=true the"
+ " retention window is ignored and every finished job is released"
+ " immediately.")
@PreAuthorize("hasRole('ADMIN')")
public ResponseEntity<?> cleanupOldJobs() {
int beforeCount = taskManager.getJobStats().getTotalJobs();
taskManager.cleanupOldJobs();
int afterCount = taskManager.getJobStats().getTotalJobs();
int removedCount = beforeCount - afterCount;
public ResponseEntity<?> cleanupOldJobs(
@RequestParam(name = "force", defaultValue = "false") boolean force) {
TaskManager.CleanupSummary summary =
force
? taskManager.cleanupFinishedJobsNow(jobId -> true)
: taskManager.cleanupOldJobs();
log.info(
"Admin triggered job cleanup: removed {} jobs, {} remaining",
removedCount,
afterCount);
"Admin triggered job cleanup (force={}): removed {} jobs and {} files, {} remaining",
force,
summary.jobsRemoved(),
summary.filesDeleted(),
summary.jobsRetained());
return ResponseEntity.ok(
Map.of(
"message", "Cleanup complete",
"removedJobs", removedCount,
"remainingJobs", afterCount));
"removedJobs", summary.jobsRemoved(),
"filesDeleted", summary.filesDeleted(),
"remainingJobs", summary.jobsRetained()));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/failure/EditorFailureReport.java
+47
View File
@@ -0,0 +1,47 @@
package stirling.software.proprietary.failure;
import java.util.List;
/**
* A failure a user hit in the editor, reported by their own client. The editor calls tools directly
* rather than through the policy engine, so nothing server-side sees these unless the client says
* so.
*
* <p>Note what the client cannot supply: no team, no actor, no document name. The first two come
* from the authenticated session, and the third is never stored.
*
* @param operation the tool that failed, e.g. {@code remove-password}
* @param errorCode the code from the tool's Problem Details response, or null when there was none
* @param fileIds opaque client-side ids of the documents involved; empty when none is attributable
* @param detail the message the user saw
*/
public record EditorFailureReport(
String operation, String errorCode, List<String> fileIds, String detail) {
/**
* Cap on the files one report may name. Each one becomes a permanent incident and this endpoint
* is open to any authenticated user, so without a bound a single call can flood a leader's
* queue. 200 is several times the largest batch an editor session plausibly fails on, and the
* most the review queue shows in one page, so a real report never meets it.
*
* <p>An oversized report is refused whole rather than trimmed: see {@link
* FileRunEventController#report}.
*/
static final int MAX_FILE_IDS = 200;
public EditorFailureReport {
fileIds = fileIds == null ? List.of() : List.copyOf(fileIds);
}
boolean hasOperation() {
return operation != null && !operation.isBlank();
}
/**
* Counted before the blank ids are dropped, because this bounds the request rather than the
* rows it would produce.
*/
boolean namesTooManyFiles() {
return fileIds.size() > MAX_FILE_IDS;
}
}
app/proprietary/src/main/java/stirling/software/proprietary/failure/FailureKind.java
+4 -1
View File
@@ -23,6 +23,10 @@ import lombok.Getter;
* <p>Actions are declared here but implemented in {@link FailureAction} beans resolved by id, so a
* new kind ships as a registry entry plus copy. Two members today: {@link #UNKNOWN} gives every
* failed run a record, and kinds get promoted out of it as production shows what occurs.
*
* <p>A kind offers an acknowledgement only where there is something to acknowledge <em>doing</em>.
* With nothing to fix, "seen it" and "clear it" are the same decision, so the row offers only the
* one that clears it.
*/
@Getter
public enum FailureKind {
@@ -43,7 +47,6 @@ public enum FailureKind {
FailureScope.RUN,
noErrorCodes(),
fallback("This run failed for a reason Stirling does not yet recognise."),
offer(ACKNOWLEDGE),
offer(DISMISS));
private static final String KEY_PREFIX = "portal.failures.kind.";
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEvent.java
+2
View File
@@ -20,6 +20,7 @@ public record FileRunEvent(
FailureOrigin origin,
String policyId,
String runId,
String sourceId,
String fileId,
String detail,
String dedupKey,
@@ -46,6 +47,7 @@ public record FileRunEvent(
entity.getOrigin(),
entity.getPolicyId(),
entity.getRunId(),
entity.getSourceId(),
entity.getFileId(),
entity.getDetail(),
entity.getDedupKey(),
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventController.java
+61
View File
@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@@ -91,6 +92,50 @@ public class FileRunEventController {
}
}
@PostMapping("/reports")
@Operation(
summary = "Report a failure hit in the editor",
description =
"For failures the server never sees, because the editor calls tools directly."
+ " Open to any authenticated user, unlike the read and triage endpoints:"
+ " whoever's work failed can say so, and a leader reviews it. Rejected"
+ " with 400 if it names more files than one report may carry.")
public ResponseEntity<Void> report(@RequestBody EditorFailureReport report) {
if (report == null || !report.hasOperation()) {
throw new ResponseStatusException(
HttpStatus.BAD_REQUEST, "operation is required to report a failure");
}
// Refused whole rather than trimmed, and refused before the first write, so an oversized
// report leaves no rows at all. Trimming would hand a reviewer part of a set with nothing
// saying the rest existed, which is what the cap inside the service used to do. The limit
// is stated in the message because the editor reports in the background: a client author
// reading a log is the only person who will ever see this.
if (report.namesTooManyFiles()) {
throw new ResponseStatusException(
HttpStatus.BAD_REQUEST,
"a report may name at most "
+ EditorFailureReport.MAX_FILE_IDS
+ " files, and this one named "
+ report.fileIds().size());
}
service.report(report);
// No body: the editor reports and moves on, and has nothing to do with the row.
return ResponseEntity.noContent().build();
}
@PostMapping("/removed-files")
@Operation(
summary = "Close the incidents about files deleted from the editor",
description =
"Deleting the document leaves nothing to act on, so its incidents drop out of"
+ " the queue while the rows stay for audit. Open to any authenticated"
+ " user, and applies only to their own editor rows.")
public ResponseEntity<Void> filesRemoved(@RequestBody(required = false) RemovedFiles request) {
service.forgetFiles(request == null ? List.of() : request.safeFileIds());
// No body: the editor is telling the server, not asking it anything.
return ResponseEntity.noContent().build();
}
@GetMapping("/kinds")
@Operation(
summary = "List known failure kinds",
@@ -136,6 +181,22 @@ public class FileRunEventController {
/** Wrapped rather than a bare array so pagination can be added without breaking clients. */
public record FileRunEventsResponse(List<FileRunEventView> events) {}
/**
* Files gone from the caller's editor. Opaque ids only, as everywhere else on this API.
*
* <p>Deliberately uncapped where a report is capped, because this creates nothing: it closes
* rows the caller already owns, so however long the list is, it can only ever touch incidents
* that already exist. Refusing an oversized one would also be the harmful direction here, since
* the editor says this once and never retries: those incidents would sit in the queue asking
* for attention about files that no longer exist.
*/
public record RemovedFiles(List<String> fileIds) {
List<String> safeFileIds() {
return fileIds == null ? List.of() : fileIds;
}
}
/** Inputs an action declared it needs. Empty for both actions that exist today. */
public record ActionRequest(Map<String, String> inputs) {
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventEntity.java
+4
View File
@@ -84,6 +84,10 @@ public class FileRunEventEntity implements Serializable {
@Column(name = "run_id")
private String runId;
/** Which folder, bucket or webhook fed the run. Null when a user supplied the file. */
@Column(name = "source_id")
private String sourceId;
@Column(name = "file_id")
private String fileId;
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventRepository.java
+36 -9
View File
@@ -17,19 +17,21 @@ import org.springframework.transaction.annotation.Transactional;
public interface FileRunEventRepository extends JpaRepository<FileRunEventEntity, String> {
/**
* This team's events, newest first, scoped in the query rather than loaded and filtered. A
* {@code null} teamId matches the rows with no team (login disabled), mirroring {@link
* stirling.software.proprietary.policy.source.SourceRepository#findByTeam}, since a plain
* {@code = null} would return nothing.
* As {@link #findByTeamAndStatus} but for a set of statuses, e.g. the open ones. The kind
* filter is in the query, before the limit: filtering an already-limited page could return
* nothing while matching rows exist.
*/
@Query(
"select e from FileRunEventEntity e where ((:teamId is null and e.teamId is null) or"
+ " e.teamId = :teamId) and (:kindId is null or e.kindId = :kindId)"
+ " order by e.lastSeenAt desc")
List<FileRunEventEntity> findByTeam(
@Param("teamId") Long teamId, @Param("kindId") String kindId, Pageable pageable);
+ " e.teamId = :teamId) and e.status in :statuses"
+ " and (:kindId is null or e.kindId = :kindId) order by e.lastSeenAt desc")
List<FileRunEventEntity> findByTeamAndStatusIn(
@Param("teamId") Long teamId,
@Param("statuses") List<FileRunEventStatus> statuses,
@Param("kindId") String kindId,
Pageable pageable);
/** As {@link #findByTeam} but restricted to one status, for the review surface's filters. */
/** As {@link #findByTeamAndStatusIn} but for exactly one status, for the surface's filters. */
@Query(
"select e from FileRunEventEntity e where ((:teamId is null and e.teamId is null) or"
+ " e.teamId = :teamId) and e.status = :status"
@@ -85,6 +87,31 @@ public interface FileRunEventRepository extends JpaRepository<FileRunEventEntity
@Param("now") Instant now,
@Param("allowedFrom") Collection<FileRunEventStatus> allowedFrom);
/**
* Close the incidents about documents their owner deleted from the editor: the queue is what
* needs attention, and a document that no longer exists needs none.
*
* <p>Restricted to that owner's own editor rows. File ids are minted by the client, so scoping
* on team alone would let one caller close a colleague's incidents by naming ids. Processor
* rows are excluded outright: nothing was deleted from an editor there.
*/
@Modifying(clearAutomatically = true)
@Transactional
@Query(
"update FileRunEventEntity e set e.status ="
+ " stirling.software.proprietary.failure.FileRunEventStatus.FILE_REMOVED,"
+ " e.statusActor = :actor, e.statusAt = :now where e.origin ="
+ " stirling.software.proprietary.failure.FailureOrigin.TOOL and ((:teamId is"
+ " null and e.teamId is null) or e.teamId = :teamId) and ((:actor is null and"
+ " e.actor is null) or e.actor = :actor) and e.fileId in :fileIds and e.status in"
+ " :allowedFrom")
int markFilesRemoved(
@Param("teamId") Long teamId,
@Param("actor") String actor,
@Param("fileIds") Collection<String> fileIds,
@Param("now") Instant now,
@Param("allowedFrom") Collection<FileRunEventStatus> allowedFrom);
/**
* The most recent row for this exact failure, so the rollup can increment an existing incident
* instead of opening a new one. Team-scoped, so the same failure in two teams stays two rows.
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventService.java
+61
View File
@@ -31,6 +31,67 @@ public class FileRunEventService {
private final UserServiceInterface userService;
private final ApplicationProperties applicationProperties;
/**
* Record a failure a user hit in the editor. One incident per named file, so each document
* stays separately actionable; one unattributed incident when the report names none.
*
* <p>The team and actor come from the session rather than the report, and the kind is
* classified from the reported code, falling back to {@link FailureKind#UNKNOWN} for a code no
* kind claims.
*/
public List<FileRunEvent> report(EditorFailureReport report) {
FailureKind kind = FailureKind.byErrorCode(report.errorCode()).orElse(FailureKind.UNKNOWN);
Long teamId = scope().teamId();
String actor = currentActor();
String detail = detailFor(report);
List<String> fileIds =
report.fileIds().stream().filter(id -> id != null && !id.isBlank()).toList();
if (fileIds.isEmpty()) {
return List.of(recordReported(kind, teamId, actor, null, detail));
}
// Every named file gets its row. An earlier cap here silently dropped the rest, which lost
// failures a reviewer needed and was inconsistent with the processor path, where a sweep
// records one row per failing file with no limit at all. How many files one report may name
// is bounded at the boundary instead (see EditorFailureReport#MAX_FILE_IDS), where an
// oversized report can be refused whole before anything is written.
return fileIds.stream()
.map(fileId -> recordReported(kind, teamId, actor, fileId, detail))
.toList();
}
private FileRunEvent recordReported(
FailureKind kind, Long teamId, String actor, String fileId, String detail) {
return store.record(RecordFailure.forEditor(kind, teamId, actor, fileId, detail));
}
/**
* The operation is the context a reviewer needs, since an editor failure has no policy or run.
*/
private String detailFor(EditorFailureReport report) {
String message = report.detail() == null ? "" : report.detail();
return message.isBlank() ? report.operation() : report.operation() + ": " + message;
}
/**
* Close the incidents about documents the caller has deleted from their editor. The queue means
* "needs attention", and a document that no longer exists needs none; the rows stay for audit.
*
* <p>Best-effort by nature: this only arrives if the browser that owns the file says so, and a
* cleared cache or another device never will. Rows left open that way are retention's problem,
* not this method's.
*
* @return how many incidents were closed
*/
public int forgetFiles(List<String> fileIds) {
TeamScope scope = scope();
if (!scope.permitted()) {
return 0;
}
List<String> named = fileIds.stream().filter(id -> id != null && !id.isBlank()).toList();
return store.markFilesRemoved(scope.teamId(), currentActor(), named);
}
/** The calling user's events, newest first. Empty when their team cannot be resolved. */
public List<FileRunEvent> list(FileRunEventStatus status, String kindId, int limit) {
TeamScope scope = scope();
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventStatus.java
+8 -1
View File
@@ -11,7 +11,14 @@ public enum FileRunEventStatus {
NEW(false),
ACKNOWLEDGED(false),
DISMISSED(true),
RESOLVED(true);
RESOLVED(true),
/**
* The document this incident was about was deleted from its owner's editor, so there is nothing
* left to act on. Distinct from {@code DISMISSED}, which is a reviewer's decision, and from
* {@code RESOLVED}, which reopens on recurrence: this one cannot recur, the file is gone.
*/
FILE_REMOVED(true);
/** The statuses a review queue shows by default: everything still needing a decision. */
private static final List<FileRunEventStatus> OPEN =
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventStore.java
+27 -4
View File
@@ -98,6 +98,7 @@ public class FileRunEventStore {
entity.setOrigin(command.origin());
entity.setPolicyId(command.policyId());
entity.setRunId(command.runId());
entity.setSourceId(command.sourceId());
entity.setFileId(command.fileId());
entity.setDetail(command.detail());
entity.setDedupKey(dedupKey);
@@ -112,9 +113,14 @@ public class FileRunEventStore {
}
/**
* A page of incidents, newest first, optionally narrowed to one status and one kind. The kind
* filter lives in the query, before the limit: filtering a already-limited page could return
* nothing while matching rows exist.
* A page of incidents, newest first, optionally narrowed to one status and one kind.
*
* <p>With no status asked for this is the <em>open</em> queue rather than every row ever
* recorded: a dismissed failure has been dealt with, and leaving it in the default view means
* the list can never be cleared. Ask for a status to see closed rows.
*
* <p>Both filters live in the query, before the limit: filtering an already-limited page could
* return nothing while matching rows exist.
*/
@Transactional(readOnly = true)
public List<FileRunEvent> list(
@@ -122,7 +128,8 @@ public class FileRunEventStore {
Pageable page = PageRequest.of(0, Math.max(1, limit));
List<FileRunEventEntity> rows =
status == null
? repository.findByTeam(teamId, kindId, page)
? repository.findByTeamAndStatusIn(
teamId, FileRunEventStatus.open(), kindId, page)
: repository.findByTeamAndStatus(teamId, status, kindId, page);
return rows.stream().map(FileRunEvent::of).toList();
}
@@ -184,6 +191,22 @@ public class FileRunEventStore {
.orElseThrow(() -> refusalFor(id, teamId));
}
/**
* Close this owner's open incidents about {@code fileIds}, because the documents are gone. The
* rows stay for audit; they just leave the queue. Only open rows move, so a reviewer's dismiss
* keeps its meaning and its actor.
*
* @return how many incidents were closed
*/
@Transactional
public int markFilesRemoved(Long teamId, String actor, Collection<String> fileIds) {
if (fileIds.isEmpty()) {
return 0;
}
return repository.markFilesRemoved(
teamId, actor, fileIds, Instant.now(), FileRunEventStatus.open());
}
/**
* Why the guarded UPDATE refused, worked out only once it has. Missing and closed are told
* apart after the fact rather than before, so the answer describes the row the UPDATE saw.
app/proprietary/src/main/java/stirling/software/proprietary/failure/FileRunEventView.java
+2
View File
@@ -22,6 +22,7 @@ public record FileRunEventView(
String detail,
String policyId,
String runId,
String sourceId,
String fileId,
String actor,
int occurrences,
@@ -48,6 +49,7 @@ public record FileRunEventView(
event.detail(),
event.policyId(),
event.runId(),
event.sourceId(),
event.fileId(),
event.actor(),
event.occurrences(),
app/proprietary/src/main/java/stirling/software/proprietary/failure/PolicyFailureRecorder.java
+21 -6
View File
@@ -30,11 +30,12 @@ public class PolicyFailureRecorder {
public void recordRunFailure(
String runId,
String policyId,
String actor,
String sourceId,
String fileIdentity,
String actor,
String detail,
Throwable cause) {
record(classifier.classify(cause), runId, policyId, actor, fileIdentity, detail);
record(classifier.classify(cause), runId, policyId, sourceId, fileIdentity, actor, detail);
}
/**
@@ -44,21 +45,35 @@ public class PolicyFailureRecorder {
* pick the wrong one.
*/
public void recordRunFailureAs(
FailureKind kind, String runId, String policyId, String actor, String detail) {
record(kind, runId, policyId, actor, null, detail);
FailureKind kind,
String runId,
String policyId,
String sourceId,
String actor,
String detail) {
// No document reference: a run rejected at admission never got as far as one.
record(kind, runId, policyId, sourceId, null, actor, detail);
}
private void record(
FailureKind kind,
String runId,
String policyId,
String actor,
String sourceId,
String fileIdentity,
String actor,
String detail) {
try {
store.record(
RecordFailure.forRun(
kind, teamFor(policyId), actor, policyId, runId, fileIdentity, detail));
kind,
teamFor(policyId),
actor,
policyId,
runId,
sourceId,
fileIdentity,
detail));
} catch (RuntimeException e) {
// Deliberately swallowed: see the class comment.
log.warn("Could not record failure event for run {} (kind {})", runId, kind.getId(), e);
app/proprietary/src/main/java/stirling/software/proprietary/failure/RecordFailure.java
+40 -10
View File
@@ -38,17 +38,40 @@ public record RecordFailure(
detail = truncate(detail);
}
/** A processor-side failure with no file or source context, e.g. a run that failed outright. */
/**
* A processor-side run failure. {@code sourceId} says which folder, bucket or webhook fed the
* run, and is the only attribution an unattended failure has: there is no user to name. {@code
* fileId} is the source's opaque reference to the document, already hashed upstream.
*/
public static RecordFailure forRun(
FailureKind kind,
Long teamId,
String actor,
String policyId,
String runId,
String sourceId,
String fileId,
String detail) {
return new RecordFailure(
kind, FailureOrigin.POLICY, teamId, actor, policyId, runId, null, fileId, detail);
kind,
FailureOrigin.POLICY,
teamId,
actor,
policyId,
runId,
sourceId,
fileId,
detail);
}
/**
* A failure a user hit in their own editor. There is no policy, run or source: the user is the
* attribution, and {@code fileId} may be null when the report named no file.
*/
public static RecordFailure forEditor(
FailureKind kind, Long teamId, String actor, String fileId, String detail) {
return new RecordFailure(
kind, FailureOrigin.TOOL, teamId, actor, null, null, null, fileId, detail);
}
/**
@@ -56,14 +79,21 @@ public record RecordFailure(
* reference are the same incident; see {@link #dedupKey()}.
*/
public String scopeRef() {
return switch (kind.getScope()) {
case FILE -> nullToEmpty(policyId) + "|" + fileOrRun();
case RUN -> nullToEmpty(runId);
case POLICY -> nullToEmpty(policyId);
case SOURCE -> nullToEmpty(sourceId);
// One server-wide condition is one incident regardless of which run tripped over it.
case SERVER -> "";
};
String about =
switch (kind.getScope()) {
case FILE -> nullToEmpty(policyId) + "|" + fileOrRun();
// An editor report has no run, so a RUN-scoped kind would otherwise put every
// such failure in a team into one incident: fall back to the document.
case RUN -> isBlank(runId) ? fileOrRun() : nullToEmpty(runId);
case POLICY -> nullToEmpty(policyId);
case SOURCE -> nullToEmpty(sourceId);
// One server-wide condition is one incident regardless of which run hit it.
case SERVER -> "";
};
// An editor failure belongs to the person who hit it, so two colleagues hitting the same
// thing are two incidents. Folding them would credit one actor for both and offer the
// wrong person the row. Unattended runs have no such owner and are unaffected.
return origin == FailureOrigin.TOOL ? nullToEmpty(actor) + "|" + about : about;
}
/**
app/proprietary/src/main/java/stirling/software/proprietary/model/Team.java
+34 -2
View File
@@ -2,13 +2,19 @@ package stirling.software.proprietary.model;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import org.hibernate.proxy.HibernateProxy;
import com.fasterxml.jackson.annotation.JsonIgnore;
import jakarta.persistence.*;
import lombok.*;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import stirling.software.proprietary.security.model.User;
@@ -18,7 +24,6 @@ import stirling.software.proprietary.security.model.User;
@NoArgsConstructor
@Getter
@Setter
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@ToString(onlyExplicitlyIncluded = true)
public class Team implements Serializable {
@@ -47,4 +52,31 @@ public class Team implements Serializable {
users.remove(user);
user.setTeam(null);
}
@Override
public final boolean equals(Object o) {
if (this == o) return true;
if (o == null) return false;
Class<?> oEffectiveClass =
o instanceof HibernateProxy
? ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass()
: o.getClass();
Class<?> thisEffectiveClass =
this instanceof HibernateProxy
? ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass()
: this.getClass();
if (thisEffectiveClass != oEffectiveClass) return false;
Team team = (Team) o;
return getId() != null && Objects.equals(getId(), team.getId());
}
@Override
public final int hashCode() {
return this instanceof HibernateProxy
? ((HibernateProxy) this)
.getHibernateLazyInitializer()
.getPersistentClass()
.hashCode()
: getClass().hashCode();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/model/security/PersistentAuditEvent.java
+38 -4
View File
@@ -1,6 +1,9 @@
package stirling.software.proprietary.model.security;
import java.time.Instant;
import java.util.Objects;
import org.hibernate.proxy.HibernateProxy;
import jakarta.persistence.*;
@@ -28,7 +31,9 @@ import lombok.*;
name = "idx_audit_source_timestamp_principal",
columnList = "source,timestamp,principal")
})
@Data
@Getter
@Setter
@ToString(onlyExplicitlyIncluded = true)
@Builder
@NoArgsConstructor
@AllArgsConstructor
@@ -36,14 +41,43 @@ public class PersistentAuditEvent {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@ToString.Include
private Long id;
private String principal;
private String type;
@ToString.Include private String principal;
@ToString.Include private String type;
private String source;
@Column(columnDefinition = "text")
private String data; // JSON blob
private Instant timestamp;
@ToString.Include private Instant timestamp;
@Override
public final boolean equals(Object o) {
if (this == o) return true;
if (o == null) return false;
Class<?> oEffectiveClass =
o instanceof HibernateProxy
? ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass()
: o.getClass();
Class<?> thisEffectiveClass =
this instanceof HibernateProxy
? ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass()
: this.getClass();
if (thisEffectiveClass != oEffectiveClass) return false;
PersistentAuditEvent that = (PersistentAuditEvent) o;
return getId() != null && Objects.equals(getId(), that.getId());
}
@Override
public final int hashCode() {
return this instanceof HibernateProxy
? ((HibernateProxy) this)
.getHibernateLazyInitializer()
.getPersistentClass()
.hashCode()
: getClass().hashCode();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java
+26 -14
View File
@@ -130,25 +130,27 @@ public class PolicyEngine {
// worker.
String principal = currentActingPrincipal();
return submitForPrincipal(
principal, principal, policyId, definition, inputs, null, listener);
principal, principal, policyId, definition, inputs, listener, null, null);
}
/** Run a stored policy on demand. {@code enabled} gates triggers, not explicit runs. */
public PolicyRunHandle runPolicy(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
return runPolicy(policy, inputs, null, listener);
return runPolicy(policy, inputs, listener, null, null);
}
/**
* As {@link #runPolicy(Policy, PolicyInputs, PolicyProgressListener)}, with the source's opaque
* reference to the document being run. Carried so a failure can say which document it was
* about, and so the same document failing again folds into one incident.
* As {@link #runPolicy(Policy, PolicyInputs, PolicyProgressListener)}, recording which source
* fed the run and its opaque reference to the document. The first says where an unattended
* failure came from; the second says which document, and is what lets the same document failing
* again fold into one incident. Both null for a user's upload.
*/
public PolicyRunHandle runPolicy(
Policy policy,
PolicyInputs inputs,
String fileIdentity,
PolicyProgressListener listener) {
PolicyProgressListener listener,
String sourceId,
String fileIdentity) {
// Bill the policy owner: trigger-fired runs have no security context, and the async worker
// doesn't inherit the caller's, so the owner (stamped at policy creation) is the reliable
// billing identity — and for org-wide policies the org/owner is meant to pay. But own the
@@ -172,9 +174,12 @@ public class PolicyEngine {
fileOwner,
policy.id(),
definition,
// main's asset-resolved inputs, not the raw ones: stored certificates and watermark
// images bind here, before the async hop, because worker threads have no principal.
resolved,
fileIdentity,
listener);
listener,
sourceId,
fileIdentity);
}
private PolicyRunHandle submitForPrincipal(
@@ -183,8 +188,9 @@ public class PolicyEngine {
String policyId,
PipelineDefinition definition,
PolicyInputs inputs,
String fileIdentity,
PolicyProgressListener listener) {
PolicyProgressListener listener,
String sourceId,
String fileIdentity) {
// Scope the run id to the current user (this request thread) so the file-download
// ownership check passes. No-op when security is off.
String runId = jobOwnershipService.createScopedJobKey(UUID.randomUUID().toString());
@@ -193,7 +199,7 @@ public class PolicyEngine {
if (policyId != null) {
taskManager.putMetadata(runId, "policyId", policyId);
}
PolicyRun run = new PolicyRun(runId, policyId, definition, fileIdentity);
PolicyRun run = new PolicyRun(runId, policyId, definition, sourceId, fileIdentity);
registry.register(run);
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
PolicyProgressListener tracking = trackingListener(runId, run, listener);
@@ -359,7 +365,12 @@ public class PolicyEngine {
// No exception to classify here: nothing was thrown by a tool, the run simply was not
// admitted. Record it explicitly so a run lost to load pressure is still accounted for.
failureRecorder.recordRunFailureAs(
FailureKind.UNKNOWN, run.getRunId(), run.getPolicyId(), null, message);
FailureKind.UNKNOWN,
run.getRunId(),
run.getPolicyId(),
run.getSourceId(),
null,
message);
completion.complete(run);
}
return null;
@@ -373,8 +384,9 @@ public class PolicyEngine {
failureRecorder.recordRunFailure(
run.getRunId(),
run.getPolicyId(),
MDC.get(AUDIT_PRINCIPAL_MDC_KEY),
run.getSourceId(),
run.getFileIdentity(),
MDC.get(AUDIT_PRINCIPAL_MDC_KEY),
message,
cause);
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
+16 -4
View File
@@ -79,7 +79,8 @@ public class PolicyRunner {
// Generator pipeline: one run with no input. Still fall through to the cleanup
// below so rows recorded for its folder outputs are pruned like anything else,
// instead of accumulating until the policy is deleted.
runIds.add(startRun(policy, PolicyInputs.of(List.of()), null, unused -> {}));
// Generator pipeline: no input, so neither a source nor a document to attribute to.
runIds.add(startRun(policy, null, null, PolicyInputs.of(List.of()), unused -> {}));
}
for (PipelineInput input : inputs) {
String sourceId = input.sourceId();
@@ -167,7 +168,13 @@ public class PolicyRunner {
List<String> runIds = new ArrayList<>();
long docsFed = 0;
for (ResolvedInput unit : work) {
runIds.add(startRun(policy, unit.inputs(), unit.fileIdentity(), unit.onComplete()));
runIds.add(
startRun(
policy,
sourceId,
unit.fileIdentity(),
unit.inputs(),
unit.onComplete()));
docsFed += unit.inputs().primary().size();
}
docCounter.record(sourceId, docsFed);
@@ -175,10 +182,15 @@ public class PolicyRunner {
}
private String startRun(
Policy policy, PolicyInputs inputs, String fileIdentity, Consumer<Boolean> onComplete) {
Policy policy,
String sourceId,
String fileIdentity,
PolicyInputs inputs,
Consumer<Boolean> onComplete) {
log.info("Running policy {} ({})", policy.id(), policy.name());
PolicyRunHandle handle =
policyEngine.runPolicy(policy, inputs, fileIdentity, PolicyProgressListener.NOOP);
policyEngine.runPolicy(
policy, inputs, PolicyProgressListener.NOOP, sourceId, fileIdentity);
handle.completion()
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
return handle.runId();
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/PolicyRun.java
+18 -1
View File
@@ -21,6 +21,12 @@ public class PolicyRun {
/** ID of the stored policy that produced this run; null for ad-hoc pipelines. */
private final String policyId;
/**
* ID of the source the input came from (folder, S3, webhook), or null when a user supplied the
* files. Recorded on a failure so a reviewer can see where an unattended file came from.
*/
private final String sourceId;
private final PipelineDefinition definition;
/**
@@ -56,10 +62,21 @@ public class PolicyRun {
private volatile List<ResultFile> outputs = List.of();
private volatile Instant updatedAt = Instant.now();
/**
* Both references are required rather than defaulted: a run with neither is a real case (a
* user's upload, an ad-hoc pipeline), but it should be stated at the call site. Overloads that
* omitted them would make losing the attribution the frictionless option, which is how both
* fields went unpopulated in the first place.
*/
public PolicyRun(
String runId, String policyId, PipelineDefinition definition, String fileIdentity) {
String runId,
String policyId,
PipelineDefinition definition,
String sourceId,
String fileIdentity) {
this.runId = runId;
this.policyId = policyId;
this.sourceId = sourceId;
this.definition = definition;
this.fileIdentity = fileIdentity;
}

Some files were not shown because too many files have changed in this diff Show More