mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
60c81e20d0 | ||
|
|
4564ed5bec | ||
|
|
7743720f0a | ||
|
|
620f246ca4 | ||
|
|
dd24440b73 | ||
|
|
c0374266e7 | ||
|
|
730ed43af9 | ||
|
|
56ff1e5092 | ||
|
|
05b80fbe4f | ||
|
|
4047d02086 | ||
|
|
5f78083470 | ||
|
|
fc048872d7 | ||
|
|
6ee533c427 | ||
|
|
a13683e84a | ||
|
|
914b474d84 | ||
|
|
51f69a718f | ||
|
|
7f597051e4 | ||
|
|
e113df6252 | ||
|
|
06b4a4184b | ||
|
|
19982d7207 | ||
|
|
cdfefb02bc | ||
|
|
5ba6a65c1a | ||
|
|
017c8d59fa | ||
|
|
0a50e765b7 | ||
|
|
48027ee9d6 | ||
|
|
22d56bb1fb |
+2
-2
@@ -34,8 +34,8 @@ frontend/playwright-report/
|
||||
# Tauri/desktop builds
|
||||
src-tauri/target/
|
||||
src-tauri/dist/
|
||||
frontend/src-tauri/target/
|
||||
frontend/src-tauri/dist/
|
||||
frontend/editor/src-tauri/target/
|
||||
frontend/editor/src-tauri/dist/
|
||||
|
||||
# IDE and editor
|
||||
.idea/
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
###############################################################################
|
||||
# Stirling-PDF SaaS local environment template.
|
||||
#
|
||||
# Copy this file to `.env.saas.local` (gitignored) and fill in real values.
|
||||
# Loaded by `task backend:dev:saas` via Taskfile's `dotenv:` directive, then
|
||||
# read by Spring Boot's `${...}` placeholders in application-saas.properties
|
||||
# and application-dev.properties.
|
||||
#
|
||||
# DO NOT commit `.env.saas.local`. Only `.env.saas.example` is checked in.
|
||||
###############################################################################
|
||||
|
||||
# ---------- Supabase project ----------
|
||||
# Project reference (the subdomain part of <ref>.supabase.co). Required.
|
||||
# Example dev project:
|
||||
SAAS_DB_PROJECT_REF=
|
||||
|
||||
# Edge function secret used by billing/license rollup calls.
|
||||
SUPABASE_EDGE_FUNCTION_SECRET=
|
||||
|
||||
# ---------- Database (saas profile) ----------
|
||||
# Direct JDBC URL to the Supabase Postgres. Required when running the plain
|
||||
# `saas` profile (i.e. without `--spring.profiles.include=dev`).
|
||||
# Example: jdbc:postgresql://db.<project-ref>.supabase.co:5432/postgres
|
||||
SAAS_DB_URL=
|
||||
SAAS_DB_USERNAME=postgres
|
||||
SAAS_DB_PASSWORD=
|
||||
|
||||
# ---------- Database (dev profile overrides) ----------
|
||||
# Used when `--spring.profiles.include=dev` is active. The dev profile
|
||||
# defaults the URL/username to the shared dev Supabase project, but the
|
||||
# password must still be provided here.
|
||||
SAAS_DEV_DB_URL=
|
||||
SAAS_DEV_DB_USERNAME=postgres
|
||||
SAAS_DEV_DB_PASSWORD=
|
||||
@@ -8,7 +8,6 @@ openapi: &openapi
|
||||
|
||||
docker-base: &docker-base
|
||||
- docker/base/Dockerfile
|
||||
- ".github/workflows/push-docker-base.yml"
|
||||
|
||||
docker: &docker
|
||||
- docker/embedded/Dockerfile
|
||||
@@ -58,12 +57,12 @@ frontend: &frontend
|
||||
# Files that affect the Tauri desktop bundle. Gate the multi-OS Tauri build
|
||||
# job on changes to any of these.
|
||||
tauri: &tauri
|
||||
- frontend/src-tauri/**
|
||||
- frontend/src/desktop/**
|
||||
- frontend/tsconfig.desktop.json
|
||||
- frontend/editor/src-tauri/**
|
||||
- frontend/editor/src/desktop/**
|
||||
- frontend/editor/tsconfig.desktop.vite.json
|
||||
- frontend/package.json
|
||||
- frontend/package-lock.json
|
||||
- frontend/vite.config.ts
|
||||
- frontend/editor/vite.config.ts
|
||||
- .github/workflows/tauri-build.yml
|
||||
|
||||
# Files that affect the AI engine (Python tool models, fixers, tests). Gate
|
||||
@@ -78,7 +77,7 @@ licenses-frontend: &licenses-frontend
|
||||
- ".github/workflows/frontend-backend-licenses-update.yml"
|
||||
- "frontend/package.json"
|
||||
- "frontend/package-lock.json"
|
||||
- "frontend/scripts/generate-licenses.js"
|
||||
- "frontend/editor/scripts/generate-licenses.js"
|
||||
|
||||
licenses-backend: &licenses-backend
|
||||
- ".github/workflows/frontend-backend-licenses-update.yml"
|
||||
@@ -88,8 +87,8 @@ licenses-backend: &licenses-backend
|
||||
# Playwright job on changes to any of these on PRs.
|
||||
proprietary: &proprietary
|
||||
- app/proprietary/**
|
||||
- frontend/src/proprietary/**
|
||||
- frontend/src/core/tests/enterprise/**
|
||||
- frontend/editor/src/proprietary/**
|
||||
- frontend/editor/src/core/tests/enterprise/**
|
||||
- testing/compose/docker-compose-keycloak-oauth.yml
|
||||
- testing/compose/docker-compose-keycloak-saml.yml
|
||||
- testing/compose/keycloak-realm-oauth.json
|
||||
|
||||
@@ -25,6 +25,8 @@ updates:
|
||||
- "/docker/frontend"
|
||||
- "/docker/base"
|
||||
- "/docker/engine"
|
||||
- "/docker/unoserver"
|
||||
- "/engine"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
@@ -126,9 +128,9 @@ updates:
|
||||
|
||||
- package-ecosystem: cargo
|
||||
directories:
|
||||
- /frontend/src-tauri
|
||||
- /frontend/src-tauri/thumbnail-handler
|
||||
- /frontend/src-tauri/provisioner
|
||||
- /frontend/editor/src-tauri
|
||||
- /frontend/editor/src-tauri/thumbnail-handler
|
||||
- /frontend/editor/src-tauri/provisioner
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
|
||||
@@ -51,7 +51,7 @@ labels:
|
||||
|
||||
- label: 'Translation'
|
||||
files:
|
||||
- 'frontend/public/locales/[a-zA-Z]{2}-[a-zA-Z\-]{2,7}/translation.toml'
|
||||
- 'frontend/editor/public/locales/[a-zA-Z]{2}-[a-zA-Z\-]{2,7}/translation.toml'
|
||||
- 'scripts/ignore_translation.toml'
|
||||
- 'scripts/remove_translation_keys.sh'
|
||||
- 'scripts/replace_translation_line.sh'
|
||||
@@ -69,8 +69,8 @@ labels:
|
||||
|
||||
- label: 'Tauri'
|
||||
files:
|
||||
- 'frontend/src-tauri/**'
|
||||
- 'frontend/src-tauri/.*'
|
||||
- 'frontend/editor/src-tauri/**'
|
||||
- 'frontend/editor/src-tauri/.*'
|
||||
|
||||
- label: 'engine'
|
||||
files:
|
||||
|
||||
@@ -23,9 +23,13 @@ permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
check-pr:
|
||||
if: (github.event_name == 'pull_request' && github.event.action != 'closed') || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
outputs:
|
||||
should_deploy: ${{ steps.decide.outputs.should_deploy }}
|
||||
is_fork: ${{ steps.resolve.outputs.is_fork }}
|
||||
@@ -35,7 +39,7 @@ jobs:
|
||||
pr_ref: ${{ steps.resolve.outputs.ref }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -97,8 +101,8 @@ jobs:
|
||||
echo "allow_fork=${allow_fork:-false}" >> $GITHUB_OUTPUT
|
||||
|
||||
deploy-v2-pr:
|
||||
needs: check-pr
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, check-pr]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
if: needs.check-pr.outputs.should_deploy == 'true' && (needs.check-pr.outputs.is_fork == 'false' || needs.check-pr.outputs.allow_fork == 'true')
|
||||
# Concurrency control - only one deployment per PR at a time
|
||||
concurrency:
|
||||
@@ -108,10 +112,14 @@ jobs:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -179,7 +187,12 @@ jobs:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fetch-depth: 0 # Fetch full history for commit hash detection
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
@@ -224,8 +237,20 @@ jobs:
|
||||
echo "Image needs to be built"
|
||||
fi
|
||||
|
||||
- name: Build and push V2 image
|
||||
if: steps.check-image.outputs.exists == 'false'
|
||||
- name: Build and push V2 image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-image.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-${{ steps.commit-hash.outputs.app_short }}
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push V2 image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-image.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
@@ -351,7 +376,8 @@ jobs:
|
||||
|
||||
cleanup-v2-deployment:
|
||||
if: github.event.action == 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
@@ -359,7 +385,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -34,8 +34,12 @@ permissions:
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
check-comment:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
issues: write
|
||||
if: |
|
||||
@@ -70,7 +74,7 @@ jobs:
|
||||
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -175,15 +179,19 @@ jobs:
|
||||
}
|
||||
|
||||
deploy-pr:
|
||||
needs: check-comment
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, check-comment]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -232,7 +240,12 @@ jobs:
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
@@ -241,7 +254,22 @@ jobs:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Build and push PR-specific image
|
||||
- name: Build and push PR-specific image (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
|
||||
build-args: |
|
||||
VERSION_TAG=alpha
|
||||
PROTOTYPES_BUILD=${{ needs.check-comment.outputs.enable_prototypes }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push PR-specific image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
@@ -255,8 +283,19 @@ jobs:
|
||||
PROTOTYPES_BUILD=${{ needs.check-comment.outputs.enable_prototypes }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push engine image
|
||||
if: needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
- name: Build and push engine image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: ./engine
|
||||
file: ./engine/Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:engine-pr-${{ needs.check-comment.outputs.pr_number }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push engine image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && needs.check-comment.outputs.enable_prototypes == 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./engine
|
||||
@@ -471,10 +510,11 @@ jobs:
|
||||
|
||||
handle-label-commands:
|
||||
if: ${{ github.event.issue.pull_request != null }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: _runner-pick
|
||||
|
||||
# Tiny reusable workflow that classifies the trigger as either a "fork PR
|
||||
# from an untrusted contributor" or a "trusted commit" so downstream jobs
|
||||
# can pick a runner class without each one duplicating the 200-char gate
|
||||
# expression in their own `runs-on:`.
|
||||
#
|
||||
# Caller pattern:
|
||||
#
|
||||
# jobs:
|
||||
# pick:
|
||||
# uses: ./.github/workflows/_runner-pick.yml
|
||||
#
|
||||
# real-work:
|
||||
# needs: pick
|
||||
# runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-8' }}
|
||||
# steps: [...]
|
||||
#
|
||||
# Output:
|
||||
# is_fork: "true" when the trigger is a pull_request from a fork or an
|
||||
# untrusted author_association, "false" otherwise.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
is_fork:
|
||||
description: '"true" if the trigger is an untrusted fork PR.'
|
||||
value: ${{ jobs.pick.outputs.is_fork }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 1
|
||||
outputs:
|
||||
is_fork: ${{ steps.decide.outputs.is_fork }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Classify the trigger
|
||||
id: decide
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_REPO_FORK: ${{ github.event.pull_request.head.repo.fork }}
|
||||
AUTHOR_ASSOC: ${{ github.event.pull_request.author_association }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "${PR_NUMBER:-}" ]; then
|
||||
# Not a pull_request event at all (push, schedule, workflow_dispatch,
|
||||
# workflow_call from a non-PR trigger) -> trusted by default.
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${HEAD_REPO_FORK}" = "true" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
case "${AUTHOR_ASSOC}" in
|
||||
OWNER|MEMBER|COLLABORATOR)
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
esac
|
||||
@@ -17,14 +17,16 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
- name: AI PR Title Analysis
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: ai-title-analysis
|
||||
uses: actions/ai-inference@e09e65981758de8b2fdab13c2bfb7c7d5493b0b6 # v2.0.7
|
||||
uses: actions/ai-inference@17ff458cb182449bbb2e43701fcd98f6af8f6570 # v2.1.0
|
||||
with:
|
||||
model: openai/gpt-4o
|
||||
system-prompt-file: ".github/config/system-prompt.txt"
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -70,12 +70,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository (for PKGBUILD templates)
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Update stirling-pdf-desktop PKGBUILD
|
||||
env:
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Publish stirling-pdf-desktop to AUR
|
||||
if: ${{ github.event_name == 'release' || inputs.dry_run == false }}
|
||||
uses: KSXGitHub/github-actions-deploy-aur@v4.1.3
|
||||
uses: KSXGitHub/github-actions-deploy-aur@da03e160361ce01bf087e790b6ffd196d7dccff7 # v4.1.3
|
||||
with:
|
||||
pkgname: stirling-pdf-desktop
|
||||
pkgbuild: .github/aur/stirling-pdf-desktop/PKGBUILD
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: Backend build, format check, and coverage
|
||||
|
||||
# Reusable workflow called from build.yml. Runs the full backend build matrix
|
||||
# (JDK 21/25 × spring-security on/off), Spotless formatting check, JUnit, and
|
||||
# Reusable workflow called from build.yml. Runs the backend build matrix
|
||||
# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and
|
||||
# posts Jacoco coverage to PRs.
|
||||
on:
|
||||
workflow_call:
|
||||
@@ -13,16 +13,22 @@ permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-8' }}
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk-version: [21, 25]
|
||||
jdk-version: [25]
|
||||
spring-security: [true, false]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -17,11 +17,23 @@ name: Enterprise E2E (Playwright)
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
|
||||
required: false
|
||||
type: string
|
||||
default: "8"
|
||||
push:
|
||||
branches: ["main"]
|
||||
schedule:
|
||||
- cron: "0 4 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
|
||||
required: false
|
||||
type: string
|
||||
default: "8"
|
||||
|
||||
# No `concurrency:` block here on purpose. When this workflow is called via
|
||||
# workflow_call from build.yml, ${{ github.workflow }}/event_name/pr_number
|
||||
@@ -35,16 +47,21 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
playwright-e2e-enterprise:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
PREMIUM_KEY: ${{ secrets.PREMIUM_KEY_ENTERPRISE }}
|
||||
PREMIUM_ENABLED: "true"
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -268,5 +285,5 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-enterprise-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -135,6 +135,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/test-build-docker.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
@@ -197,6 +198,11 @@ jobs:
|
||||
- dependency-review
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Verify every required job passed (or was legitimately skipped)
|
||||
env:
|
||||
RESULTS: |
|
||||
|
||||
@@ -11,9 +11,11 @@ permissions:
|
||||
jobs:
|
||||
check-licence:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -10,11 +10,17 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
check-generate-openapi-docs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- "frontend/public/locales/*/translation.toml"
|
||||
- "frontend/editor/public/locales/*/translation.toml"
|
||||
- ".github/scripts/check_language_toml.py"
|
||||
- ".github/workflows/check_toml.yml"
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
# Get changed files and filter for TOML translation files
|
||||
gh pr view ${{ steps.get-pr-data.outputs.pr_number }} --json files -q ".files[].path" | grep -E '^frontend/public/locales/[a-zA-Z-]+/translation\.toml$' > changed_files.txt || echo "No matching TOML files found in PR"
|
||||
gh pr view ${{ steps.get-pr-data.outputs.pr_number }} --json files -q ".files[].path" | grep -E '^frontend/editor/public/locales/[a-zA-Z-]+/translation\.toml$' > changed_files.txt || echo "No matching TOML files found in PR"
|
||||
# Check if any files were found
|
||||
if [ ! -s changed_files.txt ]; then
|
||||
echo "No TOML translation files changed in this PR"
|
||||
@@ -166,12 +166,12 @@ jobs:
|
||||
|
||||
// Determine reference file
|
||||
let referenceFilePath;
|
||||
if (changedFiles.includes("frontend/public/locales/en-GB/translation.toml")) {
|
||||
if (changedFiles.includes("frontend/editor/public/locales/en-GB/translation.toml")) {
|
||||
console.log("Using PR branch reference file.");
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: prRepoOwner,
|
||||
repo: prRepoName,
|
||||
path: "frontend/public/locales/en-GB/translation.toml",
|
||||
path: "frontend/editor/public/locales/en-GB/translation.toml",
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
path: "frontend/public/locales/en-GB/translation.toml",
|
||||
path: "frontend/editor/public/locales/en-GB/translation.toml",
|
||||
ref: "main",
|
||||
});
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -10,22 +10,37 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
deploy-v2-on-push:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
concurrency:
|
||||
group: deploy-v2-push-V2
|
||||
cancel-in-progress: true
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get commit hashes for frontend and backend
|
||||
@@ -90,8 +105,22 @@ jobs:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Build and push frontend image
|
||||
if: steps.check-frontend.outputs.exists == 'false'
|
||||
- name: Build and push frontend image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-frontend.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/frontend/Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-${{ steps.commit-hashes.outputs.frontend_short }}
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-frontend-latest
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push frontend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-frontend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
@@ -105,8 +134,22 @@ jobs:
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push backend image
|
||||
if: steps.check-backend.outputs.exists == 'false'
|
||||
- name: Build and push backend image (Depot)
|
||||
if: env.USE_DEPOT == 'true' && steps.check-backend.outputs.exists == 'false'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/backend/Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-${{ steps.commit-hashes.outputs.backend_short }}
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/test:v2-backend-latest
|
||||
build-args: VERSION_TAG=v2-alpha
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push backend image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true' && steps.check-backend.outputs.exists == 'false'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
|
||||
@@ -11,21 +11,32 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: "false"
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking. Tuned to 4 because bench showed 16 was within noise of 4."
|
||||
required: false
|
||||
type: string
|
||||
default: "4"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
docker-compose-tests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '4') }}
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
checks: write
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -5,17 +5,27 @@ name: Playwright E2E (live backend)
|
||||
# server.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
|
||||
required: false
|
||||
type: string
|
||||
default: "8"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
playwright-e2e-live:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -60,5 +70,5 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-live-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -5,16 +5,26 @@ name: Playwright E2E (stubbed)
|
||||
# mocks API responses in the browser.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking. Tuned to 8 to match the other playwright workflows; bench showed flat scaling above 8."
|
||||
required: false
|
||||
type: string
|
||||
default: "8"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
playwright-e2e:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -40,5 +50,5 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-stubbed-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
@@ -19,16 +19,20 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
files-changed:
|
||||
name: detect what files changed
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
timeout-minutes: 3
|
||||
outputs:
|
||||
licenses-frontend: ${{ steps.changes.outputs.licenses-frontend }}
|
||||
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -44,15 +48,15 @@ jobs:
|
||||
generate-frontend-license-report:
|
||||
if: needs.files-changed.outputs.licenses-frontend == 'true'
|
||||
name: Generate Frontend License Report
|
||||
needs: files-changed
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, files-changed]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -114,23 +118,23 @@ jobs:
|
||||
env:
|
||||
PR_IS_FORK: "true"
|
||||
run: |
|
||||
node base/frontend/scripts/generate-licenses.js \
|
||||
--input frontend/src/assets/3rdPartyLicenses.json
|
||||
node base/frontend/editor/scripts/generate-licenses.js \
|
||||
--input frontend/editor/src/assets/3rdPartyLicenses.json
|
||||
|
||||
- name: Copy postprocessed artifacts back (fork PRs)
|
||||
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true
|
||||
run: |
|
||||
mkdir -p frontend/src/assets
|
||||
if [ -f "base/frontend/src/assets/3rdPartyLicenses.json" ]; then
|
||||
cp base/frontend/src/assets/3rdPartyLicenses.json frontend/src/assets/3rdPartyLicenses.json
|
||||
mkdir -p frontend/editor/src/assets
|
||||
if [ -f "base/frontend/editor/src/assets/3rdPartyLicenses.json" ]; then
|
||||
cp base/frontend/editor/src/assets/3rdPartyLicenses.json frontend/editor/src/assets/3rdPartyLicenses.json
|
||||
fi
|
||||
if [ -f "base/frontend/src/assets/license-warnings.json" ]; then
|
||||
cp base/frontend/src/assets/license-warnings.json frontend/src/assets/license-warnings.json
|
||||
if [ -f "base/frontend/editor/src/assets/license-warnings.json" ]; then
|
||||
cp base/frontend/editor/src/assets/license-warnings.json frontend/editor/src/assets/license-warnings.json
|
||||
fi
|
||||
|
||||
- name: Check for license warnings
|
||||
run: |
|
||||
if [ -f "frontend/src/assets/license-warnings.json" ]; then
|
||||
if [ -f "frontend/editor/src/assets/license-warnings.json" ]; then
|
||||
echo "LICENSE_WARNINGS_EXIST=true" >> $GITHUB_ENV
|
||||
else
|
||||
echo "LICENSE_WARNINGS_EXIST=false" >> $GITHUB_ENV
|
||||
@@ -178,10 +182,10 @@ jobs:
|
||||
echo ""
|
||||
if [ "${LICENSE_WARNINGS_EXIST}" = "true" ]; then
|
||||
echo "❌ **Failed** – incompatible or unknown licenses found."
|
||||
if [ -f "frontend/src/assets/license-warnings.json" ]; then
|
||||
if [ -f "frontend/editor/src/assets/license-warnings.json" ]; then
|
||||
echo ""
|
||||
echo "### Warnings"
|
||||
jq -r '.warnings[] | "- \(.message)"' frontend/src/assets/license-warnings.json || true
|
||||
jq -r '.warnings[] | "- \(.message)"' frontend/editor/src/assets/license-warnings.json || true
|
||||
fi
|
||||
else
|
||||
echo "✅ **Passed** – no license warnings detected."
|
||||
@@ -207,7 +211,7 @@ jobs:
|
||||
const fs = require('fs');
|
||||
let warningDetails = '';
|
||||
try {
|
||||
const warnings = JSON.parse(fs.readFileSync('frontend/src/assets/license-warnings.json', 'utf8'));
|
||||
const warnings = JSON.parse(fs.readFileSync('frontend/editor/src/assets/license-warnings.json', 'utf8'));
|
||||
warningDetails = warnings.warnings.map(w => `- ${w.message}`).join('\n');
|
||||
} catch (e) {
|
||||
warningDetails = 'Unable to read warning details';
|
||||
@@ -247,7 +251,7 @@ jobs:
|
||||
- name: Commit changes (Push only)
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
git add frontend/src/assets/3rdPartyLicenses.json
|
||||
git add frontend/editor/src/assets/3rdPartyLicenses.json
|
||||
# Note: Do NOT commit license-warnings.json - it's only for PR review
|
||||
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
|
||||
|
||||
@@ -265,7 +269,7 @@ jobs:
|
||||
|
||||
The following licenses may require review for corporate compatibility:
|
||||
|
||||
$(cat frontend/src/assets/license-warnings.json | jq -r '.warnings[].message')
|
||||
$(cat frontend/editor/src/assets/license-warnings.json | jq -r '.warnings[].message')
|
||||
|
||||
Please review these licenses to ensure they are acceptable for your use case."
|
||||
fi
|
||||
@@ -307,16 +311,18 @@ jobs:
|
||||
|
||||
generate-backend-license-report:
|
||||
if: needs.files-changed.outputs.licenses-backend == 'true'
|
||||
needs: files-changed
|
||||
needs: [pick, files-changed]
|
||||
name: Generate Backend License Report
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
repository-projects: write # Required for enabling automerge
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -11,11 +11,15 @@ permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
frontend-validation:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -109,5 +113,5 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: frontend-build
|
||||
path: frontend/dist/
|
||||
path: frontend/editor/dist/
|
||||
retention-days: 3
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -36,15 +36,19 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
determine-matrix:
|
||||
if: ${{ vars.CI_PROFILE != 'lite' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -90,26 +94,28 @@ jobs:
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
case "${{ github.event.inputs.platform }}" in
|
||||
"windows")
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64"}]}' >> $GITHUB_OUTPUT
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64","jpdfium_platforms":"windows-x64"}]}' >> $GITHUB_OUTPUT
|
||||
;;
|
||||
"macos")
|
||||
echo 'matrix={"include":[{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal"}]}' >> $GITHUB_OUTPUT
|
||||
echo 'matrix={"include":[{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal","jpdfium_platforms":"darwin-arm64,darwin-x64"}]}' >> $GITHUB_OUTPUT
|
||||
;;
|
||||
"linux")
|
||||
echo 'matrix={"include":[{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64"}]}' >> $GITHUB_OUTPUT
|
||||
echo 'matrix={"include":[{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64","jpdfium_platforms":"linux-x64"}]}' >> $GITHUB_OUTPUT
|
||||
;;
|
||||
*)
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64"},{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal"},{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64"}]}' >> $GITHUB_OUTPUT
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64","jpdfium_platforms":"windows-x64"},{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal","jpdfium_platforms":"darwin-arm64,darwin-x64"},{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64","jpdfium_platforms":"linux-x64"}]}' >> $GITHUB_OUTPUT
|
||||
;;
|
||||
esac
|
||||
else
|
||||
# For push/release events, build all platforms
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64"},{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal"},{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64"}]}' >> $GITHUB_OUTPUT
|
||||
echo 'matrix={"include":[{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64","jpdfium_platforms":"windows-x64"},{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal","jpdfium_platforms":"darwin-arm64,darwin-x64"},{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64","jpdfium_platforms":"linux-x64"}]}' >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
build-jars:
|
||||
needs: determine-matrix
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, determine-matrix]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
strategy:
|
||||
matrix:
|
||||
variant:
|
||||
@@ -127,7 +133,7 @@ jobs:
|
||||
file_suffix: "-server"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -189,9 +195,10 @@ jobs:
|
||||
SM_API_KEY: ${{ secrets.SM_API_KEY }}
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
allowed-endpoints: >
|
||||
@@ -256,15 +263,17 @@ jobs:
|
||||
if: matrix.platform == 'macos-15'
|
||||
env:
|
||||
AARCH64_JAVA_HOME: ${{ env.JAVA_HOME }}
|
||||
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
|
||||
run: task desktop:jlink:universal-mac
|
||||
|
||||
- name: Prepare desktop build
|
||||
run: task desktop:prepare
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
|
||||
run: task desktop:prepare
|
||||
|
||||
# DigiCert KeyLocker Setup (Cloud HSM)
|
||||
- name: Setup DigiCert KeyLocker
|
||||
@@ -418,7 +427,7 @@ jobs:
|
||||
env:
|
||||
KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
|
||||
run: |
|
||||
cat > ./frontend/src-tauri/tauri.windows.conf.json <<EOF
|
||||
cat > ./frontend/editor/src-tauri/tauri.windows.conf.json <<EOF
|
||||
{
|
||||
"bundle": {
|
||||
"windows": {
|
||||
@@ -431,7 +440,11 @@ jobs:
|
||||
}
|
||||
EOF
|
||||
echo "Generated tauri.windows.conf.json (alias masked):"
|
||||
sed "s/${KEYPAIR_ALIAS}/***/g" ./frontend/src-tauri/tauri.windows.conf.json
|
||||
sed "s/${KEYPAIR_ALIAS}/***/g" ./frontend/editor/src-tauri/tauri.windows.conf.json
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Import release GPG signing key (Linux)
|
||||
if: matrix.platform == 'ubuntu-22.04' && env.RELEASE_GPG_PRIVATE_KEY != '' && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.sign != 'false') || github.ref == 'refs/heads/V2-master')
|
||||
@@ -479,7 +492,7 @@ jobs:
|
||||
SM_CODE_SIGNING_CERT_SHA1_HASH: ${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }}
|
||||
CI: true
|
||||
with:
|
||||
projectPath: ./frontend
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
args: ${{ matrix.args }}
|
||||
|
||||
@@ -505,7 +518,7 @@ jobs:
|
||||
$allSigned = $true
|
||||
|
||||
# Check MSI installer (outer wrapper - what users download)
|
||||
$msiFiles = Get-ChildItem -Path "./frontend/src-tauri/target" -Filter "*.msi" -Recurse -File
|
||||
$msiFiles = Get-ChildItem -Path "./frontend/editor/src-tauri/target" -Filter "*.msi" -Recurse -File
|
||||
if ($msiFiles.Count -eq 0) {
|
||||
Write-Host "[ERROR] No MSI found under target/"
|
||||
exit 1
|
||||
@@ -574,7 +587,7 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ./dist
|
||||
cd ./frontend/src-tauri/target
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
# Find and rename artifacts based on platform
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
@@ -600,13 +613,13 @@ jobs:
|
||||
|
||||
create-release:
|
||||
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_mode != 'true') || github.event_name == 'release' || github.ref == 'refs/heads/V2-master'
|
||||
needs: [determine-matrix, build, build-jars]
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, determine-matrix, build, build-jars]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -13,12 +13,16 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
playwright-all-browsers:
|
||||
name: Playwright (chromium + firefox + webkit)
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -45,5 +49,5 @@ jobs:
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-nightly-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
path: frontend/editor/playwright-report/
|
||||
retention-days: 14
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Update Package Manager Manifests
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [ released ]
|
||||
types: [released]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -79,12 +79,12 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout homebrew-stirling-pdf tap (also hosts Scoop bucket)
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: Stirling-Tools/homebrew-stirling-pdf
|
||||
token: ${{ secrets.HOMEBREW_TAP_TOKEN }}
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
PIP_DISABLE_PIP_VERSION_CHECK: "1"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -75,6 +75,6 @@ jobs:
|
||||
|
||||
# Upload the results to GitHub's code scanning dashboard.
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3.29.5
|
||||
uses: github/codeql-action/upload-sarif@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v3.29.5
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -22,12 +22,18 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
push:
|
||||
if: ${{ vars.CI_PROFILE != 'lite' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ on:
|
||||
- "app/core/build.gradle"
|
||||
- "app/proprietary/build.gradle"
|
||||
- "README.md"
|
||||
- "frontend/public/locales/*/translation.toml"
|
||||
- "frontend/editor/public/locales/*/translation.toml"
|
||||
- "app/core/src/main/resources/static/3rdPartyLicenses.json"
|
||||
- "scripts/ignore_translation.toml"
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Sync translation TOML files
|
||||
run: |
|
||||
python .github/scripts/check_language_toml.py --reference-file "frontend/public/locales/en-GB/translation.toml" --branch main
|
||||
python .github/scripts/check_language_toml.py --reference-file "frontend/editor/public/locales/en-GB/translation.toml" --branch main
|
||||
|
||||
- name: pre-commit run
|
||||
run: |
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Commit translation files
|
||||
run: |
|
||||
git add frontend/public/locales/*/translation.toml
|
||||
git add frontend/editor/public/locales/*/translation.toml
|
||||
git diff --staged --quiet || git commit -m ":memo: Sync translation files (TOML)" || echo "No changes detected"
|
||||
|
||||
- name: Sync README.md
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
|
||||
|
||||
#### **1. Synchronization of Translation Files**
|
||||
- Updated translation files (`frontend/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`.
|
||||
- Updated translation files (`frontend/editor/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`.
|
||||
- Ensured consistency and synchronization across all supported language files.
|
||||
- Highlighted any missing or incomplete translations.
|
||||
- **Format**: TOML
|
||||
@@ -125,5 +125,5 @@ jobs:
|
||||
sign-commits: true
|
||||
add-paths: |
|
||||
README.md
|
||||
frontend/public/locales/*/translation.toml
|
||||
frontend/editor/public/locales/*/translation.toml
|
||||
scripts/ignore_translation.toml
|
||||
|
||||
@@ -4,6 +4,10 @@ name: Build Tauri Applications
|
||||
# Linux). Called from build.yml on PRs that touch desktop sources (gated
|
||||
# via the `tauri` filter in .github/config/.files.yaml). Also runnable
|
||||
# on demand via workflow_dispatch with a per-platform selector.
|
||||
#
|
||||
# Note: editing this file is itself enough to make the `tauri` path filter
|
||||
# match, which is how non-desktop PRs (e.g. backend-only fixes) opt into a
|
||||
# desktop smoke build.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
@@ -37,7 +41,7 @@ jobs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -47,12 +51,10 @@ jobs:
|
||||
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
|
||||
PLATFORM: ${{ inputs.platform }}
|
||||
run: |
|
||||
WINDOWS='{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64"}'
|
||||
MACOS='{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal"}'
|
||||
LINUX='{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64"}'
|
||||
WINDOWS='{"platform":"windows-latest","args":"--target x86_64-pc-windows-msvc","name":"windows-x86_64","jpdfium_platforms":"windows-x64"}'
|
||||
MACOS='{"platform":"macos-15","args":"--target universal-apple-darwin","name":"macos-universal","jpdfium_platforms":"darwin-arm64,darwin-x64"}'
|
||||
LINUX='{"platform":"ubuntu-22.04","args":"","name":"linux-x86_64","jpdfium_platforms":"linux-x64"}'
|
||||
|
||||
# Resolve requested platform — populated by either workflow_dispatch
|
||||
# or workflow_call inputs; both paths default to "all".
|
||||
case "$PLATFORM" in
|
||||
windows) ENTRIES=("$WINDOWS") ;;
|
||||
macos) ENTRIES=("$MACOS") ;;
|
||||
@@ -84,9 +86,10 @@ jobs:
|
||||
WINDOWS_CERTIFICATE: ${{ secrets.WINDOWS_CERTIFICATE }}
|
||||
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -112,10 +115,6 @@ jobs:
|
||||
toolchain: stable
|
||||
targets: ${{ matrix.platform == 'macos-15' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
|
||||
|
||||
# x86_64 JDK is set up first so the aarch64 step below can leave its
|
||||
# JAVA_HOME as the active one. The macOS universal JRE build needs
|
||||
# jmods from both arches; the x64 path is captured into the env
|
||||
# before the second setup-java overwrites JAVA_HOME.
|
||||
- name: Set up x86_64 JDK 25 (macOS universal JRE)
|
||||
if: matrix.platform == 'macos-15'
|
||||
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
|
||||
@@ -142,21 +141,21 @@ jobs:
|
||||
- name: Setup Task
|
||||
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
|
||||
|
||||
# Build the universal JRE before desktop:prepare so the jlink:runtime
|
||||
# task short-circuits on its `test -d runtime/jre` status check.
|
||||
- name: Build universal macOS JRE
|
||||
if: matrix.platform == 'macos-15'
|
||||
env:
|
||||
AARCH64_JAVA_HOME: ${{ env.JAVA_HOME }}
|
||||
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
|
||||
run: task desktop:jlink:universal-mac
|
||||
|
||||
- name: Prepare desktop build
|
||||
run: task desktop:prepare
|
||||
env:
|
||||
MAVEN_USER: ${{ secrets.MAVEN_USER }}
|
||||
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
JPDFIUM_PLATFORMS: ${{ matrix.jpdfium_platforms }}
|
||||
run: task desktop:prepare
|
||||
|
||||
# DigiCert KeyLocker Setup (Cloud HSM)
|
||||
- name: Setup DigiCert KeyLocker
|
||||
@@ -269,6 +268,10 @@ jobs:
|
||||
echo "APPLE_SIGNING_IDENTITY=$CERT_ID" >> $GITHUB_ENV
|
||||
echo "Certificate imported successfully."
|
||||
|
||||
- name: Sign JPDFium dylibs inside bootJar (macOS only)
|
||||
if: matrix.platform == 'macos-15' && env.APPLE_CERTIFICATE != ''
|
||||
run: bash frontend/scripts/sign-jpdfium-dylibs-in-bootjar.sh
|
||||
|
||||
- name: Check DMG creation dependencies (macOS only)
|
||||
if: matrix.platform == 'macos-15'
|
||||
run: |
|
||||
@@ -299,7 +302,7 @@ jobs:
|
||||
env:
|
||||
KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }}
|
||||
run: |
|
||||
cat > ./frontend/src-tauri/tauri.windows.conf.json <<EOF
|
||||
cat > ./frontend/editor/src-tauri/tauri.windows.conf.json <<EOF
|
||||
{
|
||||
"bundle": {
|
||||
"windows": {
|
||||
@@ -357,7 +360,7 @@ jobs:
|
||||
SM_CODE_SIGNING_CERT_SHA1_HASH: ${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }}
|
||||
CI: true
|
||||
with:
|
||||
projectPath: ./frontend
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
# Linux: build deb+rpm only here. AppImage runs in its own
|
||||
# continue-on-error step below so its persistent linuxdeploy
|
||||
@@ -382,7 +385,7 @@ jobs:
|
||||
VITE_SAAS_BACKEND_API_URL: ${{ secrets.VITE_SAAS_BACKEND_API_URL || 'https://api.stirlingpdf.com' }}
|
||||
CI: true
|
||||
with:
|
||||
projectPath: ./frontend
|
||||
projectPath: ./frontend/editor
|
||||
tauriScript: npx tauri
|
||||
args: --bundles appimage
|
||||
|
||||
@@ -400,7 +403,7 @@ jobs:
|
||||
if: matrix.platform == 'macos-15'
|
||||
run: |
|
||||
echo "🔍 Verifying notarization status..."
|
||||
cd ./frontend/src-tauri/target
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
DMG_FILE=$(find . -name "*.dmg" | head -1)
|
||||
if [ -n "$DMG_FILE" ]; then
|
||||
echo "Found DMG: $DMG_FILE"
|
||||
@@ -415,7 +418,7 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ./dist
|
||||
cd ./frontend/src-tauri/target
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
# Find and rename artifacts based on platform
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
@@ -505,7 +508,7 @@ jobs:
|
||||
- name: Verify build artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
cd ./frontend/src-tauri/target
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
|
||||
# Check for expected artifacts based on platform
|
||||
if [ "${{ matrix.platform }}" = "windows-latest" ]; then
|
||||
@@ -536,7 +539,7 @@ jobs:
|
||||
- name: Test artifact sizes
|
||||
shell: bash
|
||||
run: |
|
||||
cd ./frontend/src-tauri/target
|
||||
cd ./frontend/editor/src-tauri/target
|
||||
echo "Artifact sizes for ${{ matrix.name }}:"
|
||||
find . -name "*.exe" -o -name "*.dmg" -o -name "*.deb" -o -name "*.rpm" -o -name "*.AppImage" -o -name "*.msi" | while read file; do
|
||||
if [ -f "$file" ]; then
|
||||
@@ -557,7 +560,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -648,7 +651,7 @@ jobs:
|
||||
if: always()
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -12,11 +12,19 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: "false"
|
||||
depot_cores:
|
||||
description: "Depot runner vCPU count (used in runs-on). Override for benchmarking."
|
||||
required: false
|
||||
type: string
|
||||
default: "8"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
# TODO: extract a pre-matrix `prepare` job that runs once and produces
|
||||
# shared artifacts for the three matrix entries below to consume:
|
||||
# 1. `task backend:build` — currently runs 3× in parallel with
|
||||
@@ -32,7 +40,14 @@ jobs:
|
||||
# spring-security=true matrix entry if `task backend:build` and
|
||||
# `task backend:build:ci` produce equivalent JARs (verify before wiring).
|
||||
test-build-docker-images:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' && inputs.docker-base-changed != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -48,7 +63,7 @@ jobs:
|
||||
cache-scope: stirling-pdf-fat
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -105,10 +120,16 @@ jobs:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
@@ -134,7 +155,22 @@ jobs:
|
||||
echo "platforms=linux/amd64,linux/arm64/v8" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build ${{ matrix.docker-rev }}
|
||||
- name: Build ${{ matrix.docker-rev }} (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./${{ matrix.docker-rev }}
|
||||
push: false
|
||||
platforms: ${{ steps.build-params.outputs.platforms }}
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ steps.build-params.outputs.base_image }}
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Build ${{ matrix.docker-rev }} (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
@@ -162,24 +198,52 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
|
||||
test-build-unoserver-image:
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' && inputs.docker-base-changed != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile
|
||||
- name: Build docker/unoserver/Dockerfile (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/unoserver/Dockerfile
|
||||
push: false
|
||||
load: true
|
||||
platforms: linux/amd64
|
||||
tags: stirling-unoserver:pr-test
|
||||
provenance: false
|
||||
sbom: false
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
|
||||
@@ -20,12 +20,22 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pick:
|
||||
uses: ./.github/workflows/_runner-pick.yml
|
||||
|
||||
deploy:
|
||||
if: ${{ vars.CI_PROFILE != 'lite' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
env:
|
||||
USE_DEPOT: ${{ needs.pick.outputs.is_fork != 'true' }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -51,7 +61,12 @@ jobs:
|
||||
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
|
||||
- name: Set up Depot CLI
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.0.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
@@ -66,7 +81,20 @@ jobs:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Build and push test image
|
||||
- name: Build and push test image (Depot)
|
||||
if: env.USE_DEPOT == 'true'
|
||||
uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.16.0
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
context: .
|
||||
file: ./docker/embedded/Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
|
||||
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Build and push test image (Docker fork fallback)
|
||||
if: env.USE_DEPOT != 'true'
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
@@ -125,13 +153,14 @@ jobs:
|
||||
files-changed:
|
||||
if: always()
|
||||
name: detect what files changed
|
||||
runs-on: ubuntu-latest
|
||||
needs: pick
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
timeout-minutes: 3
|
||||
outputs:
|
||||
frontend: ${{ steps.changes.outputs.frontend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -145,12 +174,11 @@ jobs:
|
||||
|
||||
test:
|
||||
if: needs.files-changed.outputs.frontend == 'true'
|
||||
needs: [deploy, files-changed]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
needs: [pick, deploy, files-changed]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -180,13 +208,13 @@ jobs:
|
||||
FORCE_COLOR: "3"
|
||||
|
||||
cleanup:
|
||||
needs: [deploy, test]
|
||||
runs-on: ubuntu-latest
|
||||
needs: [pick, deploy, test]
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
+2
-1
@@ -157,7 +157,7 @@ app/proprietary/build
|
||||
common/build
|
||||
proprietary/build
|
||||
stirling-pdf/build
|
||||
frontend/src-tauri/provisioner/target
|
||||
frontend/editor/src-tauri/provisioner/target
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
@@ -174,6 +174,7 @@ venv.bak/
|
||||
|
||||
# Env files (secrets / local overrides). Subproject .gitignore files whitelist any committed defaults.
|
||||
.env*
|
||||
!.env.saas.example
|
||||
|
||||
# VS Code
|
||||
/.vscode/**/*
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.11
|
||||
rev: v0.15.14
|
||||
hooks:
|
||||
- id: ruff
|
||||
args:
|
||||
@@ -20,7 +20,7 @@ repos:
|
||||
- --skip="./.*,*.csv,*.json,*.ambr"
|
||||
- --quiet-level=2
|
||||
files: \.(html|css|js|py|md)$
|
||||
exclude: (.vscode|.devcontainer|app/core/src/main/resources|app/proprietary/src/main/resources|frontend/public/vendor|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
|
||||
exclude: (.vscode|.devcontainer|app/core/src/main/resources|app/proprietary/src/main/resources|frontend/editor/public/vendor|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.30.0
|
||||
hooks:
|
||||
@@ -38,7 +38,7 @@ repos:
|
||||
rev: v0.24.4
|
||||
hooks:
|
||||
- id: toml-sort-fix
|
||||
files: frontend/public/locales/.*\.toml$
|
||||
files: frontend/editor/public/locales/.*\.toml$
|
||||
args: ['--in-place', '--all', '--ignore-case']
|
||||
# - repo: https://github.com/thibaudcolas/pre-commit-stylelint
|
||||
# rev: v16.21.1
|
||||
|
||||
@@ -39,6 +39,25 @@ tasks:
|
||||
- cmd: ./gradlew clean bootRun -PbuildWithFrontend=true
|
||||
platforms: [linux, darwin]
|
||||
|
||||
dev:saas:
|
||||
desc: "Start backend in SaaS flavor against Supabase (loads .env.saas.local)"
|
||||
# `dotenv:` reads from the root Taskfile's directory (".") because this
|
||||
# subtaskfile is included with `dir: .`. Drop the file at the repo root.
|
||||
dotenv: ['.env.saas.local']
|
||||
ignore_error: true
|
||||
vars:
|
||||
PORT: '{{.PORT | default "8080"}}'
|
||||
# Override to "" to run the pure `saas` profile against your own SAAS_DB_*.
|
||||
PROFILES: '{{.PROFILES | default "dev"}}'
|
||||
env:
|
||||
SERVER_PORT: '{{.PORT}}'
|
||||
STIRLING_FLAVOR: saas
|
||||
cmds:
|
||||
- cmd: cmd /c ".\gradlew.bat :stirling-pdf:bootRun {{if .PROFILES}}--args=\"--spring.profiles.include={{.PROFILES}}\"{{end}}"
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew :stirling-pdf:bootRun {{if .PROFILES}}--args='--spring.profiles.include={{.PROFILES}}'{{end}}
|
||||
platforms: [linux, darwin]
|
||||
|
||||
build:
|
||||
desc: "Full backend build"
|
||||
cmds:
|
||||
|
||||
+39
-11
@@ -3,6 +3,22 @@ version: '3'
|
||||
vars:
|
||||
JLINK_MODULES: "java.base,java.compiler,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,java.xml.crypto,jdk.crypto.ec,jdk.crypto.cryptoki,jdk.unsupported"
|
||||
|
||||
# Override via JPDFIUM_PLATFORMS env (csv of platform keys, or 'all').
|
||||
JPDFIUM_PLATFORMS:
|
||||
sh: |
|
||||
if [ -n "${JPDFIUM_PLATFORMS:-}" ]; then
|
||||
echo "$JPDFIUM_PLATFORMS"
|
||||
else
|
||||
case "{{OS}}-{{ARCH}}" in
|
||||
darwin-arm64) echo "darwin-arm64";;
|
||||
darwin-amd64) echo "darwin-x64";;
|
||||
linux-amd64) echo "linux-x64";;
|
||||
linux-arm64) echo "linux-arm64";;
|
||||
windows-amd64) echo "windows-x64";;
|
||||
*) echo "all";;
|
||||
esac
|
||||
fi
|
||||
|
||||
tasks:
|
||||
prepare:
|
||||
desc: "Prepare desktop build dependencies"
|
||||
@@ -15,6 +31,7 @@ tasks:
|
||||
provisioner:
|
||||
desc: "Build installer provisioner"
|
||||
platforms: [windows]
|
||||
dir: editor
|
||||
cmds:
|
||||
- node scripts/build-provisioner.mjs
|
||||
|
||||
@@ -22,41 +39,48 @@ tasks:
|
||||
desc: "Start Tauri desktop dev mode"
|
||||
deps: [prepare]
|
||||
ignore_error: true
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri dev --no-watch
|
||||
|
||||
build:
|
||||
desc: "Build Tauri desktop app (production)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build
|
||||
|
||||
build:dev:
|
||||
desc: "Build Tauri desktop app (dev, no bundling)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --no-bundle
|
||||
|
||||
build:dev:mac:
|
||||
desc: "Build Tauri desktop .app bundle (macOS)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles app
|
||||
|
||||
build:dev:windows:
|
||||
desc: "Build Tauri desktop NSIS installer (Windows)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles nsis
|
||||
|
||||
build:dev:linux:
|
||||
desc: "Build Tauri desktop AppImage (Linux)"
|
||||
deps: [prepare]
|
||||
dir: editor
|
||||
cmds:
|
||||
- npx tauri build --bundles appimage
|
||||
|
||||
clean:
|
||||
desc: "Clean Tauri/Cargo build artifacts"
|
||||
dir: editor
|
||||
cmds:
|
||||
- task: jlink:clean
|
||||
- cd src-tauri && cargo clean
|
||||
@@ -71,27 +95,29 @@ tasks:
|
||||
deps: [jlink:jar, jlink:runtime]
|
||||
|
||||
jlink:jar:
|
||||
desc: "Build backend JAR for Tauri bundling"
|
||||
desc: "Build backend JAR for Tauri bundling (host-OS natives only by default)"
|
||||
run: once
|
||||
dir: ..
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: "true"
|
||||
cmds:
|
||||
- cmd: cmd /c gradlew.bat bootJar --no-daemon
|
||||
- echo "Building bootJar with JPDFium natives for {{.JPDFIUM_PLATFORMS}}"
|
||||
- cmd: cmd /c gradlew.bat bootJar --no-daemon -PjpdfiumPlatforms={{.JPDFIUM_PLATFORMS}}
|
||||
platforms: [windows]
|
||||
- cmd: ./gradlew bootJar --no-daemon
|
||||
- cmd: ./gradlew bootJar --no-daemon -PjpdfiumPlatforms={{.JPDFIUM_PLATFORMS}}
|
||||
platforms: [linux, darwin]
|
||||
- mkdir -p frontend/src-tauri/libs
|
||||
- cp app/core/build/libs/stirling-pdf-*.jar frontend/src-tauri/libs/
|
||||
- mkdir -p frontend/editor/src-tauri/libs
|
||||
- cp app/core/build/libs/stirling-pdf-*.jar frontend/editor/src-tauri/libs/
|
||||
status:
|
||||
- test -f frontend/src-tauri/libs/stirling-pdf-*.jar
|
||||
- test -f frontend/editor/src-tauri/libs/stirling-pdf-*.jar
|
||||
|
||||
jlink:runtime:
|
||||
desc: "Create custom JRE with jlink"
|
||||
deps: [jlink:jar]
|
||||
dir: editor/src-tauri
|
||||
cmds:
|
||||
- rm -rf src-tauri/runtime/jre
|
||||
- mkdir -p src-tauri/runtime
|
||||
- rm -rf runtime/jre
|
||||
- mkdir -p runtime
|
||||
- >-
|
||||
jlink
|
||||
--add-modules {{.JLINK_MODULES}}
|
||||
@@ -99,14 +125,15 @@ tasks:
|
||||
--compress=zip-6
|
||||
--no-header-files
|
||||
--no-man-pages
|
||||
--output src-tauri/runtime/jre
|
||||
--output runtime/jre
|
||||
status:
|
||||
- test -d src-tauri/runtime/jre
|
||||
- test -d editor/src-tauri/runtime/jre
|
||||
|
||||
jlink:clean:
|
||||
desc: "Remove JLink runtime and bundled JARs"
|
||||
dir: editor/src-tauri
|
||||
cmds:
|
||||
- rm -rf src-tauri/libs src-tauri/runtime
|
||||
- rm -rf libs runtime
|
||||
|
||||
# macOS-only. Replaces jlink:runtime's single-arch JRE with a universal
|
||||
# (arm64 + x86_64) one for the universal Tauri shell. Runs the x86_64
|
||||
@@ -117,6 +144,7 @@ tasks:
|
||||
desc: "Create universal (arm64+x86_64) JRE for the macOS Tauri build"
|
||||
deps: [jlink:jar]
|
||||
platforms: [darwin]
|
||||
dir: editor
|
||||
env:
|
||||
JLINK_MODULES: "{{.JLINK_MODULES}}"
|
||||
OUTPUT_DIR: src-tauri/runtime/jre
|
||||
|
||||
+5
-5
@@ -3,14 +3,14 @@ version: '3'
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install Playwright browsers"
|
||||
dir: frontend
|
||||
dir: frontend/editor
|
||||
deps: [ ':frontend:install' ]
|
||||
cmds:
|
||||
- npx playwright install {{.CLI_ARGS}} --with-deps
|
||||
|
||||
stubbed:
|
||||
desc: "Run stubbed E2E tests"
|
||||
dir: frontend
|
||||
dir: frontend/editor
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed {{.CLI_ARGS}}
|
||||
@@ -72,7 +72,7 @@ tasks:
|
||||
live:runner:
|
||||
internal: true
|
||||
deps: [ ':frontend:prepare' ]
|
||||
dir: frontend
|
||||
dir: frontend/editor
|
||||
vars:
|
||||
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
|
||||
cmds:
|
||||
@@ -141,14 +141,14 @@ tasks:
|
||||
up first with:
|
||||
task e2e:oauth:up
|
||||
task e2e:saml:up
|
||||
dir: frontend
|
||||
dir: frontend/editor
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=enterprise {{.CLI_ARGS}}
|
||||
|
||||
cross-browser:
|
||||
desc: "Run stubbed E2E tests on Chromium, Firefox, and WebKit"
|
||||
dir: frontend
|
||||
dir: frontend/editor
|
||||
deps: [ ':frontend:prepare' ]
|
||||
cmds:
|
||||
- npx playwright test --project=stubbed --project=stubbed-firefox --project=stubbed-webkit {{.CLI_ARGS}}
|
||||
|
||||
+28
-23
@@ -1,5 +1,10 @@
|
||||
version: '3'
|
||||
|
||||
# Tasks operate from the workspace root (frontend/). Editor commands pass
|
||||
# `editor` as the vite project root (positional after `build` / before the
|
||||
# mode flag) or use `--project editor/...` for tsc — so the editor lives
|
||||
# under frontend/editor/ without each task needing a cd.
|
||||
|
||||
tasks:
|
||||
install:
|
||||
desc: "Install dependencies"
|
||||
@@ -21,19 +26,19 @@ tasks:
|
||||
vars:
|
||||
MODE: '{{.MODE | default ""}}'
|
||||
cmds:
|
||||
- npx tsx scripts/setup-env.ts{{if .MODE}} --{{.MODE}}{{end}}
|
||||
- npx tsx editor/scripts/setup-env.mts{{if .MODE}} --{{.MODE}}{{end}}
|
||||
sources:
|
||||
- scripts/setup-env.ts
|
||||
- editor/scripts/setup-env.mts
|
||||
generates:
|
||||
- .env.local
|
||||
- .env{{if .MODE}}.{{.MODE}}{{end}}.local
|
||||
- editor/.env.local
|
||||
- editor/.env{{if .MODE}}.{{.MODE}}{{end}}.local
|
||||
|
||||
prepare:icons:
|
||||
internal: true
|
||||
run: once
|
||||
deps: [install]
|
||||
cmds:
|
||||
- node scripts/generate-icons.js
|
||||
- node editor/scripts/generate-icons.js
|
||||
|
||||
prepare:
|
||||
desc: "Set up dev environment"
|
||||
@@ -60,7 +65,7 @@ tasks:
|
||||
env:
|
||||
BACKEND_URL: '{{.BACKEND_URL}}'
|
||||
cmds:
|
||||
- npx vite --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
|
||||
- npx vite editor --mode {{.MODE}} --port {{.PORT}}{{if .OPEN}} --open{{end}}
|
||||
|
||||
dev:
|
||||
desc: "Start frontend dev server"
|
||||
@@ -115,19 +120,19 @@ tasks:
|
||||
desc: "Production build (default mode)"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vite build
|
||||
- npx vite build editor
|
||||
|
||||
build:core:
|
||||
desc: "Build for core mode"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vite build --mode core
|
||||
- npx vite build editor --mode core
|
||||
|
||||
build:proprietary:
|
||||
desc: "Build for proprietary mode"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vite build --mode proprietary
|
||||
- npx vite build editor --mode proprietary
|
||||
|
||||
build:saas:
|
||||
desc: "Build for SaaS mode"
|
||||
@@ -135,7 +140,7 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: saas }
|
||||
cmds:
|
||||
- npx vite build --mode saas
|
||||
- npx vite build editor --mode saas
|
||||
|
||||
build:desktop:
|
||||
desc: "Build for desktop mode"
|
||||
@@ -143,13 +148,13 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: desktop }
|
||||
cmds:
|
||||
- npx vite build --mode desktop
|
||||
- npx vite build editor --mode desktop
|
||||
|
||||
build:prototypes:
|
||||
desc: "Build for prototypes mode"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx vite build --mode prototypes
|
||||
- npx vite build editor --mode prototypes
|
||||
|
||||
# ============================================================
|
||||
# Code quality
|
||||
@@ -160,7 +165,7 @@ tasks:
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx eslint --max-warnings=0
|
||||
- npx dpdm src --circular --no-warning --no-tree --exit-code circular:1
|
||||
- npx dpdm editor/src --circular --no-warning --no-tree --exit-code circular:1
|
||||
|
||||
lint:fix:
|
||||
desc: "Auto-fix lint issues"
|
||||
@@ -195,13 +200,13 @@ tasks:
|
||||
desc: "Typecheck core build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/core/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/src/core/tsconfig.json
|
||||
|
||||
typecheck:proprietary:
|
||||
desc: "Typecheck proprietary build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/proprietary/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/src/proprietary/tsconfig.json
|
||||
|
||||
typecheck:saas:
|
||||
desc: "Typecheck SaaS build variant"
|
||||
@@ -209,7 +214,7 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: saas }
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/saas/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/src/saas/tsconfig.json
|
||||
|
||||
typecheck:desktop:
|
||||
desc: "Typecheck desktop build variant"
|
||||
@@ -217,19 +222,19 @@ tasks:
|
||||
- task: prepare
|
||||
vars: { MODE: desktop }
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/desktop/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/src/desktop/tsconfig.json
|
||||
|
||||
typecheck:scripts:
|
||||
desc: "Typecheck scripts"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project scripts/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/scripts/tsconfig.json
|
||||
|
||||
typecheck:prototypes:
|
||||
desc: "Typecheck prototypes build variant"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- npx tsc --noEmit --project src/prototypes/tsconfig.json
|
||||
- npx tsc --noEmit --project editor/src/prototypes/tsconfig.json
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
@@ -270,19 +275,19 @@ tasks:
|
||||
desc: "Run tests"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest run
|
||||
- npx vitest run --root editor
|
||||
|
||||
test:watch:
|
||||
desc: "Run tests in watch mode"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest --watch
|
||||
- npx vitest --watch --root editor
|
||||
|
||||
test:coverage:
|
||||
desc: "Run tests with coverage"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- npx vitest --coverage
|
||||
- npx vitest --coverage --root editor
|
||||
|
||||
# ============================================================
|
||||
# Code Generation
|
||||
@@ -292,4 +297,4 @@ tasks:
|
||||
desc: "Generate frontend license report"
|
||||
deps: [install]
|
||||
cmds:
|
||||
- node scripts/generate-licenses.js
|
||||
- node editor/scripts/generate-licenses.js
|
||||
|
||||
+6
-6
@@ -10,14 +10,14 @@ When adding tools, follow this systematic approach using the established pattern
|
||||
|
||||
Create these files in the correct directories:
|
||||
```
|
||||
frontend/src/hooks/tools/[toolName]/
|
||||
frontend/editor/src/hooks/tools/[toolName]/
|
||||
├── use[ToolName]Parameters.ts # Parameter definitions and validation
|
||||
└── use[ToolName]Operation.ts # Tool operation logic using useToolOperation
|
||||
|
||||
frontend/src/components/tools/[toolName]/
|
||||
frontend/editor/src/components/tools/[toolName]/
|
||||
└── [ToolName]Settings.tsx # Settings UI component (if needed)
|
||||
|
||||
frontend/src/tools/
|
||||
frontend/editor/src/tools/
|
||||
└── [ToolName].tsx # Main tool component
|
||||
```
|
||||
|
||||
@@ -128,7 +128,7 @@ export default [ToolName] as ToolComponent;
|
||||
## 3. Register Tool in System
|
||||
Update these files to register your new tool:
|
||||
|
||||
**Tool Registry** (`frontend/src/data/useTranslatedToolRegistry.tsx`):
|
||||
**Tool Registry** (`frontend/editor/src/data/useTranslatedToolRegistry.tsx`):
|
||||
1. Add imports at the top:
|
||||
```typescript
|
||||
import [ToolName] from "../tools/[ToolName]";
|
||||
@@ -155,7 +155,7 @@ import [ToolName]Settings from "../components/tools/[toolName]/[ToolName]Setting
|
||||
## 4. Add Tooltips (Optional but Recommended)
|
||||
Create user-friendly tooltips to help non-technical users understand your tool. **Use simple, clear language - avoid technical jargon:**
|
||||
|
||||
**Tooltip Hook** (`frontend/src/components/tooltips/use[ToolName]Tips.ts`):
|
||||
**Tooltip Hook** (`frontend/editor/src/components/tooltips/use[ToolName]Tips.ts`):
|
||||
```typescript
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { TooltipContent } from '../../types/tips';
|
||||
@@ -202,7 +202,7 @@ const [ToolName] = (props: BaseToolProps) => {
|
||||
## 5. Add Translations
|
||||
Update translation files. **Important: Only update `en-GB` files** - other languages are handled separately.
|
||||
|
||||
**File to update:** `frontend/public/locales/en-GB/translation.toml`
|
||||
**File to update:** `frontend/editor/public/locales/en-GB/translation.toml`
|
||||
|
||||
**Required Translation Keys**:
|
||||
```toml
|
||||
|
||||
@@ -139,9 +139,9 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
|
||||
#### Environment Variables
|
||||
- All `VITE_*` variables must be declared in the appropriate committed env file:
|
||||
- `frontend/.env` — core, proprietary, and shared vars
|
||||
- `frontend/.env.saas` — SaaS-only vars (layered on top of `.env` in SaaS mode)
|
||||
- `frontend/.env.desktop` — desktop (Tauri)-only vars (layered on top of `.env` in desktop mode)
|
||||
- `frontend/editor/.env` — core, proprietary, and shared vars
|
||||
- `frontend/editor/.env.saas` — SaaS-only vars (layered on top of `.env` in SaaS mode)
|
||||
- `frontend/editor/.env.desktop` — desktop (Tauri)-only vars (layered on top of `.env` in desktop mode)
|
||||
- These files are committed to Git and must not contain private keys
|
||||
- Local overrides (API keys, machine-specific settings) go in uncommitted sibling `.env.local` / `.env.saas.local` / `.env.desktop.local` files — Vite automatically layers them on top
|
||||
- Never use `|| 'hardcoded-fallback'` inline — put defaults in the committed env files
|
||||
@@ -152,7 +152,7 @@ The project structure is defined in `engine/pyproject.toml`. Any new dependencie
|
||||
#### Import Paths - CRITICAL
|
||||
**ALWAYS use `@app/*` for imports.** Do not use `@core/*` or `@proprietary/*` unless explicitly wrapping/extending a lower layer implementation.
|
||||
|
||||
For a broader explanation of the frontend layering and override architecture, see [frontend/DeveloperGuide.md](frontend/DeveloperGuide.md).
|
||||
For a broader explanation of the frontend layering and override architecture, see [frontend/editor/DeveloperGuide.md](frontend/editor/DeveloperGuide.md).
|
||||
|
||||
```typescript
|
||||
// ✅ CORRECT - Use @app/* for all imports
|
||||
@@ -243,7 +243,7 @@ Frontend designed for **stateful document processing**:
|
||||
- No file reloading between tools - performance critical for large PDFs (up to 100GB+)
|
||||
|
||||
#### FileContext - Central State Management
|
||||
**Location**: `frontend/src/core/contexts/FileContext.tsx`
|
||||
**Location**: `frontend/editor/src/core/contexts/FileContext.tsx`
|
||||
- **Active files**: Currently loaded PDFs and their variants
|
||||
- **Tool navigation**: Current mode (viewer/pageEditor/fileEditor/toolName)
|
||||
- **Memory management**: PDF document cleanup, blob URL lifecycle, Web Worker management
|
||||
@@ -268,7 +268,7 @@ Without cleanup: browser crashes with memory leaks.
|
||||
|
||||
**Architecture**: Modular hook-based system with clear separation of concerns:
|
||||
|
||||
- **useToolOperation** (`frontend/src/core/hooks/tools/shared/useToolOperation.ts`): Main orchestrator hook
|
||||
- **useToolOperation** (`frontend/editor/src/core/hooks/tools/shared/useToolOperation.ts`): Main orchestrator hook
|
||||
- Coordinates all tool operations with consistent interface
|
||||
- Integrates with FileContext for operation tracking
|
||||
- Handles validation, error handling, and UI state management
|
||||
@@ -356,7 +356,7 @@ return useToolOperation({
|
||||
### Frontend Directory Structure
|
||||
The frontend is organized with a clear separation of concerns:
|
||||
|
||||
- **`frontend/src/core/`**: Main application code (shared, production-ready components)
|
||||
- **`frontend/editor/src/core/`**: Main application code (shared, production-ready components)
|
||||
- **`core/components/`**: React components organized by feature
|
||||
- `core/components/tools/`: Individual PDF tool implementations
|
||||
- `core/components/viewer/`: PDF viewer components
|
||||
@@ -374,17 +374,17 @@ The frontend is organized with a clear separation of concerns:
|
||||
- **`core/data/`**: Static data (tool taxonomy, etc.)
|
||||
- **`core/services/`**: Business logic services (PDF processing, storage, etc.)
|
||||
|
||||
- **`frontend/src/desktop/`**: Desktop-specific (Tauri) code
|
||||
- **`frontend/src/proprietary/`**: Proprietary/licensed features
|
||||
- **`frontend/src-tauri/`**: Tauri (Rust) native desktop application code
|
||||
- **`frontend/public/`**: Static assets served directly
|
||||
- **`frontend/editor/src/desktop/`**: Desktop-specific (Tauri) code
|
||||
- **`frontend/editor/src/proprietary/`**: Proprietary/licensed features
|
||||
- **`frontend/editor/src-tauri/`**: Tauri (Rust) native desktop application code
|
||||
- **`frontend/editor/public/`**: Static assets served directly
|
||||
- `public/locales/`: Translation JSON files
|
||||
|
||||
### Component Architecture
|
||||
- **Static Assets**: CSS, JS, and resources in `src/main/resources/static/` (legacy) + `frontend/public/` (modern)
|
||||
- **Static Assets**: CSS, JS, and resources in `src/main/resources/static/` (legacy) + `frontend/editor/public/` (modern)
|
||||
- **Internationalization**:
|
||||
- Backend: `messages_*.properties` files
|
||||
- Frontend: JSON files in `frontend/public/locales/` (converted from .properties)
|
||||
- Frontend: JSON files in `frontend/editor/public/locales/` (converted from .properties)
|
||||
- Conversion Script: `scripts/convert_properties_to_json.py`
|
||||
|
||||
### Configuration Modes
|
||||
@@ -409,7 +409,7 @@ The frontend is organized with a clear separation of concerns:
|
||||
4. **Code Style**: Spotless enforces Google Java Format automatically (`task backend:format`)
|
||||
5. **Translations**:
|
||||
- Backend: Use helper scripts in `/scripts` for multi-language updates
|
||||
- Frontend: Update JSON files in `frontend/public/locales/` or use conversion script
|
||||
- Frontend: Update JSON files in `frontend/editor/public/locales/` or use conversion script
|
||||
6. **Documentation**: API docs auto-generated and available at `/swagger-ui/index.html`
|
||||
|
||||
## Frontend Architecture Status
|
||||
@@ -427,11 +427,11 @@ The frontend is organized with a clear separation of concerns:
|
||||
## Translation Rules
|
||||
|
||||
- **CRITICAL**: Always update translations in `en-GB` only, never `en-US`
|
||||
- Translation files are located in `frontend/public/locales/`
|
||||
- Translation files are located in `frontend/editor/public/locales/`
|
||||
|
||||
## Important Notes
|
||||
|
||||
- **Java Version**: Minimum JDK 21, supports and recommends JDK 25
|
||||
- **Java Version**: Requires JDK 25.
|
||||
- **Lombok**: Used extensively - ensure IDE plugin is installed
|
||||
- **File Persistence**:
|
||||
- **Backend**: Designed to be stateless - files are processed in memory/temp locations only
|
||||
@@ -458,3 +458,26 @@ The frontend is organized with a clear separation of concerns:
|
||||
- Confirm approach before making structural changes
|
||||
- Request guidance on preferences (cross-platform vs specific tools, etc.)
|
||||
- Verify understanding of requirements before proceeding
|
||||
|
||||
|
||||
## Stack reality check (don't trust LLM training data) <!-- bleeding-edge-stack-note -->
|
||||
|
||||
This codebase is on bleeding-edge versions of its core JVM stack: **Spring Boot 4.0.6**,
|
||||
**Jackson 3 (`tools.jackson`)**, **JDK 21/25 source/target with JDK 25 toolchain**.
|
||||
All three are *post*-2024 releases and your training corpus is overwhelmingly Spring Boot 2/3 and
|
||||
Jackson 2 patterns — those patterns will compile, run differently, or hallucinate APIs that no
|
||||
longer exist.
|
||||
|
||||
Before writing or editing Spring / Jackson / JDK code:
|
||||
|
||||
1. Open an existing module in `app/core/` or `app/common/` and grep for the actual imports being
|
||||
used — `import tools.jackson...` not `import com.fasterxml.jackson...`, and the new
|
||||
`org.springframework.boot` 4.x package layout.
|
||||
2. If you're not sure whether an API exists in this stack version, **check the source on disk
|
||||
first** (the dependency JARs are downloaded under `~/.gradle/caches/modules-2/`).
|
||||
3. Do not silently downgrade a Spring Boot 4 pattern to a Spring Boot 3 equivalent. If something
|
||||
doesn't work, surface it to the human — don't guess.
|
||||
|
||||
Same goes for Jackson 3's API surface (renamed `ObjectMapper` builder methods, new
|
||||
`tools.jackson.databind` namespace) and JDK 25 preview features. Ground your code in this repo's
|
||||
actual imports, not what worked three years ago.
|
||||
|
||||
+24
-22
@@ -11,7 +11,7 @@ This guide focuses on developing for Stirling 2.0, including both the React fron
|
||||
**Stirling 2.0** is built using:
|
||||
|
||||
**Backend:**
|
||||
- Spring Boot (Java 21+, JDK 25 recommended)
|
||||
- Spring Boot (requires JDK 25)
|
||||
- PDFBox for core PDF operations
|
||||
- LibreOffice for document conversions
|
||||
- qpdf for PDF optimization
|
||||
@@ -45,7 +45,7 @@ This guide focuses on developing for Stirling 2.0, including both the React fron
|
||||
- [Task](https://taskfile.dev/installation/) — unified command runner (recommended)
|
||||
- Docker
|
||||
- Git
|
||||
- Java JDK 21 or later (JDK 25 recommended)
|
||||
- Java JDK 25
|
||||
- Node.js 18+ and npm (required for frontend development)
|
||||
- Gradle 7.0 or later (Included within the repo)
|
||||
- [uv](https://docs.astral.sh/uv/) — Python package manager (required for engine development)
|
||||
@@ -61,7 +61,7 @@ This guide focuses on developing for Stirling 2.0, including both the React fron
|
||||
cd Stirling-PDF
|
||||
```
|
||||
|
||||
2. Install Docker and JDK 21 (or JDK 25 recommended) if not already installed.
|
||||
2. Install Docker and JDK 25 if not already installed.
|
||||
|
||||
3. Install a recommended Java IDE such as Eclipse, IntelliJ, or VSCode
|
||||
1. Only VSCode
|
||||
@@ -128,24 +128,26 @@ See [the frontend README](frontend/README.md#tauri) for detailed build instructi
|
||||
Stirling-PDF/
|
||||
├── .github/ # GitHub-specific files (workflows, issue templates)
|
||||
├── configs/ # Configuration files used by stirling at runtime (generated at runtime)
|
||||
├── frontend/ # React SPA frontend (Stirling 2.0)
|
||||
│ ├── src/
|
||||
│ │ ├── components/ # React components
|
||||
│ │ ├── tools/ # Tool-specific React components
|
||||
│ │ ├── hooks/ # Custom React hooks
|
||||
│ │ ├── services/ # API and utility services
|
||||
│ │ ├── types/ # TypeScript type definitions
|
||||
│ │ └── utils/ # Utility functions
|
||||
│ ├── src-tauri/ # Tauri desktop app configuration
|
||||
│ │ ├── src/ # Rust backend code
|
||||
│ │ ├── libs/ # JAR files (generated by build scripts)
|
||||
│ │ ├── runtime/ # Bundled JRE (generated by build scripts)
|
||||
│ │ ├── Cargo.toml # Rust dependencies
|
||||
│ │ └── tauri.conf.json # Tauri configuration
|
||||
│ ├── public/
|
||||
│ │ └── locales/ # Internationalization files (JSON)
|
||||
│ ├── package.json # Frontend dependencies
|
||||
│ └── vite.config.ts # Vite configuration
|
||||
├── frontend/ # Frontend workspace (Stirling 2.0)
|
||||
│ ├── editor/ # PDF editor app (the original React SPA)
|
||||
│ │ ├── src/
|
||||
│ │ │ ├── components/ # React components
|
||||
│ │ │ ├── tools/ # Tool-specific React components
|
||||
│ │ │ ├── hooks/ # Custom React hooks
|
||||
│ │ │ ├── services/ # API and utility services
|
||||
│ │ │ ├── types/ # TypeScript type definitions
|
||||
│ │ │ └── utils/ # Utility functions
|
||||
│ │ ├── src-tauri/ # Tauri desktop app configuration
|
||||
│ │ │ ├── src/ # Rust backend code
|
||||
│ │ │ ├── libs/ # JAR files (generated by build scripts)
|
||||
│ │ │ ├── runtime/ # Bundled JRE (generated by build scripts)
|
||||
│ │ │ ├── Cargo.toml # Rust dependencies
|
||||
│ │ │ └── tauri.conf.json # Tauri configuration
|
||||
│ │ ├── public/
|
||||
│ │ │ └── locales/ # Internationalization files (JSON)
|
||||
│ │ └── vite.config.ts # Vite configuration
|
||||
│ ├── package.json # Shared workspace dependencies
|
||||
│ └── eslint.config.mjs # Shared lint config
|
||||
├── customFiles/ # Custom static files and templates (generated at runtime used to replace existing files)
|
||||
├── docs/ # Documentation files
|
||||
├── exampleYmlFiles/ # Example YAML configuration files
|
||||
@@ -445,7 +447,7 @@ For Stirling 2.0, new features are built as React components:
|
||||
|
||||
1. **Create the React Component:**
|
||||
```typescript
|
||||
// frontend/src/tools/NewTool.tsx
|
||||
// frontend/editor/src/tools/NewTool.tsx
|
||||
import { useState } from 'react';
|
||||
import { Button, FileInput, Container } from '@mantine/core';
|
||||
|
||||
|
||||
+1
-1
@@ -86,7 +86,7 @@ If you're using Tauri's built-in updater feature:
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### 1. Tauri Configuration (frontend/src-tauri/tauri.conf.json)
|
||||
### 1. Tauri Configuration (frontend/editor/src-tauri/tauri.conf.json)
|
||||
|
||||
The Windows signing configuration is already set up:
|
||||
|
||||
|
||||
@@ -60,6 +60,27 @@ dependencies {
|
||||
exclude group: 'com.google.code.gson', module: 'gson'
|
||||
}
|
||||
|
||||
api 'com.stirling:jpdfium:1.0.1'
|
||||
|
||||
// -PjpdfiumPlatforms=all|<csv of linux-x64,linux-arm64,darwin-x64,darwin-arm64,windows-x64>
|
||||
def jpdfiumPlatformsProp = (project.findProperty('jpdfiumPlatforms') ?: 'all').toString().trim()
|
||||
def jpdfiumAllPlatforms = ['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'windows-x64']
|
||||
def jpdfiumPlatforms = jpdfiumPlatformsProp == 'all'
|
||||
? jpdfiumAllPlatforms
|
||||
: jpdfiumPlatformsProp.split(',').collect { it.trim() }.findAll { it }
|
||||
def jpdfiumInvalid = jpdfiumPlatforms.findAll { !jpdfiumAllPlatforms.contains(it) }
|
||||
if (jpdfiumInvalid) {
|
||||
throw new GradleException("Unknown jpdfiumPlatforms value(s): ${jpdfiumInvalid.join(', ')}. " +
|
||||
"Valid: ${jpdfiumAllPlatforms.join(', ')} or 'all'.")
|
||||
}
|
||||
logger.lifecycle("JPDFium native platforms: ${jpdfiumPlatforms.join(', ')}")
|
||||
jpdfiumPlatforms.each { platform ->
|
||||
runtimeOnly "com.stirling:jpdfium-natives-${platform}:1.0.1"
|
||||
}
|
||||
|
||||
// Bucket4j (local in-process token bucket for RateLimitStore default impl)
|
||||
implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
|
||||
|
||||
// ArchUnit: enforces module dependency direction (see ArchitectureTest)
|
||||
testImplementation 'com.tngtech.archunit:archunit-junit5:1.4.2'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
/** Health and identity facade for the active cluster backplane. */
|
||||
public interface ClusterBackplane {
|
||||
|
||||
/** Returns {@code true} when the backplane is reachable; used for health endpoints. */
|
||||
boolean isHealthy();
|
||||
|
||||
/** Returns {@code "inprocess"} or {@code "valkey"}. */
|
||||
String backplaneType();
|
||||
|
||||
/** Returns this JVM's stable node id (matches {@code Cluster.resolvedNodeId()}). */
|
||||
String localNodeId();
|
||||
|
||||
/**
|
||||
* Whether this JVM should run the local {@link
|
||||
* stirling.software.common.service.TaskManager#cleanupOldJobs()} loop. Distributed backplanes
|
||||
* own job expiry via their own TTL, so they should override this to return {@code false}.
|
||||
* Defaults to {@code true} so in-process behavior is preserved without an explicit override.
|
||||
*/
|
||||
default boolean shouldRunLocalCleanup() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
/**
|
||||
* Validates that cluster mode is internally consistent.
|
||||
*
|
||||
* <p>Cluster settings are bound on the central {@link ApplicationProperties} under {@code
|
||||
* cluster.*}; this class reads {@link ApplicationProperties#getCluster()} and runs guards in {@link
|
||||
* PostConstruct}. When {@code cluster.enabled=false} (the default) all checks are skipped so a
|
||||
* single-instance install needs no new config.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class ClusterConfig {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@PostConstruct
|
||||
void validate() {
|
||||
Cluster cluster = applicationProperties.getCluster();
|
||||
if (!cluster.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
String backplane = cluster.getBackplane();
|
||||
if ("valkey".equalsIgnoreCase(backplane)) {
|
||||
String url = cluster.getValkey() == null ? null : cluster.getValkey().getUrl();
|
||||
if (url == null || url.isBlank()) {
|
||||
throw new IllegalStateException(
|
||||
"cluster.enabled=true with backplane=valkey requires"
|
||||
+ " cluster.valkey.url to be set (e.g."
|
||||
+ " redis://valkey:6379).");
|
||||
}
|
||||
} else if ("inprocess".equalsIgnoreCase(backplane)) {
|
||||
// enabled+inprocess only coordinates the local JVM; cross-node lookups will 410.
|
||||
log.warn(
|
||||
"cluster.enabled=true with backplane=inprocess - only the local"
|
||||
+ " JVM is coordinated. Cross-node lookups and the file proxy will fail."
|
||||
+ " Use backplane=valkey for real multi-node deployments.");
|
||||
} else {
|
||||
// Fail fast on typos like "valky" so Spring doesn't later report a cryptic
|
||||
// "no ClusterBackplane bean" - the operator-facing error names the bad value.
|
||||
throw new IllegalStateException(
|
||||
"cluster.enabled=true with unknown backplane '"
|
||||
+ backplane
|
||||
+ "'. Valid values: inprocess | valkey.");
|
||||
}
|
||||
log.info(
|
||||
"Cluster mode enabled (backplane={}, nodeRole={}, nodeId={}).",
|
||||
backplane,
|
||||
cluster.resolvedRole(),
|
||||
cluster.resolvedNodeId());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Snapshot of a peer node as recorded in the {@link InstanceRegistry}.
|
||||
*
|
||||
* @param internalAddress {@code host:port} the node listens on for {@code /internal/cluster/**}
|
||||
* @param role one of {@code WEB}, {@code WORKER}, {@code BOTH}
|
||||
*/
|
||||
public record ClusterNode(
|
||||
String nodeId, String internalAddress, Instant lastHeartbeat, String role) {}
|
||||
@@ -0,0 +1,21 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Cluster-wide mutual exclusion primitive; non-reentrant by contract. */
|
||||
public interface DistributedLock {
|
||||
|
||||
Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime);
|
||||
|
||||
interface LockHandle extends AutoCloseable {
|
||||
void release();
|
||||
|
||||
boolean renew(Duration leaseTime);
|
||||
|
||||
@Override
|
||||
default void close() {
|
||||
release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/** Low-level storage seam for result/job files. */
|
||||
public interface FileStore {
|
||||
|
||||
/** Stored file record. */
|
||||
record Stored(String fileId, long size) {}
|
||||
|
||||
/** Store the given stream and return a generated file id and total bytes written. */
|
||||
Stored store(InputStream in, String originalName) throws IOException;
|
||||
|
||||
/**
|
||||
* Store the file at {@code source} and return a generated file id and total bytes written.
|
||||
*
|
||||
* <p>Default implementation opens {@code source} as a stream and delegates to {@link
|
||||
* #store(InputStream, String)}. Local-disk implementations should override to use a direct
|
||||
* file-to-file copy ({@code Files.copy(source, dest)} can use {@code sendfile(2)} on Linux),
|
||||
* which avoids the two-memory-copy hit of streaming a disk-backed upload through the JVM heap.
|
||||
*/
|
||||
default Stored store(Path source, String originalName) throws IOException {
|
||||
try (InputStream in = Files.newInputStream(source)) {
|
||||
return store(in, originalName);
|
||||
}
|
||||
}
|
||||
|
||||
/** Open the stored file for streaming reads. Caller closes. */
|
||||
InputStream retrieve(String fileId) throws IOException;
|
||||
|
||||
/** Load the stored file into a byte array. */
|
||||
byte[] retrieveBytes(String fileId) throws IOException;
|
||||
|
||||
/** Size of the stored file in bytes. */
|
||||
long size(String fileId) throws IOException;
|
||||
|
||||
/** Delete the stored file. Returns true if a file was removed. */
|
||||
boolean delete(String fileId);
|
||||
|
||||
/** Whether the file id exists in the store. */
|
||||
boolean exists(String fileId);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Maps {@code nodeId} to its internal cluster address, with TTL'd heartbeats. */
|
||||
public interface InstanceRegistry {
|
||||
|
||||
/** Register or refresh this node. Idempotent so a wiped backplane self-heals on next tick. */
|
||||
void register(ClusterNode node, Duration heartbeatTtl);
|
||||
|
||||
Optional<ClusterNode> lookup(String nodeId);
|
||||
|
||||
Collection<ClusterNode> activeNodes();
|
||||
|
||||
void deregister(String nodeId);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Cluster-visible storage for job status and result metadata, with TTL'd entries. */
|
||||
public interface JobStore {
|
||||
|
||||
/** Persist or overwrite a job entry. {@code ttl} sets the lifetime of the entry. */
|
||||
void put(JobStoreEntry entry, Duration ttl);
|
||||
|
||||
Optional<JobStoreEntry> get(String jobId);
|
||||
|
||||
void delete(String jobId);
|
||||
|
||||
boolean exists(String jobId);
|
||||
|
||||
/** Reverse lookup: which job owns this result file id? */
|
||||
Optional<String> findJobIdByFileId(String fileId);
|
||||
|
||||
/** Snapshot of every active entry. Used by admin/stats endpoints; may be O(n). */
|
||||
Collection<JobStoreEntry> all();
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Cluster-visible projection of a job's status and result metadata, as persisted in {@link
|
||||
* JobStore}.
|
||||
*
|
||||
* @param owningNodeId the node id that originally executed the job
|
||||
*/
|
||||
public record JobStoreEntry(
|
||||
String jobId,
|
||||
JobState state,
|
||||
String owningNodeId,
|
||||
Instant createdAt,
|
||||
Instant completedAt,
|
||||
String error,
|
||||
List<String> fileIds,
|
||||
Map<String, String> resultMeta) {
|
||||
|
||||
/** Lifecycle states for a job as observed by the cluster. */
|
||||
public enum JobState {
|
||||
PENDING,
|
||||
RUNNING,
|
||||
COMPLETE,
|
||||
FAILED
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Short-TTL namespaced key/value cache backed by the cluster backplane. */
|
||||
public interface KeyValueCache {
|
||||
|
||||
void put(String namespace, String key, String value, Duration ttl);
|
||||
|
||||
Optional<String> get(String namespace, String key);
|
||||
|
||||
void evict(String namespace, String key);
|
||||
|
||||
void evictNamespace(String namespace);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** Token-bucket rate limiting backed by the cluster backplane. */
|
||||
public interface RateLimitStore {
|
||||
|
||||
/**
|
||||
* Attempt to consume one token from the bucket identified by {@code bucketKey}.
|
||||
*
|
||||
* @param bucketKey opaque key identifying the bucket (e.g. {@code api:user:123})
|
||||
* @param capacity bucket capacity
|
||||
* @param refillPeriod time window over which {@code capacity} tokens refill
|
||||
*/
|
||||
RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod);
|
||||
|
||||
record RateLimitDecision(boolean allowed, long remainingTokens, long nanosToWaitForRefill) {}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
/** Records one increment per sticky-session miss (a 410 Gone for a job owned by another node). */
|
||||
@FunctionalInterface
|
||||
public interface StickyMissRecorder {
|
||||
void recordStickyMiss();
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
@Slf4j
|
||||
public class InProcessClusterBackplane implements ClusterBackplane {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
public InProcessClusterBackplane(ApplicationProperties applicationProperties) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isHealthy() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String backplaneType() {
|
||||
return "inprocess";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String localNodeId() {
|
||||
return applicationProperties.getCluster().resolvedNodeId();
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
import stirling.software.common.cluster.InstanceRegistry;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
/**
|
||||
* Default cluster backplane wiring: every interface gets an {@code InProcess*} bean. Active when
|
||||
* cluster mode is off or {@code cluster.backplane=inprocess}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@ConditionalOnExpression(
|
||||
"!${cluster.enabled:false} ||"
|
||||
+ " '${cluster.backplane:inprocess}'.equalsIgnoreCase('inprocess')")
|
||||
public class InProcessClusterConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public ClusterBackplane clusterBackplane(ApplicationProperties applicationProperties) {
|
||||
log.info("Cluster backplane: in-process (single node)");
|
||||
return new InProcessClusterBackplane(applicationProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public JobStore jobStore() {
|
||||
return new InProcessJobStore();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public RateLimitStore rateLimitStore() {
|
||||
return new InProcessRateLimitStore();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public DistributedLock distributedLock() {
|
||||
return new InProcessDistributedLock();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public KeyValueCache keyValueCache() {
|
||||
return new InProcessKeyValueCache();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public InstanceRegistry instanceRegistry() {
|
||||
return new InProcessInstanceRegistry();
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
|
||||
/**
|
||||
* In-process {@link DistributedLock}, non-reentrant per the interface contract, with lease-expiry
|
||||
* semantics that mirror a SET-NX-EX style distributed backend.
|
||||
*
|
||||
* <p>Each lock state carries a per-acquire {@code ownerToken} and an {@code expiryNanos}; another
|
||||
* caller can take over once the lease has elapsed even if the original holder never called {@link
|
||||
* LockHandle#release()}. This matters mostly for parity with the Valkey-backed implementation
|
||||
* (Redis {@code SETEX} auto-expires the key); within a single JVM a crashed holder takes its lock
|
||||
* state with it, but tests and code that rely on the {@code leaseTime} parameter still need it to
|
||||
* be honored.
|
||||
*
|
||||
* <p>Expiry is lazy: an expired lock state lingers in the map until the next acquire attempt for
|
||||
* the same key replaces it. Per-key cleanup also happens on explicit {@link LockHandle#release()},
|
||||
* so a balanced acquire/release workload keeps the map size bounded.
|
||||
*/
|
||||
public class InProcessDistributedLock implements DistributedLock {
|
||||
|
||||
private final ConcurrentHashMap<String, LockState> locks = new ConcurrentHashMap<>();
|
||||
private final AtomicLong tokenSeq = new AtomicLong();
|
||||
|
||||
/**
|
||||
* Lease state for a single acquired lock. {@code ownerToken} prevents a former holder from
|
||||
* releasing or renewing a lock now owned by someone else after lease expiry; {@code
|
||||
* expiryNanos} is read/written only inside {@link ConcurrentHashMap#compute} so the bin lock
|
||||
* provides the necessary happens-before guarantee.
|
||||
*/
|
||||
private static final class LockState {
|
||||
final long ownerToken;
|
||||
long expiryNanos;
|
||||
|
||||
LockState(long ownerToken, long expiryNanos) {
|
||||
this.ownerToken = ownerToken;
|
||||
this.expiryNanos = expiryNanos;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
|
||||
long token = tokenSeq.incrementAndGet();
|
||||
long nowNanos = System.nanoTime();
|
||||
long expiryNanos = nowNanos + leaseTime.toNanos();
|
||||
boolean[] acquired = {false};
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null || existing.expiryNanos - nowNanos <= 0L) {
|
||||
// No lock, or the previous lease has expired - we take it. Subtraction
|
||||
// form avoids the long-overflow trap that would bite a naive
|
||||
// expiryNanos <= nowNanos comparison around System.nanoTime() rollover.
|
||||
acquired[0] = true;
|
||||
return new LockState(token, expiryNanos);
|
||||
}
|
||||
return existing;
|
||||
});
|
||||
if (!acquired[0]) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(new InProcessHandle(lockKey, token));
|
||||
}
|
||||
|
||||
private void releaseInternal(String lockKey, long token) {
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null || existing.ownerToken != token) {
|
||||
// Already removed, expired-and-replaced, or never ours.
|
||||
return existing;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private boolean renewInternal(String lockKey, long token, Duration leaseTime) {
|
||||
long nowNanos = System.nanoTime();
|
||||
boolean[] renewed = {false};
|
||||
locks.compute(
|
||||
lockKey,
|
||||
(k, existing) -> {
|
||||
if (existing == null
|
||||
|| existing.ownerToken != token
|
||||
|| existing.expiryNanos - nowNanos <= 0L) {
|
||||
// Lock is gone or expired; renewal is a no-op so the caller can detect it.
|
||||
return existing;
|
||||
}
|
||||
existing.expiryNanos = nowNanos + leaseTime.toNanos();
|
||||
renewed[0] = true;
|
||||
return existing;
|
||||
});
|
||||
return renewed[0];
|
||||
}
|
||||
|
||||
private final class InProcessHandle implements LockHandle {
|
||||
private final String lockKey;
|
||||
private final long token;
|
||||
private boolean released;
|
||||
|
||||
InProcessHandle(String lockKey, long token) {
|
||||
this.lockKey = lockKey;
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void release() {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
releaseInternal(lockKey, token);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized boolean renew(Duration leaseTime) {
|
||||
if (released) {
|
||||
return false;
|
||||
}
|
||||
return renewInternal(lockKey, token, leaseTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import stirling.software.common.cluster.ClusterNode;
|
||||
import stirling.software.common.cluster.InstanceRegistry;
|
||||
|
||||
public class InProcessInstanceRegistry implements InstanceRegistry {
|
||||
|
||||
private final AtomicReference<ClusterNode> self = new AtomicReference<>();
|
||||
|
||||
@Override
|
||||
public void register(ClusterNode node, Duration heartbeatTtl) {
|
||||
self.set(node);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ClusterNode> lookup(String nodeId) {
|
||||
ClusterNode current = self.get();
|
||||
return current != null && current.nodeId().equals(nodeId)
|
||||
? Optional.of(current)
|
||||
: Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<ClusterNode> activeNodes() {
|
||||
ClusterNode current = self.get();
|
||||
return current == null ? Collections.emptyList() : Collections.singletonList(current);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deregister(String nodeId) {
|
||||
ClusterNode current = self.get();
|
||||
if (current != null && current.nodeId().equals(nodeId)) {
|
||||
self.set(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
|
||||
@Slf4j
|
||||
public class InProcessJobStore implements JobStore {
|
||||
|
||||
private final ConcurrentHashMap<String, Holder> entries = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public void put(JobStoreEntry entry, Duration ttl) {
|
||||
Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
|
||||
entries.put(entry.jobId(), new Holder(entry, expiry));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<JobStoreEntry> get(String jobId) {
|
||||
Holder h = entries.get(jobId);
|
||||
if (h == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (h.isExpired()) {
|
||||
entries.remove(jobId, h);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(h.entry);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String jobId) {
|
||||
entries.remove(jobId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String jobId) {
|
||||
return get(jobId).isPresent();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> findJobIdByFileId(String fileId) {
|
||||
for (Map.Entry<String, Holder> e : entries.entrySet()) {
|
||||
Holder h = e.getValue();
|
||||
if (h.isExpired()) {
|
||||
continue;
|
||||
}
|
||||
List<String> fileIds = h.entry.fileIds();
|
||||
if (fileIds != null && fileIds.contains(fileId)) {
|
||||
return Optional.of(e.getKey());
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<JobStoreEntry> all() {
|
||||
List<JobStoreEntry> result = new ArrayList<>(entries.size());
|
||||
for (Holder h : entries.values()) {
|
||||
if (!h.isExpired()) {
|
||||
result.add(h.entry);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Drop entries whose TTL has elapsed. Called by the {@code TaskManager} cleanup scheduler. */
|
||||
public int purgeExpired() {
|
||||
int removed = 0;
|
||||
Instant now = Instant.now();
|
||||
for (Map.Entry<String, Holder> e : entries.entrySet()) {
|
||||
if (!e.getValue().expiry.equals(Instant.MAX) && e.getValue().expiry.isBefore(now)) {
|
||||
if (entries.remove(e.getKey(), e.getValue())) {
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
private record Holder(JobStoreEntry entry, Instant expiry) {
|
||||
boolean isExpired() {
|
||||
return !expiry.equals(Instant.MAX) && expiry.isBefore(Instant.now());
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
|
||||
public class InProcessKeyValueCache implements KeyValueCache {
|
||||
|
||||
private final ConcurrentHashMap<String, ConcurrentHashMap<String, Expiring>> namespaces =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public void put(String namespace, String key, String value, Duration ttl) {
|
||||
Instant expiry = ttl == null ? Instant.MAX : Instant.now().plus(ttl);
|
||||
namespaces
|
||||
.computeIfAbsent(namespace, n -> new ConcurrentHashMap<>())
|
||||
.put(key, new Expiring(value, expiry));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> get(String namespace, String key) {
|
||||
Map<String, Expiring> ns = namespaces.get(namespace);
|
||||
if (ns == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Expiring e = ns.get(key);
|
||||
if (e == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (e.expiry.isBefore(Instant.now())) {
|
||||
ns.remove(key, e);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(e.value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void evict(String namespace, String key) {
|
||||
Map<String, Expiring> ns = namespaces.get(namespace);
|
||||
if (ns != null) {
|
||||
ns.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void evictNamespace(String namespace) {
|
||||
namespaces.remove(namespace);
|
||||
}
|
||||
|
||||
private record Expiring(String value, Instant expiry) {}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import io.github.bucket4j.Bandwidth;
|
||||
import io.github.bucket4j.Bucket;
|
||||
import io.github.bucket4j.ConsumptionProbe;
|
||||
import io.github.bucket4j.local.LocalBucketBuilder;
|
||||
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
|
||||
/** Bucket4j-backed token bucket implementation of {@link RateLimitStore}. */
|
||||
public class InProcessRateLimitStore implements RateLimitStore {
|
||||
|
||||
/** Cap to bound memory; oldest accessed buckets are evicted. */
|
||||
private static final int MAX_BUCKETS = 10_000;
|
||||
|
||||
private final Map<String, Bucket> buckets =
|
||||
Collections.synchronizedMap(
|
||||
new LinkedHashMap<String, Bucket>(256, 0.75f, true) {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<String, Bucket> eldest) {
|
||||
return size() > MAX_BUCKETS;
|
||||
}
|
||||
});
|
||||
|
||||
@Override
|
||||
public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
|
||||
String compositeKey = bucketKey + "|" + capacity + "|" + refillPeriod.toNanos();
|
||||
Bucket bucket =
|
||||
buckets.computeIfAbsent(compositeKey, k -> buildBucket(capacity, refillPeriod));
|
||||
ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
|
||||
return new RateLimitDecision(
|
||||
probe.isConsumed(),
|
||||
probe.getRemainingTokens(),
|
||||
probe.isConsumed() ? 0L : probe.getNanosToWaitForRefill());
|
||||
}
|
||||
|
||||
private static Bucket buildBucket(long capacity, Duration refillPeriod) {
|
||||
Bandwidth limit =
|
||||
Bandwidth.builder().capacity(capacity).refillGreedy(capacity, refillPeriod).build();
|
||||
LocalBucketBuilder builder = Bucket.builder();
|
||||
builder.addLimit(limit);
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/** Local-disk {@link FileStore} storing files under a base directory keyed by a UUID file id. */
|
||||
@Slf4j
|
||||
public class LocalDiskFileStore implements FileStore {
|
||||
|
||||
private final String baseDirPath;
|
||||
|
||||
public LocalDiskFileStore(String baseDirPath) {
|
||||
this.baseDirPath = baseDirPath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stored store(InputStream in, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try {
|
||||
long size = Files.copy(in, filePath);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* File-to-file copy. {@link Files#copy(Path, Path, java.nio.file.CopyOption...)} can use {@code
|
||||
* sendfile(2)} on Linux for a zero-copy kernel transfer when source and destination share a
|
||||
* filesystem, avoiding the streaming overhead of pulling the bytes through the JVM heap. Reads
|
||||
* the source size before copying so the post-copy stat is unnecessary.
|
||||
*/
|
||||
@Override
|
||||
public Stored store(Path source, String originalName) throws IOException {
|
||||
String fileId = UUID.randomUUID().toString();
|
||||
Path filePath = resolve(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.size(source);
|
||||
boolean success = false;
|
||||
try {
|
||||
Files.copy(source, filePath);
|
||||
success = true;
|
||||
return new Stored(fileId, size);
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream retrieve(String fileId) throws IOException {
|
||||
return new BufferedInputStream(Files.newInputStream(resolve(fileId)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
Path filePath = resolve(fileId);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
return Files.readAllBytes(filePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size(String fileId) throws IOException {
|
||||
Path filePath = resolve(fileId);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
return Files.size(filePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(String fileId) {
|
||||
try {
|
||||
return Files.deleteIfExists(resolve(fileId));
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String fileId) {
|
||||
return Files.exists(resolve(fileId));
|
||||
}
|
||||
|
||||
public Path resolve(String fileId) {
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
Path basePath = Path.of(baseDirPath).normalize().toAbsolutePath();
|
||||
Path resolvedPath = basePath.resolve(fileId).normalize();
|
||||
if (!resolvedPath.startsWith(basePath)) {
|
||||
throw new IllegalArgumentException("File ID resolves to an invalid path");
|
||||
}
|
||||
return resolvedPath;
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/**
|
||||
* Always-on wiring for the per-node local-disk {@link FileStore}. Active when {@code
|
||||
* cluster.artifactStore=local} (the default; {@code matchIfMissing=true}). The S3 artifact-store
|
||||
* supplies its own bean when {@code cluster.artifactStore=s3}.
|
||||
*/
|
||||
@Configuration
|
||||
@ConditionalOnProperty(
|
||||
prefix = "cluster",
|
||||
name = "artifactStore",
|
||||
havingValue = "local",
|
||||
matchIfMissing = true)
|
||||
public class LocalDiskFileStoreConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public FileStore fileStore(@Value("${stirling.tempDir:/tmp/stirling-files}") String tempDir) {
|
||||
return new LocalDiskFileStore(tempDir);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@@ -77,6 +78,7 @@ public class ApplicationProperties {
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
|
||||
@Bean
|
||||
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
|
||||
@@ -254,6 +256,106 @@ public class ApplicationProperties {
|
||||
private int longRunningTimeoutSeconds = 600;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cluster backplane configuration. All keys live under the top-level {@code cluster.*} prefix
|
||||
* (e.g. env var {@code CLUSTER_ENABLED}). The master switch is {@link #enabled} and defaults to
|
||||
* off; when off the in-process backplane is wired and no other cluster keys are required.
|
||||
*/
|
||||
@Data
|
||||
public static class Cluster {
|
||||
|
||||
/** Master switch. When {@code false} (default) the in-process backplane is wired. */
|
||||
private boolean enabled = false;
|
||||
|
||||
/** Backplane implementation selector. Valid values: {@code inprocess} | {@code valkey}. */
|
||||
private String backplane = "inprocess";
|
||||
|
||||
/**
|
||||
* Transient cluster job-artifact store selector. Valid values: {@code local} | {@code s3}.
|
||||
*
|
||||
* <p>This is distinct from {@code storage.provider}, which selects the backend for
|
||||
* persistent user-uploaded files. The two switches exist because the user-facing storage
|
||||
* feature is optional ({@code storage.enabled=false} is common) but every multi-node
|
||||
* cluster still needs a shared artifact store to serve cross-node downloads. Both
|
||||
* implementations share credentials from {@code storage.s3.*} when set to {@code s3}.
|
||||
*/
|
||||
private String artifactStore = "local";
|
||||
|
||||
private Valkey valkey = new Valkey();
|
||||
private Node node = new Node();
|
||||
|
||||
private transient String cachedNodeId;
|
||||
|
||||
public NodeRole resolvedRole() {
|
||||
if (node == null || node.getRole() == null) {
|
||||
return NodeRole.BOTH;
|
||||
}
|
||||
String value = node.getRole().trim().toUpperCase(Locale.ROOT);
|
||||
try {
|
||||
return NodeRole.valueOf(value);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return NodeRole.BOTH;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized String resolvedNodeId() {
|
||||
if (node != null && node.getId() != null && !node.getId().isBlank()) {
|
||||
return node.getId();
|
||||
}
|
||||
if (cachedNodeId == null) {
|
||||
cachedNodeId = UUID.randomUUID().toString();
|
||||
}
|
||||
return cachedNodeId;
|
||||
}
|
||||
|
||||
public enum NodeRole {
|
||||
WEB,
|
||||
WORKER,
|
||||
BOTH
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Valkey {
|
||||
/**
|
||||
* {@code redis://host:6379} or {@code rediss://...} for TLS. Required when cluster mode
|
||||
* is on and backplane is valkey.
|
||||
*/
|
||||
private String url = "";
|
||||
|
||||
private Tls tls = new Tls();
|
||||
|
||||
@Data
|
||||
public static class Tls {
|
||||
/**
|
||||
* When {@code true}, skip Valkey/Redis TLS certificate verification (dev/test
|
||||
* only). Leave {@code false} in production.
|
||||
*/
|
||||
private boolean skipCertVerification = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Node {
|
||||
/** Optional explicit node id. Blank = auto-generated UUID at startup. */
|
||||
private String id = "";
|
||||
|
||||
/** {@code web} | {@code worker} | {@code both}. */
|
||||
private String role = "both";
|
||||
|
||||
/**
|
||||
* Internal cluster address advertised in the instance registry (host:port). Blank =
|
||||
* derived at startup.
|
||||
*/
|
||||
private String internalAddress = "";
|
||||
|
||||
/** {@code http} | {@code https} - scheme used when peers call this node. */
|
||||
private String scheme = "http";
|
||||
|
||||
/** Heartbeat publish interval for the instance registry, in milliseconds. */
|
||||
private long heartbeatIntervalMs = 5000;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP timeouts for loopback calls to internal Stirling API endpoints, used by the AI workflow
|
||||
* executor and the pipeline processor. A bounded read timeout prevents a hung tool (e.g. an
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
import java.io.PipedInputStream;
|
||||
import java.io.PipedOutputStream;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
@@ -18,9 +16,11 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
/**
|
||||
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
|
||||
* system to handle file references.
|
||||
* system to handle file references. Disk I/O is delegated to the injected {@link FileStore} bean.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -30,251 +30,150 @@ public class FileStorage {
|
||||
/** Holds the result of a stream-to-disk store operation: the file ID and the bytes written. */
|
||||
public record StoredFile(String fileId, long size) {}
|
||||
|
||||
@Value("${stirling.tempDir:/tmp/stirling-files}")
|
||||
private String tempDirPath;
|
||||
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
private final FileStore fileStore;
|
||||
|
||||
/**
|
||||
* Store a file and return its unique ID
|
||||
*
|
||||
* @param file The file to store
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeFile(MultipartFile file) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Transfer the file to the storage location
|
||||
file.transferTo(filePath.toFile());
|
||||
|
||||
log.debug("Stored file with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a byte array as a file and return its unique ID
|
||||
*
|
||||
* @param bytes The byte array to store
|
||||
* @param originalName The original name of the file (for extension)
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Write the bytes to the file
|
||||
Files.write(filePath, bytes);
|
||||
|
||||
log.debug("Stored byte array with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a MultipartFile
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a MultipartFile
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
// Fast path: when Spring buffered the multipart to disk (typical for large uploads), the
|
||||
// backing Resource exposes a real File. Hand the Path to the FileStore so it can do a
|
||||
// file-to-file copy (Linux sendfile, no copy through Java heap) rather than streaming
|
||||
// the bytes through an 8K buffer. Falls back to the InputStream path for in-memory
|
||||
// multiparts, exotic Resource impls, and anything that does not back onto a File.
|
||||
Resource res;
|
||||
try {
|
||||
res = file.getResource();
|
||||
} catch (RuntimeException ignored) {
|
||||
res = null;
|
||||
}
|
||||
if (res != null && res.isFile()) {
|
||||
try {
|
||||
FileStore.Stored stored =
|
||||
fileStore.store(res.getFile().toPath(), file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {} (fast path)", stored.fileId());
|
||||
return stored.fileId();
|
||||
} catch (IOException ex) {
|
||||
// Some Resource impls advertise isFile()=true but throw on getFile(); fall through.
|
||||
log.debug("Resource fast path failed, falling back to stream copy", ex);
|
||||
}
|
||||
}
|
||||
try (InputStream in = file.getInputStream()) {
|
||||
FileStore.Stored stored = fileStore.store(in, file.getOriginalFilename());
|
||||
log.debug("Stored file with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
}
|
||||
|
||||
byte[] fileData = Files.readAllBytes(filePath);
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
FileStore.Stored stored = fileStore.store(new ByteArrayInputStream(bytes), originalName);
|
||||
log.debug("Stored byte array with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
byte[] fileData = fileStore.retrieveBytes(fileId);
|
||||
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a byte array
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a byte array
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.readAllBytes(filePath);
|
||||
return fileStore.retrieveBytes(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a streaming InputStream. The caller is responsible for closing
|
||||
* the returned stream.
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return A buffered InputStream for the file
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public InputStream retrieveInputStream(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
// Let Files.newInputStream throw NoSuchFileException naturally — avoids TOCTOU race
|
||||
// between exists-check and open when another thread may delete concurrently.
|
||||
return new BufferedInputStream(Files.newInputStream(filePath));
|
||||
return fileStore.retrieve(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store data from an InputStream as a file and return its unique ID and byte count. Streams
|
||||
* directly to disk without buffering the entire content in heap.
|
||||
*
|
||||
* @param inputStream The input stream to read from
|
||||
* @param originalName The original name of the file (unused, kept for API symmetry)
|
||||
* @return A {@link StoredFile} containing the file ID and the number of bytes written
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public StoredFile storeInputStream(InputStream inputStream, String originalName)
|
||||
throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
long size = Files.copy(inputStream, filePath);
|
||||
log.debug("Stored input stream with ID: {}", fileId);
|
||||
return new StoredFile(fileId, size);
|
||||
FileStore.Stored stored = fileStore.store(inputStream, originalName);
|
||||
log.debug("Stored input stream with ID: {}", stored.fileId());
|
||||
return new StoredFile(stored.fileId(), stored.size());
|
||||
}
|
||||
|
||||
public String storeFromStreamingBody(StreamingResponseBody body, String originalName)
|
||||
throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(filePath))) {
|
||||
body.writeTo(os);
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
// Hold Throwable not IOException: an unchecked failure (NPE, IllegalState, OOM, etc.)
|
||||
// from the body writer would otherwise close the pipe with EOF and the consumer would
|
||||
// return a truncated file with no error surfaced to the caller.
|
||||
AtomicReference<Throwable> bodyError = new AtomicReference<>();
|
||||
try (PipedOutputStream out = new PipedOutputStream();
|
||||
PipedInputStream in = new PipedInputStream(out, 8192)) {
|
||||
var executor = Executors.newSingleThreadExecutor(Thread.ofVirtual().factory());
|
||||
java.util.concurrent.Future<?> task = null;
|
||||
try {
|
||||
task =
|
||||
executor.submit(
|
||||
() -> {
|
||||
try {
|
||||
body.writeTo(out);
|
||||
} catch (Throwable ex) {
|
||||
bodyError.set(ex);
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
} catch (IOException ignored) {
|
||||
// closed on the consumer side too
|
||||
}
|
||||
}
|
||||
});
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
Throwable writerErr = bodyError.get();
|
||||
if (writerErr != null) {
|
||||
// Body failed mid-write: the FileStore persisted a truncated entry.
|
||||
// Best-effort delete so we don't leak partial files; never let cleanup
|
||||
// mask the original writer error.
|
||||
try {
|
||||
fileStore.delete(stored.fileId());
|
||||
} catch (RuntimeException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to delete partial file {} after writer error: {}",
|
||||
stored.fileId(),
|
||||
cleanupEx.getMessage());
|
||||
}
|
||||
if (writerErr instanceof IOException ioe) {
|
||||
throw ioe;
|
||||
}
|
||||
throw new IOException(
|
||||
"StreamingResponseBody writer failed: " + writerErr.getMessage(),
|
||||
writerErr);
|
||||
}
|
||||
log.debug("Stored StreamingResponseBody with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
} finally {
|
||||
// Interrupt and join the writer task: shutdown() alone returns immediately and a
|
||||
// failed store leaves the writer running, leaking a thread per failed upload.
|
||||
if (task != null && !task.isDone()) {
|
||||
task.cancel(true);
|
||||
}
|
||||
executor.shutdown();
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
if (!executor.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
executor.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
log.debug("Stored StreamingResponseBody with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a {@link Resource} body to disk, returning the generated file ID. Used by the async
|
||||
* job pipeline to capture {@code ResponseEntity<Resource>} results produced by controllers.
|
||||
*/
|
||||
public String storeFromResource(Resource resource, String originalName) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
Files.createDirectories(filePath.getParent());
|
||||
boolean success = false;
|
||||
try (InputStream in = resource.getInputStream()) {
|
||||
Files.copy(in, filePath);
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
try {
|
||||
Files.deleteIfExists(filePath);
|
||||
} catch (IOException cleanupEx) {
|
||||
log.warn(
|
||||
"Failed to clean up partial file {} after store failure",
|
||||
filePath,
|
||||
cleanupEx);
|
||||
}
|
||||
}
|
||||
FileStore.Stored stored = fileStore.store(in, originalName);
|
||||
log.debug("Stored Resource with ID: {}", stored.fileId());
|
||||
return stored.fileId();
|
||||
}
|
||||
log.debug("Stored Resource with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to delete
|
||||
* @return true if the file was deleted, false otherwise
|
||||
*/
|
||||
public boolean deleteFile(String fileId) {
|
||||
try {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.deleteIfExists(filePath);
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
return fileStore.delete(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to check
|
||||
* @return true if the file exists, false otherwise
|
||||
*/
|
||||
public boolean fileExists(String fileId) {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.exists(filePath);
|
||||
return fileStore.exists(fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of a file by its ID without loading the content into memory
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The size of the file in bytes
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public long getFileSize(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.size(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path for a file ID
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The path to the file
|
||||
* @throws IllegalArgumentException if fileId contains path traversal characters or resolves
|
||||
* outside base directory
|
||||
*/
|
||||
private Path getFilePath(String fileId) {
|
||||
// Validate fileId to prevent path traversal
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
|
||||
Path basePath = Path.of(tempDirPath).normalize().toAbsolutePath();
|
||||
Path resolvedPath = basePath.resolve(fileId).normalize();
|
||||
|
||||
// Ensure resolved path is within the base directory
|
||||
if (!resolvedPath.startsWith(basePath)) {
|
||||
throw new IllegalArgumentException("File ID resolves to an invalid path");
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique file ID
|
||||
*
|
||||
* @return A unique file ID
|
||||
*/
|
||||
private String generateFileId() {
|
||||
return UUID.randomUUID().toString();
|
||||
return fileStore.size(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,13 @@ package stirling.software.common.service;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -16,6 +20,7 @@ import java.util.concurrent.TimeUnit;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -26,6 +31,10 @@ import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
import stirling.software.common.model.job.JobStats;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
@@ -40,20 +49,20 @@ public class TaskManager {
|
||||
private int jobResultExpiryMinutes = 30;
|
||||
|
||||
private final FileStorage fileStorage;
|
||||
private final JobStore jobStore;
|
||||
private final ClusterBackplane clusterBackplane;
|
||||
private final ScheduledExecutorService cleanupExecutor =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
Thread.ofVirtual().name("task-cleanup-", 0).factory());
|
||||
|
||||
/** Initialize the task manager and start the cleanup scheduler */
|
||||
public TaskManager(FileStorage fileStorage) {
|
||||
@Autowired
|
||||
public TaskManager(
|
||||
FileStorage fileStorage, JobStore jobStore, ClusterBackplane clusterBackplane) {
|
||||
this.fileStorage = fileStorage;
|
||||
this.jobStore = jobStore;
|
||||
this.clusterBackplane = clusterBackplane;
|
||||
|
||||
// Schedule periodic cleanup of old job results
|
||||
cleanupExecutor.scheduleAtFixedRate(
|
||||
this::cleanupOldJobs,
|
||||
10, // Initial delay
|
||||
10, // Interval
|
||||
TimeUnit.MINUTES);
|
||||
cleanupExecutor.scheduleAtFixedRate(this::cleanupOldJobs, 10, 10, TimeUnit.MINUTES);
|
||||
|
||||
log.debug(
|
||||
"Task manager initialized with job result expiry of {} minutes",
|
||||
@@ -66,7 +75,9 @@ public class TaskManager {
|
||||
* @param jobId The job ID
|
||||
*/
|
||||
public void createTask(String jobId) {
|
||||
jobResults.put(jobId, JobResult.createNew(jobId));
|
||||
JobResult result = JobResult.createNew(jobId);
|
||||
jobResults.put(jobId, result);
|
||||
writeThrough(jobId, result);
|
||||
log.debug("Created task with job ID: {}", jobId);
|
||||
}
|
||||
|
||||
@@ -79,6 +90,7 @@ public class TaskManager {
|
||||
public void setResult(String jobId, Object result) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithResult(result);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Set result for job ID: {}", jobId);
|
||||
}
|
||||
|
||||
@@ -101,6 +113,7 @@ public class TaskManager {
|
||||
extractZipToIndividualFiles(fileId, originalFileName);
|
||||
if (!extractedFiles.isEmpty()) {
|
||||
jobResult.completeWithFiles(extractedFiles);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files extracted from"
|
||||
+ " ZIP",
|
||||
@@ -127,6 +140,7 @@ public class TaskManager {
|
||||
"Failed to get file size for job {}: {}. Using size 0.", jobId, e.getMessage());
|
||||
jobResult.completeWithSingleFile(fileId, originalFileName, contentType, 0);
|
||||
}
|
||||
writeThrough(jobId, jobResult);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -138,6 +152,7 @@ public class TaskManager {
|
||||
public void setMultipleFileResults(String jobId, List<ResultFile> resultFiles) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithFiles(resultFiles);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files",
|
||||
jobId,
|
||||
@@ -153,6 +168,7 @@ public class TaskManager {
|
||||
public void setError(String jobId, String error) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.failWithError(error);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Set error for job ID: {}: {}", jobId, error);
|
||||
}
|
||||
|
||||
@@ -169,6 +185,7 @@ public class TaskManager {
|
||||
// If no result or error has been set, mark it as complete with an empty result
|
||||
jobResult.completeWithResult("Task completed successfully");
|
||||
}
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Marked job ID: {} as complete", jobId);
|
||||
}
|
||||
|
||||
@@ -205,6 +222,7 @@ public class TaskManager {
|
||||
JobResult jobResult = jobResults.get(jobId);
|
||||
if (jobResult != null) {
|
||||
jobResult.addNote(note);
|
||||
writeThrough(jobId, jobResult);
|
||||
log.debug("Added note to job ID: {}: {}", jobId, note);
|
||||
return true;
|
||||
}
|
||||
@@ -295,8 +313,11 @@ public class TaskManager {
|
||||
return jobResults.computeIfAbsent(jobId, JobResult::createNew);
|
||||
}
|
||||
|
||||
/** Clean up old completed job results */
|
||||
/** Clean up old completed job results. No-op in cluster mode; the backplane TTL owns expiry. */
|
||||
public void cleanupOldJobs() {
|
||||
if (clusterBackplane != null && !clusterBackplane.shouldRunLocalCleanup()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime expiryThreshold =
|
||||
LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
|
||||
int removedCount = 0;
|
||||
@@ -315,6 +336,9 @@ public class TaskManager {
|
||||
|
||||
// Remove the job result
|
||||
jobResults.remove(entry.getKey());
|
||||
if (jobStore != null) {
|
||||
jobStore.delete(entry.getKey());
|
||||
}
|
||||
removedCount++;
|
||||
}
|
||||
}
|
||||
@@ -327,6 +351,53 @@ public class TaskManager {
|
||||
}
|
||||
}
|
||||
|
||||
/** Mirror the in-memory {@code JobResult} into the cluster-visible {@link JobStore}. */
|
||||
private void writeThrough(String jobId, JobResult result) {
|
||||
if (jobStore == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
jobStore.put(toEntry(jobId, result), Duration.ofMinutes(jobResultExpiryMinutes));
|
||||
} catch (RuntimeException ex) {
|
||||
log.warn("JobStore write-through failed for job {}: {}", jobId, ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private JobStoreEntry toEntry(String jobId, JobResult result) {
|
||||
JobState state;
|
||||
if (result.isComplete()) {
|
||||
state = result.getError() != null ? JobState.FAILED : JobState.COMPLETE;
|
||||
} else {
|
||||
state = JobState.PENDING;
|
||||
}
|
||||
Instant createdAt = toInstant(result.getCreatedAt());
|
||||
Instant completedAt = toInstant(result.getCompletedAt());
|
||||
List<String> fileIds = new ArrayList<>();
|
||||
if (result.hasFiles()) {
|
||||
for (ResultFile rf : result.getAllResultFiles()) {
|
||||
fileIds.add(rf.getFileId());
|
||||
}
|
||||
}
|
||||
Map<String, String> meta = new HashMap<>();
|
||||
if (result.getNotes() != null && !result.getNotes().isEmpty()) {
|
||||
meta.put("notesCount", Integer.toString(result.getNotes().size()));
|
||||
}
|
||||
String owningNodeId = clusterBackplane == null ? "local" : clusterBackplane.localNodeId();
|
||||
return new JobStoreEntry(
|
||||
jobId,
|
||||
state,
|
||||
owningNodeId,
|
||||
createdAt,
|
||||
completedAt,
|
||||
result.getError(),
|
||||
fileIds,
|
||||
meta);
|
||||
}
|
||||
|
||||
private Instant toInstant(LocalDateTime ldt) {
|
||||
return ldt == null ? null : ldt.atZone(ZoneId.systemDefault()).toInstant();
|
||||
}
|
||||
|
||||
/** Shutdown the cleanup executor */
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
@@ -370,7 +441,7 @@ public class TaskManager {
|
||||
while ((entry = zipIn.getNextEntry()) != null) {
|
||||
if (!entry.isDirectory()) {
|
||||
String contentType = determineContentType(entry.getName());
|
||||
// storeInputStream returns the fileId and byte count — no extra stat needed
|
||||
// storeInputStream returns the fileId and byte count - no extra stat needed
|
||||
FileStorage.StoredFile stored =
|
||||
fileStorage.storeInputStream(zipIn, entry.getName());
|
||||
|
||||
@@ -458,7 +529,8 @@ public class TaskManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the job key that owns a given file ID.
|
||||
* Find the job key that owns a given file ID. Checks the local in-memory map first, then falls
|
||||
* back to the cluster-visible {@link JobStore}.
|
||||
*
|
||||
* @param fileId file identifier to look up
|
||||
* @return scoped job key if found, otherwise null
|
||||
@@ -474,6 +546,18 @@ public class TaskManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (jobStore != null) {
|
||||
// Propagate JobStore failures: returning null on a backplane outage would conflate
|
||||
// "no such file" with "lookup unavailable" and the caller would respond 404 to a
|
||||
// transient blip that should be retried. Let Spring's exception handler surface a
|
||||
// 5xx so clients know to retry.
|
||||
try {
|
||||
return jobStore.findJobIdByFileId(fileId).orElse(null);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("JobStore findJobIdByFileId failed for {}: {}", fileId, e.getMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ public class RequestUriUtils {
|
||||
"/api/v1/ui-data/footer-info") // Public footer configuration
|
||||
|| trimmedUri.startsWith("/api/v1/invite/validate")
|
||||
|| trimmedUri.startsWith("/api/v1/invite/accept")
|
||||
// Health Endoints
|
||||
// Health Endpoints
|
||||
|| trimmedUri.startsWith("/actuator/health")
|
||||
|| trimmedUri.startsWith("/health")
|
||||
|| trimmedUri.startsWith("/healthz")
|
||||
|
||||
@@ -56,4 +56,17 @@ class ArchitectureTest {
|
||||
.resideInAPackage("stirling.software.saas..");
|
||||
rule.check(commonClasses);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clusterInterfacesHaveNoImplementationDependencies() {
|
||||
ArchRule rule =
|
||||
noClasses()
|
||||
.that()
|
||||
.resideInAPackage("stirling.software.common.cluster..")
|
||||
.should()
|
||||
.dependOnClassesThat()
|
||||
.resideInAnyPackage(
|
||||
"stirling.software.proprietary..", "stirling.software.saas..");
|
||||
rule.check(commonClasses);
|
||||
}
|
||||
}
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class BackplaneContractCompilationTest {
|
||||
|
||||
@Test
|
||||
void jobStoreEntryRecordRoundTrips() {
|
||||
Instant now = Instant.now();
|
||||
JobStoreEntry entry =
|
||||
new JobStoreEntry(
|
||||
"job-1",
|
||||
JobStoreEntry.JobState.PENDING,
|
||||
"node-a",
|
||||
now,
|
||||
null,
|
||||
null,
|
||||
List.of("file-1"),
|
||||
Map.of("k", "v"));
|
||||
assertEquals("job-1", entry.jobId());
|
||||
assertEquals(JobStoreEntry.JobState.PENDING, entry.state());
|
||||
assertEquals("node-a", entry.owningNodeId());
|
||||
assertEquals(now, entry.createdAt());
|
||||
assertEquals(List.of("file-1"), entry.fileIds());
|
||||
assertEquals("v", entry.resultMeta().get("k"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void clusterNodeRecordRoundTrips() {
|
||||
Instant heartbeat = Instant.now();
|
||||
ClusterNode node = new ClusterNode("node-a", "10.0.0.1:8080", heartbeat, "BOTH");
|
||||
assertEquals("node-a", node.nodeId());
|
||||
assertEquals("10.0.0.1:8080", node.internalAddress());
|
||||
assertEquals(heartbeat, node.lastHeartbeat());
|
||||
assertEquals("BOTH", node.role());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rateLimitDecisionRecordRoundTrips() {
|
||||
RateLimitStore.RateLimitDecision d = new RateLimitStore.RateLimitDecision(true, 7, 0L);
|
||||
assertEquals(true, d.allowed());
|
||||
assertEquals(7, d.remainingTokens());
|
||||
assertEquals(0L, d.nanosToWaitForRefill());
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
class ClusterConfigValidationTest {
|
||||
|
||||
@Test
|
||||
void validationPassesWhenDisabled() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationFailsWhenValkeyEnabledWithoutUrl() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("valkey");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertThrows(IllegalStateException.class, () -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationPassesWhenValkeyEnabledWithUrl() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("valkey");
|
||||
cluster.getValkey().setUrl("redis://localhost:6379");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validationPassesWhenInProcessEnabled() {
|
||||
ApplicationProperties props = new ApplicationProperties();
|
||||
Cluster cluster = props.getCluster();
|
||||
cluster.setEnabled(true);
|
||||
cluster.setBackplane("inprocess");
|
||||
ClusterConfig config = new ClusterConfig(props);
|
||||
assertDoesNotThrow(() -> invokeValidate(config));
|
||||
}
|
||||
|
||||
private void invokeValidate(ClusterConfig config) throws Exception {
|
||||
Method m = ClusterConfig.class.getDeclaredMethod("validate");
|
||||
m.setAccessible(true);
|
||||
try {
|
||||
m.invoke(config);
|
||||
} catch (java.lang.reflect.InvocationTargetException ex) {
|
||||
if (ex.getCause() instanceof RuntimeException re) {
|
||||
throw re;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.Cluster;
|
||||
|
||||
class ClusterPropertiesTest {
|
||||
|
||||
@Test
|
||||
void defaultsAreDisabledAndInprocess() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
assertFalse(props.isEnabled());
|
||||
assertEquals("inprocess", props.getBackplane());
|
||||
assertEquals("local", props.getArtifactStore());
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
assertEquals("", props.getValkey().getUrl());
|
||||
assertFalse(props.getValkey().getTls().isSkipCertVerification());
|
||||
assertEquals("both", props.getNode().getRole());
|
||||
assertEquals("http", props.getNode().getScheme());
|
||||
assertEquals(5000L, props.getNode().getHeartbeatIntervalMs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedRoleParsesCaseInsensitively() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
props.getNode().setRole("WEB");
|
||||
assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("web");
|
||||
assertEquals(Cluster.NodeRole.WEB, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("Worker");
|
||||
assertEquals(Cluster.NodeRole.WORKER, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole("garbage");
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
|
||||
props.getNode().setRole(null);
|
||||
assertEquals(Cluster.NodeRole.BOTH, props.resolvedRole());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedNodeIdIsStableAcrossCalls() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
String first = props.resolvedNodeId();
|
||||
String second = props.resolvedNodeId();
|
||||
assertNotNull(first);
|
||||
assertEquals(first, second);
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvedNodeIdHonoursExplicitId() {
|
||||
Cluster props = new ApplicationProperties().getCluster();
|
||||
props.getNode().setId("abc");
|
||||
assertEquals("abc", props.resolvedNodeId());
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package stirling.software.common.cluster;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.InProcessClusterConfiguration;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
/**
|
||||
* Verifies the {@link InProcessClusterConfiguration} conditional wiring: in-process beans wire when
|
||||
* cluster mode is off or {@code backplane=inprocess}, and are skipped when {@code
|
||||
* backplane=valkey}.
|
||||
*/
|
||||
class InProcessConfigurationConditionalTest {
|
||||
|
||||
private final ApplicationContextRunner runner =
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(
|
||||
org.springframework.boot.autoconfigure.AutoConfigurations.of(
|
||||
PropertyPlaceholderAutoConfiguration.class))
|
||||
.withUserConfiguration(
|
||||
TestAppPropertiesConfig.class,
|
||||
ClusterConfig.class,
|
||||
InProcessClusterConfiguration.class);
|
||||
|
||||
@Test
|
||||
void inProcessBeansWireWhenClusterDisabled() {
|
||||
runner.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.hasSingleBean(ClusterBackplane.class)
|
||||
.hasSingleBean(JobStore.class)
|
||||
.hasSingleBean(RateLimitStore.class)
|
||||
.hasSingleBean(DistributedLock.class)
|
||||
.hasSingleBean(KeyValueCache.class)
|
||||
.hasSingleBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void inProcessBeansWireWhenEnabledWithInProcessBackplane() {
|
||||
runner.withPropertyValues("cluster.enabled=true", "cluster.backplane=inprocess")
|
||||
.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.hasSingleBean(ClusterBackplane.class)
|
||||
.hasSingleBean(JobStore.class)
|
||||
.hasSingleBean(RateLimitStore.class)
|
||||
.hasSingleBean(DistributedLock.class)
|
||||
.hasSingleBean(KeyValueCache.class)
|
||||
.hasSingleBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void inProcessBeansSkippedWhenEnabledWithDistributedBackplane() {
|
||||
runner.withPropertyValues(
|
||||
"cluster.enabled=true",
|
||||
"cluster.backplane=valkey",
|
||||
"cluster.valkey.url=redis://localhost:6379")
|
||||
.run(
|
||||
context ->
|
||||
assertThat(context)
|
||||
.hasNotFailed()
|
||||
.doesNotHaveBean(ClusterBackplane.class)
|
||||
.doesNotHaveBean(JobStore.class)
|
||||
.doesNotHaveBean(RateLimitStore.class)
|
||||
.doesNotHaveBean(DistributedLock.class)
|
||||
.doesNotHaveBean(KeyValueCache.class)
|
||||
.doesNotHaveBean(InstanceRegistry.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand-rolled {@link ApplicationProperties} bean: the production class loads YAML at startup
|
||||
* via a {@code @PostConstruct} hook that isn't appropriate for the slice runner, so we wire a
|
||||
* defaults-only instance here.
|
||||
*/
|
||||
@Configuration
|
||||
static class TestAppPropertiesConfig {
|
||||
@Bean
|
||||
ApplicationProperties applicationProperties() {
|
||||
return new ApplicationProperties();
|
||||
}
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.DistributedLock;
|
||||
|
||||
class InProcessDistributedLockTest {
|
||||
|
||||
@Test
|
||||
void acquireReleaseAcquire() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
h1.release();
|
||||
assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void reentryFromSameThreadFails() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
assertFalse(reentry.isPresent(), "in-process lock must be non-reentrant");
|
||||
h1.release();
|
||||
// After release, anyone can acquire again.
|
||||
assertTrue(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondAcquireFromAnotherThreadFails() throws InterruptedException {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
|
||||
CountDownLatch done = new CountDownLatch(1);
|
||||
AtomicBoolean acquired = new AtomicBoolean(true);
|
||||
Thread t =
|
||||
new Thread(
|
||||
() -> {
|
||||
Optional<DistributedLock.LockHandle> attempt =
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
acquired.set(attempt.isPresent());
|
||||
attempt.ifPresent(DistributedLock.LockHandle::release);
|
||||
done.countDown();
|
||||
});
|
||||
t.start();
|
||||
assertTrue(done.await(2, TimeUnit.SECONDS));
|
||||
assertFalse(acquired.get());
|
||||
h1.release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void leaseExpiryAllowsTakeoverEvenWithoutRelease() throws InterruptedException {
|
||||
// Acquire with a short lease, never call release, then try to acquire again after the
|
||||
// lease has elapsed. Matches Redis SET-NX-EX semantics - the second caller gets the lock
|
||||
// because the first lease auto-expired. 250ms lease + 350ms wait gives CI generous slack.
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(250)).orElseThrow();
|
||||
Thread.sleep(350);
|
||||
Optional<DistributedLock.LockHandle> takeover =
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30));
|
||||
assertTrue(
|
||||
takeover.isPresent(),
|
||||
"expired lease must release the lock so a new caller can take over");
|
||||
// Calling release() on the original handle after takeover must be a no-op (token check).
|
||||
h1.release();
|
||||
// The takeover holder is still the legitimate owner.
|
||||
assertFalse(lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent());
|
||||
takeover.get().release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewExtendsLease() throws InterruptedException {
|
||||
// Acquire with a short lease, renew it before it expires, then verify the lock is still
|
||||
// held past the original expiry point. 200ms initial + renew to 2s + wait 350ms.
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofMillis(200)).orElseThrow();
|
||||
assertTrue(h1.renew(Duration.ofSeconds(2)), "renew on a held lease must succeed");
|
||||
Thread.sleep(350);
|
||||
assertFalse(
|
||||
lock.tryAcquire("k", Duration.ofSeconds(30)).isPresent(),
|
||||
"renew should have pushed expiry well past the original 200ms");
|
||||
h1.release();
|
||||
}
|
||||
|
||||
@Test
|
||||
void renewAfterReleaseFails() {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
|
||||
h1.release();
|
||||
assertFalse(h1.renew(Duration.ofSeconds(30)), "renew on a released handle must fail");
|
||||
}
|
||||
|
||||
/**
|
||||
* Concurrency stress: many threads contending on the same key with each holder respecting the
|
||||
* lease (hold << lease). The lock behaves as a strict mutex in this regime so asserting
|
||||
* mutual exclusion is meaningful. A separate test ({@link
|
||||
* #leaseExpiryAllowsTakeoverEvenWithoutRelease}) covers the takeover-across-expiry branch,
|
||||
* which legitimately allows two holders momentarily and is split-brain behaviour inherent to
|
||||
* any lease-based lock.
|
||||
*/
|
||||
@Test
|
||||
void concurrentContentionPreservesMutualExclusion() throws InterruptedException {
|
||||
DistributedLock lock = new InProcessDistributedLock();
|
||||
int threads = 16;
|
||||
int attemptsPerThread = 200;
|
||||
// Lease far exceeds any plausible hold time, so the takeover branch never triggers in
|
||||
// this test and the lock acts as a strict mutex.
|
||||
Duration lease = Duration.ofSeconds(5);
|
||||
java.util.concurrent.atomic.AtomicInteger concurrentHolders =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicInteger maxConcurrent =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicInteger acquires =
|
||||
new java.util.concurrent.atomic.AtomicInteger();
|
||||
java.util.concurrent.atomic.AtomicReference<Throwable> firstFailure =
|
||||
new java.util.concurrent.atomic.AtomicReference<>();
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
CountDownLatch done = new CountDownLatch(threads);
|
||||
|
||||
for (int i = 0; i < threads; i++) {
|
||||
new Thread(
|
||||
() -> {
|
||||
try {
|
||||
start.await();
|
||||
for (int j = 0; j < attemptsPerThread; j++) {
|
||||
Optional<DistributedLock.LockHandle> h =
|
||||
lock.tryAcquire("hot", lease);
|
||||
if (h.isPresent()) {
|
||||
int now = concurrentHolders.incrementAndGet();
|
||||
maxConcurrent.accumulateAndGet(now, Math::max);
|
||||
acquires.incrementAndGet();
|
||||
// Trivial critical section; well within lease.
|
||||
concurrentHolders.decrementAndGet();
|
||||
h.get().release();
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
firstFailure.compareAndSet(null, t);
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
},
|
||||
"lock-stress-" + i)
|
||||
.start();
|
||||
}
|
||||
start.countDown();
|
||||
assertTrue(done.await(30, TimeUnit.SECONDS), "stress workers must finish in time");
|
||||
org.junit.jupiter.api.Assertions.assertNull(firstFailure.get(), "no worker may throw");
|
||||
org.junit.jupiter.api.Assertions.assertEquals(
|
||||
1,
|
||||
maxConcurrent.get(),
|
||||
"mutual exclusion violated: more than one holder observed simultaneously");
|
||||
assertTrue(
|
||||
acquires.get() > 0,
|
||||
"at least some acquires must succeed under contention (saw "
|
||||
+ acquires.get()
|
||||
+ ")");
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.ClusterNode;
|
||||
|
||||
class InProcessInstanceRegistryTest {
|
||||
|
||||
@Test
|
||||
void registerThenLookupAndActiveNodes() {
|
||||
InProcessInstanceRegistry registry = new InProcessInstanceRegistry();
|
||||
ClusterNode node = new ClusterNode("node-1", "127.0.0.1:8080", Instant.now(), "BOTH");
|
||||
registry.register(node, Duration.ofSeconds(30));
|
||||
|
||||
assertTrue(registry.lookup("node-1").isPresent());
|
||||
assertEquals("node-1", registry.lookup("node-1").get().nodeId());
|
||||
assertEquals(1, registry.activeNodes().size());
|
||||
|
||||
registry.deregister("node-1");
|
||||
assertTrue(registry.lookup("node-1").isEmpty());
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
|
||||
class InProcessJobStoreTest {
|
||||
|
||||
private final InProcessJobStore store = new InProcessJobStore();
|
||||
|
||||
@Test
|
||||
void putGetDeleteExistsRoundTrip() {
|
||||
JobStoreEntry entry = entry("job-1");
|
||||
store.put(entry, Duration.ofMinutes(30));
|
||||
|
||||
assertTrue(store.exists("job-1"));
|
||||
assertEquals(entry, store.get("job-1").orElseThrow());
|
||||
|
||||
store.delete("job-1");
|
||||
assertFalse(store.exists("job-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ttlExpiry() throws InterruptedException {
|
||||
store.put(entry("job-2"), Duration.ofMillis(50));
|
||||
Thread.sleep(100);
|
||||
assertFalse(store.get("job-2").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void purgeExpiredRemovesOnlyStaleEntries() throws InterruptedException {
|
||||
store.put(entry("job-fresh"), Duration.ofMinutes(30));
|
||||
store.put(entry("job-stale"), Duration.ofMillis(20));
|
||||
Thread.sleep(80);
|
||||
|
||||
int removed = store.purgeExpired();
|
||||
assertEquals(1, removed);
|
||||
assertTrue(store.exists("job-fresh"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void findJobIdByFileIdReturnsTheRightJob() {
|
||||
store.put(
|
||||
new JobStoreEntry(
|
||||
"job-a",
|
||||
JobStoreEntry.JobState.COMPLETE,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
Instant.now(),
|
||||
null,
|
||||
List.of("file-1", "file-2"),
|
||||
Map.of()),
|
||||
Duration.ofMinutes(30));
|
||||
store.put(
|
||||
new JobStoreEntry(
|
||||
"job-b",
|
||||
JobStoreEntry.JobState.COMPLETE,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
Instant.now(),
|
||||
null,
|
||||
List.of("file-3"),
|
||||
Map.of()),
|
||||
Duration.ofMinutes(30));
|
||||
|
||||
assertEquals("job-a", store.findJobIdByFileId("file-1").orElseThrow());
|
||||
assertEquals("job-b", store.findJobIdByFileId("file-3").orElseThrow());
|
||||
assertFalse(store.findJobIdByFileId("missing").isPresent());
|
||||
}
|
||||
|
||||
private JobStoreEntry entry(String id) {
|
||||
return new JobStoreEntry(
|
||||
id,
|
||||
JobStoreEntry.JobState.PENDING,
|
||||
"node-1",
|
||||
Instant.now(),
|
||||
null,
|
||||
null,
|
||||
List.of(),
|
||||
Map.of());
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.KeyValueCache;
|
||||
|
||||
class InProcessKeyValueCacheTest {
|
||||
|
||||
@Test
|
||||
void putGetEvict() {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("apikey", "a", "userA", Duration.ofMinutes(1));
|
||||
assertEquals("userA", cache.get("apikey", "a").orElseThrow());
|
||||
|
||||
cache.evict("apikey", "a");
|
||||
assertFalse(cache.get("apikey", "a").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ttlExpiry() throws InterruptedException {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("ns", "k", "v", Duration.ofMillis(40));
|
||||
Thread.sleep(80);
|
||||
assertFalse(cache.get("ns", "k").isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void evictNamespace() {
|
||||
KeyValueCache cache = new InProcessKeyValueCache();
|
||||
cache.put("ns", "a", "1", Duration.ofMinutes(1));
|
||||
cache.put("ns", "b", "2", Duration.ofMinutes(1));
|
||||
cache.evictNamespace("ns");
|
||||
assertFalse(cache.get("ns", "a").isPresent());
|
||||
assertFalse(cache.get("ns", "b").isPresent());
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.cluster.RateLimitStore;
|
||||
import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
|
||||
|
||||
class InProcessRateLimitStoreTest {
|
||||
|
||||
@Test
|
||||
void firstNConsumesAllowed() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
assertTrue(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed(), "i=" + i);
|
||||
}
|
||||
assertFalse(store.tryConsume("k", 5, Duration.ofSeconds(60)).allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void remainingTokensDecrements() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
RateLimitDecision d1 = store.tryConsume("k", 5, Duration.ofSeconds(60));
|
||||
RateLimitDecision d2 = store.tryConsume("k", 5, Duration.ofSeconds(60));
|
||||
assertTrue(d1.allowed());
|
||||
assertTrue(d2.allowed());
|
||||
assertEquals(4, d1.remainingTokens());
|
||||
assertEquals(3, d2.remainingTokens());
|
||||
}
|
||||
|
||||
@Test
|
||||
void refillRestoresTokens() throws InterruptedException {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
// Capacity 2 with smooth refill over 100 ms -> ~1 token per 50 ms.
|
||||
for (int i = 0; i < 2; i++) {
|
||||
assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
}
|
||||
assertFalse(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
Thread.sleep(150);
|
||||
assertTrue(store.tryConsume("k", 2, Duration.ofMillis(100)).allowed());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deniedConsumeReportsWaitNanos() {
|
||||
RateLimitStore store = new InProcessRateLimitStore();
|
||||
assertTrue(store.tryConsume("wait", 1, Duration.ofSeconds(10)).allowed());
|
||||
RateLimitDecision denied = store.tryConsume("wait", 1, Duration.ofSeconds(10));
|
||||
assertFalse(denied.allowed());
|
||||
assertTrue(denied.nanosToWaitForRefill() > 0L);
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package stirling.software.common.cluster.inprocess;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.common.cluster.FileStore;
|
||||
|
||||
class LocalDiskFileStoreTest {
|
||||
|
||||
@Test
|
||||
void storeRetrieveSizeDeleteExistsRoundTrip(@TempDir Path dir) throws IOException {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
byte[] payload = "hello-bytes".getBytes();
|
||||
|
||||
FileStore.Stored stored = store.store(new ByteArrayInputStream(payload), "x.txt");
|
||||
assertEquals(payload.length, stored.size());
|
||||
assertTrue(store.exists(stored.fileId()));
|
||||
assertEquals(payload.length, store.size(stored.fileId()));
|
||||
assertArrayEquals(payload, store.retrieveBytes(stored.fileId()));
|
||||
|
||||
assertTrue(store.delete(stored.fileId()));
|
||||
assertFalse(store.exists(stored.fileId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void traversalIdsAreRejected(@TempDir Path dir) {
|
||||
LocalDiskFileStore store = new LocalDiskFileStore(dir.toString());
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("../foo"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a/b"));
|
||||
assertThrows(IllegalArgumentException.class, () -> store.resolve("a\\b"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package stirling.software.common.jpdfium;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
|
||||
class JPDFiumSmokeTest {
|
||||
|
||||
@Test
|
||||
void opensExamplePdfAndReadsPageCount(@TempDir Path tmp) throws IOException {
|
||||
Path pdf = tmp.resolve("example.pdf");
|
||||
try (InputStream in = getClass().getResourceAsStream("/example.pdf")) {
|
||||
assertNotNull(in, "example.pdf must exist under src/test/resources");
|
||||
Files.copy(in, pdf);
|
||||
}
|
||||
|
||||
try (PdfDocument doc = PdfDocument.open(pdf)) {
|
||||
assertTrue(
|
||||
doc.pageCount() >= 1,
|
||||
"PdfDocument should report at least one page for example.pdf");
|
||||
}
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
|
||||
|
||||
class FileStorageDelegationTest {
|
||||
|
||||
@Test
|
||||
void storeBytesThenRetrieveBytesRoundTripsThroughFileStore(@TempDir Path tempDir)
|
||||
throws IOException {
|
||||
FileStorage fs =
|
||||
new FileStorage(
|
||||
mock(FileOrUploadService.class),
|
||||
new LocalDiskFileStore(tempDir.toString()));
|
||||
byte[] payload = "round-trip".getBytes();
|
||||
String id = fs.storeBytes(payload, "x.bin");
|
||||
assertArrayEquals(payload, fs.retrieveBytes(id));
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package stirling.software.common.service;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -13,29 +14,30 @@ import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.common.cluster.inprocess.LocalDiskFileStore;
|
||||
|
||||
class FileStorageTest {
|
||||
|
||||
@TempDir Path tempDir;
|
||||
|
||||
@Mock private FileOrUploadService fileOrUploadService;
|
||||
|
||||
@InjectMocks private FileStorage fileStorage;
|
||||
private FileStorage fileStorage;
|
||||
|
||||
private MultipartFile mockFile;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
void setUp() throws IOException {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
ReflectionTestUtils.setField(fileStorage, "tempDirPath", tempDir.toString());
|
||||
fileStorage =
|
||||
new FileStorage(fileOrUploadService, new LocalDiskFileStore(tempDir.toString()));
|
||||
|
||||
// Create a mock MultipartFile
|
||||
mockFile = mock(MultipartFile.class);
|
||||
@@ -47,17 +49,7 @@ class FileStorageTest {
|
||||
void testStoreFile() throws IOException {
|
||||
// Arrange
|
||||
byte[] fileContent = "Test PDF content".getBytes();
|
||||
when(mockFile.getBytes()).thenReturn(fileContent);
|
||||
|
||||
// Set up mock to handle transferTo by writing the file
|
||||
doAnswer(
|
||||
invocation -> {
|
||||
java.io.File file = invocation.getArgument(0);
|
||||
Files.write(file.toPath(), fileContent);
|
||||
return null;
|
||||
})
|
||||
.when(mockFile)
|
||||
.transferTo(any(java.io.File.class));
|
||||
when(mockFile.getInputStream()).thenReturn(new ByteArrayInputStream(fileContent));
|
||||
|
||||
// Act
|
||||
String fileId = fileStorage.storeFile(mockFile);
|
||||
@@ -65,7 +57,7 @@ class FileStorageTest {
|
||||
// Assert
|
||||
assertNotNull(fileId);
|
||||
assertTrue(Files.exists(tempDir.resolve(fileId)));
|
||||
verify(mockFile).transferTo(any(java.io.File.class));
|
||||
assertArrayEquals(fileContent, Files.readAllBytes(tempDir.resolve(fileId)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -247,11 +239,11 @@ class FileStorageTest {
|
||||
filesBefore = s.count();
|
||||
}
|
||||
|
||||
// Act + Assert: IOException must propagate out — not be swallowed.
|
||||
// Act + Assert: IOException must propagate out - not be swallowed.
|
||||
assertThrows(
|
||||
IOException.class, () -> fileStorage.storeFromResource(flakyResource, "n.pdf"));
|
||||
|
||||
// Assert: no partial file lingers under the storage directory — the finally
|
||||
// Assert: no partial file lingers under the storage directory - the finally
|
||||
// branch's deleteIfExists must have cleaned it up.
|
||||
long filesAfter;
|
||||
try (Stream<Path> s = Files.list(tempDir)) {
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.cluster.inprocess.InProcessClusterBackplane;
|
||||
import stirling.software.common.cluster.inprocess.InProcessJobStore;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
|
||||
class TaskManagerJobStoreDelegationTest {
|
||||
|
||||
@Mock private FileStorage fileStorage;
|
||||
|
||||
private InProcessJobStore jobStore;
|
||||
private ClusterBackplane backplane;
|
||||
private TaskManager taskManager;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
jobStore = spy(new InProcessJobStore());
|
||||
backplane = new InProcessClusterBackplane(new ApplicationProperties());
|
||||
taskManager = new TaskManager(fileStorage, jobStore, backplane);
|
||||
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createTaskWritesPendingEntry() {
|
||||
taskManager.createTask("job-1");
|
||||
JobStoreEntry entry = jobStore.get("job-1").orElseThrow();
|
||||
assertEquals(JobState.PENDING, entry.state());
|
||||
assertEquals(backplane.localNodeId(), entry.owningNodeId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void setCompleteFlipsToComplete() {
|
||||
taskManager.createTask("job-2");
|
||||
taskManager.setResult("job-2", "ok");
|
||||
taskManager.setComplete("job-2");
|
||||
JobStoreEntry entry = jobStore.get("job-2").orElseThrow();
|
||||
assertEquals(JobState.COMPLETE, entry.state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void setErrorFlipsToFailed() {
|
||||
taskManager.createTask("job-3");
|
||||
taskManager.setError("job-3", "boom");
|
||||
JobStoreEntry entry = jobStore.get("job-3").orElseThrow();
|
||||
assertEquals(JobState.FAILED, entry.state());
|
||||
assertEquals("boom", entry.error());
|
||||
}
|
||||
|
||||
@Test
|
||||
void cleanupOldJobsIsNoopWhenBackplaneIsNotInProcess() {
|
||||
ClusterBackplane mockedValkeyBackplane =
|
||||
new ClusterBackplane() {
|
||||
@Override
|
||||
public boolean isHealthy() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String backplaneType() {
|
||||
return "valkey";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String localNodeId() {
|
||||
return "node-1";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shouldRunLocalCleanup() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
TaskManager tm = new TaskManager(fileStorage, jobStore, mockedValkeyBackplane);
|
||||
ReflectionTestUtils.setField(tm, "jobResultExpiryMinutes", 30);
|
||||
tm.createTask("job-4");
|
||||
tm.setComplete("job-4");
|
||||
ageJobPastExpiry(tm, "job-4");
|
||||
tm.cleanupOldJobs();
|
||||
// cleanup must short-circuit before touching jobStore in cluster mode; the backplane
|
||||
// TTL owns expiry there. If the gate fired correctly, delete is never called.
|
||||
verify(jobStore, never()).delete(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void cleanupOldJobsDeletesFromJobStoreWhenBackplaneIsInProcess() {
|
||||
taskManager.createTask("job-5");
|
||||
taskManager.setComplete("job-5");
|
||||
ageJobPastExpiry(taskManager, "job-5");
|
||||
taskManager.cleanupOldJobs();
|
||||
verify(jobStore).delete("job-5");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static void ageJobPastExpiry(TaskManager tm, String jobId) {
|
||||
var jobResults =
|
||||
(java.util.Map<String, JobResult>) ReflectionTestUtils.getField(tm, "jobResults");
|
||||
JobResult result = jobResults.get(jobId);
|
||||
ReflectionTestUtils.setField(result, "completedAt", LocalDateTime.now().minusHours(2));
|
||||
ReflectionTestUtils.setField(result, "complete", true);
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,28 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import stirling.software.common.cluster.ClusterBackplane;
|
||||
import stirling.software.common.cluster.JobStore;
|
||||
import stirling.software.common.cluster.JobStoreEntry;
|
||||
import stirling.software.common.cluster.JobStoreEntry.JobState;
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
import stirling.software.common.model.job.JobStats;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
@@ -22,6 +30,8 @@ import stirling.software.common.model.job.ResultFile;
|
||||
class TaskManagerTest {
|
||||
|
||||
@Mock private FileStorage fileStorage;
|
||||
@Mock private JobStore jobStore;
|
||||
@Mock private ClusterBackplane clusterBackplane;
|
||||
|
||||
@InjectMocks private TaskManager taskManager;
|
||||
|
||||
@@ -30,6 +40,10 @@ class TaskManagerTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
closeable = MockitoAnnotations.openMocks(this);
|
||||
// Treat the backplane as in-process so cleanupOldJobs is not short-circuited.
|
||||
lenient().when(clusterBackplane.backplaneType()).thenReturn("inprocess");
|
||||
lenient().when(clusterBackplane.localNodeId()).thenReturn("test-node");
|
||||
lenient().when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(true);
|
||||
ReflectionTestUtils.setField(taskManager, "jobResultExpiryMinutes", 30);
|
||||
}
|
||||
|
||||
@@ -270,6 +284,33 @@ class TaskManagerTest {
|
||||
verify(fileStorage).deleteFile("file-id");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCleanupOldJobs_NoOpWhenBackplaneOwnsExpiry() {
|
||||
// When the backplane reports it should NOT run local cleanup (e.g. a distributed
|
||||
// backplane with its own TTL), the cleanup loop must leave local state untouched.
|
||||
when(clusterBackplane.shouldRunLocalCleanup()).thenReturn(false);
|
||||
|
||||
// Seed an old completed job that would normally be removed.
|
||||
String oldJobId = "old-job-distributed";
|
||||
taskManager.createTask(oldJobId);
|
||||
JobResult oldJob = taskManager.getJobResult(oldJobId);
|
||||
ReflectionTestUtils.setField(oldJob, "completedAt", LocalDateTime.now().minusHours(1));
|
||||
ReflectionTestUtils.setField(oldJob, "complete", true);
|
||||
|
||||
Map<String, JobResult> jobResultsMap =
|
||||
(Map<String, JobResult>) ReflectionTestUtils.getField(taskManager, "jobResults");
|
||||
assertNotNull(jobResultsMap);
|
||||
assertTrue(jobResultsMap.containsKey(oldJobId));
|
||||
|
||||
// Act
|
||||
taskManager.cleanupOldJobs();
|
||||
|
||||
// Assert: nothing was removed locally, and no jobStore.delete was issued.
|
||||
assertTrue(jobResultsMap.containsKey(oldJobId));
|
||||
verify(jobStore, never()).delete(anyString());
|
||||
verify(fileStorage, never()).deleteFile(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testShutdown() {
|
||||
// This mainly tests that the shutdown method doesn't throw exceptions
|
||||
@@ -310,4 +351,33 @@ class TaskManagerTest {
|
||||
// Assert
|
||||
assertFalse(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWriteThroughOnUpdate() {
|
||||
// Mutating calls must write through to the injected JobStore.
|
||||
String jobId = "write-through-job";
|
||||
taskManager.createTask(jobId);
|
||||
taskManager.setResult(jobId, "done");
|
||||
|
||||
ArgumentCaptor<JobStoreEntry> captor = ArgumentCaptor.forClass(JobStoreEntry.class);
|
||||
verify(jobStore, atLeast(2)).put(captor.capture(), any());
|
||||
|
||||
JobStoreEntry last = captor.getValue();
|
||||
assertEquals(jobId, last.jobId());
|
||||
assertEquals(JobState.COMPLETE, last.state());
|
||||
assertEquals("test-node", last.owningNodeId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFindJobKeyByFileId_FallsBackToJobStore() {
|
||||
// When the file id is not in the local map, TaskManager delegates to JobStore.
|
||||
String fileId = "remote-file-id";
|
||||
String expectedJobKey = "remote-job-key";
|
||||
when(jobStore.findJobIdByFileId(fileId)).thenReturn(Optional.of(expectedJobKey));
|
||||
|
||||
String actual = taskManager.findJobKeyByFileId(fileId);
|
||||
|
||||
assertEquals(expectedJobKey, actual);
|
||||
verify(jobStore).findJobIdByFileId(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
+20
-11
@@ -162,7 +162,8 @@ bootJar {
|
||||
manifest {
|
||||
attributes(
|
||||
'Implementation-Title': 'Stirling-PDF',
|
||||
'Implementation-Version': project.version
|
||||
'Implementation-Version': project.version,
|
||||
'Enable-Native-Access': 'ALL-UNNAMED'
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -175,6 +176,9 @@ springBoot {
|
||||
// Frontend build tasks - only enabled with -PbuildWithFrontend=true
|
||||
def buildWithFrontend = project.hasProperty('buildWithFrontend') && project.property('buildWithFrontend') == 'true'
|
||||
def buildPrototypes = project.hasProperty('prototypesMode') && project.property('prototypesMode') == 'true'
|
||||
// Workspace root holds package.json and node_modules (shared across editor /
|
||||
// future portal). Editor-specific paths (src, public, dist, tauri) live one
|
||||
// level deeper under frontend/editor/.
|
||||
|
||||
// Vite mode: -PprototypesMode > -PfrontendMode > enableSaas > disableAdditional > proprietary.
|
||||
def frontendModeOverride = project.findProperty('frontendMode')?.toString()?.toLowerCase()
|
||||
@@ -192,8 +196,13 @@ if (buildPrototypes) {
|
||||
}
|
||||
def frontendBuildTask = "frontend:build:${frontendMode}"
|
||||
|
||||
// Workspace root holds package.json and node_modules (shared across editor /
|
||||
// future portal). Editor-specific paths (src, public, dist, tauri) live one
|
||||
// level deeper under frontend/editor/. When the portal lands as an embedded
|
||||
// app, add a sibling frontendPortalDir / frontendPortalDistDir alongside.
|
||||
def frontendDir = file('../../frontend')
|
||||
def frontendDistDir = file('../../frontend/dist')
|
||||
def frontendEditorDir = file('../../frontend/editor')
|
||||
def frontendEditorDistDir = file('../../frontend/editor/dist')
|
||||
def resourcesStaticDir = file('src/main/resources/static')
|
||||
def generatedFrontendPaths = [
|
||||
'assets',
|
||||
@@ -258,13 +267,13 @@ tasks.register('npmBuild', Exec) {
|
||||
doNotTrackState("Frontend build depends on untracked npmInstall task")
|
||||
enabled = buildWithFrontend
|
||||
group = 'frontend'
|
||||
description = 'Build frontend application'
|
||||
description = 'Build editor frontend application'
|
||||
workingDir file('../..')
|
||||
commandLine = ['task', frontendBuildTask]
|
||||
inputs.dir(new File(frontendDir, 'src'))
|
||||
inputs.dir(new File(frontendDir, 'public'))
|
||||
inputs.dir(new File(frontendEditorDir, 'src'))
|
||||
inputs.dir(new File(frontendEditorDir, 'public'))
|
||||
inputs.file(new File(frontendDir, 'package.json'))
|
||||
outputs.dir(frontendDistDir)
|
||||
outputs.dir(frontendEditorDistDir)
|
||||
|
||||
// Show live output
|
||||
standardOutput = System.out
|
||||
@@ -275,17 +284,17 @@ tasks.register('npmBuild', Exec) {
|
||||
environment 'VITE_API_BASE_URL', '/'
|
||||
|
||||
doFirst {
|
||||
println "Building frontend application for production (mode=${frontendMode}, VITE_API_BASE_URL=/)"
|
||||
println "Building editor frontend application for production (mode=${frontendMode}, VITE_API_BASE_URL=/)"
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('copyFrontendAssets', Copy) {
|
||||
enabled = buildWithFrontend
|
||||
group = 'frontend'
|
||||
description = 'Copy frontend build to static resources'
|
||||
description = 'Copy editor frontend build to static resources'
|
||||
dependsOn npmBuild
|
||||
dependsOn cleanFrontendAssets
|
||||
from(frontendDistDir) {
|
||||
from(frontendEditorDistDir) {
|
||||
// Exclude files that conflict with backend static resources
|
||||
exclude 'robots.txt' // Backend already has this
|
||||
exclude 'favicon.ico' // Backend already has this
|
||||
@@ -293,7 +302,7 @@ tasks.register('copyFrontendAssets', Copy) {
|
||||
into resourcesStaticDir
|
||||
duplicatesStrategy = DuplicatesStrategy.INCLUDE // Let frontend overwrite when needed
|
||||
doFirst {
|
||||
println "Copying frontend build from ${frontendDistDir} to ${resourcesStaticDir}..."
|
||||
println "Copying frontend build from ${frontendEditorDistDir} to ${resourcesStaticDir}..."
|
||||
println "Backend static resources will be preserved"
|
||||
}
|
||||
doLast {
|
||||
@@ -325,7 +334,7 @@ tasks.named('copyFrontendAssets').configure {
|
||||
}
|
||||
|
||||
if (buildWithFrontend) {
|
||||
println "Frontend build enabled - JAR will include React frontend (mode=${frontendMode})"
|
||||
println "Editor frontend build enabled - JAR will include React frontend (mode=${frontendMode})"
|
||||
processResources.dependsOn copyFrontendAssets
|
||||
} else {
|
||||
println "Frontend build disabled - JAR will be backend-only with API landing page"
|
||||
|
||||
+166
-64
@@ -3,13 +3,14 @@ package stirling.software.SPDF.controller.api;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.multipdf.PDFMergerUtility;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
|
||||
@@ -47,6 +48,11 @@ import stirling.software.common.util.PdfErrorUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfMerge;
|
||||
import stirling.software.jpdfium.doc.Bookmark;
|
||||
import stirling.software.jpdfium.doc.PdfBookmarkEditor;
|
||||
import stirling.software.jpdfium.doc.PdfBookmarkEditor.BookmarkTree;
|
||||
|
||||
@GeneralApi
|
||||
@Slf4j
|
||||
@@ -57,7 +63,6 @@ public class MergeController {
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// Merges a list of PDDocument objects into a single PDDocument
|
||||
public PDDocument mergeDocuments(List<PDDocument> documents) throws IOException {
|
||||
PDDocument mergedDoc = pdfDocumentFactory.createNewDocument();
|
||||
boolean success = false;
|
||||
@@ -76,11 +81,8 @@ public class MergeController {
|
||||
}
|
||||
}
|
||||
|
||||
// Re-order files to match the explicit order provided by the front-end.
|
||||
// fileOrder is newline-delimited original filenames in the desired order.
|
||||
private static MultipartFile[] reorderFilesByProvidedOrder(
|
||||
MultipartFile[] files, String fileOrder) {
|
||||
// Split by various line endings and trim each entry
|
||||
String[] desired =
|
||||
stirling.software.common.util.RegexPatternUtils.getInstance()
|
||||
.getNewlineSplitPattern()
|
||||
@@ -107,7 +109,6 @@ public class MergeController {
|
||||
return ordered.toArray(new MultipartFile[0]);
|
||||
}
|
||||
|
||||
// Returns a comparator for sorting MultipartFile arrays based on the given sort type
|
||||
private Comparator<MultipartFile> getSortComparator(String sortType) {
|
||||
return switch (sortType) {
|
||||
case "byFileName" ->
|
||||
@@ -155,18 +156,16 @@ public class MergeController {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
case "orderProvided" -> (file1, file2) -> 0; // Default is the order provided
|
||||
default -> (file1, file2) -> 0; // Default is the order provided
|
||||
case "orderProvided" -> (file1, file2) -> 0;
|
||||
default -> (file1, file2) -> 0;
|
||||
};
|
||||
}
|
||||
|
||||
// Parse client file IDs from JSON string
|
||||
private String[] parseClientFileIds(String clientFileIds) {
|
||||
if (clientFileIds == null || clientFileIds.trim().isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
try {
|
||||
// Simple JSON array parsing - remove brackets and split by comma
|
||||
String trimmed = clientFileIds.trim();
|
||||
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||
String inside = trimmed.substring(1, trimmed.length() - 1).trim();
|
||||
@@ -186,39 +185,29 @@ public class MergeController {
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
// Adds a table of contents to the merged document using filenames as chapter titles
|
||||
private void addTableOfContents(PDDocument mergedDocument, MultipartFile[] files) {
|
||||
// Create the document outline
|
||||
PDDocumentOutline outline = new PDDocumentOutline();
|
||||
mergedDocument.getDocumentCatalog().setDocumentOutline(outline);
|
||||
|
||||
int pageIndex = 0; // Current page index in the merged document
|
||||
|
||||
// Iterate through the original files
|
||||
int pageIndex = 0;
|
||||
for (MultipartFile file : files) {
|
||||
// Get the filename without extension to use as bookmark title
|
||||
String filename = file.getOriginalFilename();
|
||||
String title = GeneralUtils.removeExtension(filename);
|
||||
|
||||
// Create an outline item for this file
|
||||
PDOutlineItem item = new PDOutlineItem();
|
||||
item.setTitle(title);
|
||||
|
||||
// Set the destination to the first page of this file in the merged document
|
||||
if (pageIndex < mergedDocument.getNumberOfPages()) {
|
||||
PDPage page = mergedDocument.getPage(pageIndex);
|
||||
item.setDestination(page);
|
||||
}
|
||||
|
||||
// Add the item to the outline
|
||||
outline.addLast(item);
|
||||
|
||||
// Increment page index for the next file
|
||||
try (PDDocument doc = pdfDocumentFactory.load(file)) {
|
||||
pageIndex += doc.getNumberOfPages();
|
||||
} catch (IOException e) {
|
||||
ExceptionUtils.logException("document loading for TOC generation", e);
|
||||
pageIndex++; // Increment by at least one if we can't determine page count
|
||||
pageIndex++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -236,7 +225,6 @@ public class MergeController {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to XMP metadata if Info dates are missing
|
||||
PDMetadata metadata = doc.getDocumentCatalog().getMetadata();
|
||||
if (metadata != null) {
|
||||
try (InputStream is = metadata.createInputStream()) {
|
||||
@@ -287,7 +275,7 @@ public class MergeController {
|
||||
@ModelAttribute MergePdfsRequest request,
|
||||
@RequestParam(value = "fileOrder", required = false) String fileOrder)
|
||||
throws IOException {
|
||||
List<File> filesToDelete = new ArrayList<>(); // List of temporary files to delete
|
||||
List<File> filesToDelete = new ArrayList<>();
|
||||
TempFile outputTempFile = null;
|
||||
|
||||
boolean removeCertSign = Boolean.TRUE.equals(request.getRemoveCertSign());
|
||||
@@ -298,48 +286,35 @@ public class MergeController {
|
||||
files = new MultipartFile[0];
|
||||
}
|
||||
|
||||
// If front-end provided explicit visible order, honor it and override backend sorting
|
||||
if (fileOrder != null && !fileOrder.isBlank()) {
|
||||
log.info("Reordering files based on fileOrder parameter");
|
||||
files = reorderFilesByProvidedOrder(files, fileOrder);
|
||||
} else {
|
||||
log.info("Sorting files based on sortType: {}", request.getSortType());
|
||||
Arrays.sort(
|
||||
files,
|
||||
getSortComparator(
|
||||
request.getSortType())); // Sort files based on requested sort type
|
||||
Arrays.sort(files, getSortComparator(request.getSortType()));
|
||||
}
|
||||
|
||||
try (TempFile mt = new TempFile(tempFileManager, ".pdf")) {
|
||||
|
||||
PDFMergerUtility mergerUtility = new PDFMergerUtility();
|
||||
long totalSize = 0;
|
||||
List<Path> inputPaths = new ArrayList<>(files.length);
|
||||
List<Integer> invalidIndexes = new ArrayList<>();
|
||||
for (int index = 0; index < files.length; index++) {
|
||||
MultipartFile multipartFile = files[index];
|
||||
totalSize += multipartFile.getSize();
|
||||
File tempFile =
|
||||
tempFileManager.convertMultipartFileToFile(
|
||||
multipartFile); // Convert MultipartFile to File
|
||||
filesToDelete.add(tempFile); // Add temp file to the list for later deletion
|
||||
File tempFile = tempFileManager.convertMultipartFileToFile(multipartFile);
|
||||
filesToDelete.add(tempFile);
|
||||
inputPaths.add(tempFile.toPath());
|
||||
|
||||
// Pre-validate each PDF so we can report which one(s) are broken
|
||||
// Use the original MultipartFile to avoid deleting the tempFile during validation
|
||||
try (PDDocument ignored = pdfDocumentFactory.load(multipartFile)) {
|
||||
// OK
|
||||
} catch (IOException e) {
|
||||
try (PdfDocument ignored = PdfDocument.open(tempFile.toPath())) {
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("PDF pre-validate", e);
|
||||
invalidIndexes.add(index);
|
||||
}
|
||||
mergerUtility.addSource(tempFile); // Add source file to the merger utility
|
||||
}
|
||||
|
||||
mergerUtility.setDestinationFileName(mt.getFile().getAbsolutePath());
|
||||
|
||||
int[] pageCounts;
|
||||
try {
|
||||
mergerUtility.mergeDocuments(
|
||||
pdfDocumentFactory.getStreamCacheFunction(
|
||||
totalSize)); // Merge the documents
|
||||
pageCounts =
|
||||
mergeWithJpdfium(inputPaths, files, generateToc, mt.getFile().toPath());
|
||||
} catch (IOException e) {
|
||||
ExceptionUtils.logException("PDF merge", e);
|
||||
if (PdfErrorUtils.isCorruptedPdfError(e)) {
|
||||
@@ -348,10 +323,26 @@ public class MergeController {
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Load the merged PDF document and operate on it inside try-with-resources
|
||||
try (PDDocument mergedDocument = pdfDocumentFactory.load(mt.getFile())) {
|
||||
// Remove signatures if removeCertSign is true
|
||||
if (removeCertSign) {
|
||||
boolean sigFlattenNeeded = false;
|
||||
if (removeCertSign) {
|
||||
try (PdfDocument check = PdfDocument.open(mt.getFile().toPath())) {
|
||||
sigFlattenNeeded = !check.signatures().isEmpty();
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"JPDFium signature pre-check failed; falling back to PDFBox flatten:"
|
||||
+ " {}",
|
||||
e.getMessage());
|
||||
sigFlattenNeeded = true;
|
||||
}
|
||||
if (!sigFlattenNeeded) {
|
||||
log.info(
|
||||
"removeCertSign requested but merged document has no signature"
|
||||
+ " fields; skipping PDFBox flatten pass");
|
||||
}
|
||||
}
|
||||
|
||||
if (sigFlattenNeeded) {
|
||||
try (PDDocument mergedDocument = pdfDocumentFactory.load(mt.getFile())) {
|
||||
PDDocumentCatalog catalog = mergedDocument.getDocumentCatalog();
|
||||
PDAcroForm acroForm = catalog.getAcroForm();
|
||||
if (acroForm != null) {
|
||||
@@ -359,24 +350,26 @@ public class MergeController {
|
||||
acroForm.getFields().stream()
|
||||
.filter(PDSignatureField.class::isInstance)
|
||||
.toList();
|
||||
|
||||
if (!fieldsToRemove.isEmpty()) {
|
||||
acroForm.flatten(
|
||||
fieldsToRemove,
|
||||
false); // Flatten the fields, effectively removing them
|
||||
acroForm.flatten(fieldsToRemove, false);
|
||||
}
|
||||
}
|
||||
outputTempFile = new TempFile(tempFileManager, ".pdf");
|
||||
try {
|
||||
mergedDocument.save(outputTempFile.getFile());
|
||||
} catch (Exception e) {
|
||||
outputTempFile.close();
|
||||
outputTempFile = null;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// Add table of contents if generateToc is true
|
||||
if (generateToc && files.length > 0) {
|
||||
addTableOfContents(mergedDocument, files);
|
||||
}
|
||||
|
||||
// Save the modified document to a temporary file
|
||||
} else {
|
||||
outputTempFile = new TempFile(tempFileManager, ".pdf");
|
||||
try {
|
||||
mergedDocument.save(outputTempFile.getFile());
|
||||
Files.copy(
|
||||
mt.getFile().toPath(),
|
||||
outputTempFile.getFile().toPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
} catch (Exception e) {
|
||||
outputTempFile.close();
|
||||
outputTempFile = null;
|
||||
@@ -395,7 +388,7 @@ public class MergeController {
|
||||
throw ex;
|
||||
} finally {
|
||||
for (File file : filesToDelete) {
|
||||
tempFileManager.deleteTempFile(file); // Delete temporary files
|
||||
tempFileManager.deleteTempFile(file);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,4 +398,113 @@ public class MergeController {
|
||||
|
||||
return WebResponseUtils.pdfFileToWebResponse(outputTempFile, mergedFileName);
|
||||
}
|
||||
|
||||
private int[] mergeWithJpdfium(
|
||||
List<Path> inputPaths, MultipartFile[] files, boolean generateToc, Path outputPath)
|
||||
throws IOException {
|
||||
if (inputPaths.isEmpty()) {
|
||||
try (PdfDocument empty = PdfDocument.open(new byte[0])) {
|
||||
empty.save(outputPath);
|
||||
} catch (Exception ignored) {
|
||||
Files.write(outputPath, new byte[0]);
|
||||
}
|
||||
return new int[0];
|
||||
}
|
||||
|
||||
List<PdfDocument> docs = new ArrayList<>(inputPaths.size());
|
||||
int[] pageCounts = new int[inputPaths.size()];
|
||||
int[] pageOffsets = new int[inputPaths.size()];
|
||||
List<List<Bookmark>> sourceBookmarks = new ArrayList<>(inputPaths.size());
|
||||
int runningOffset = 0;
|
||||
try {
|
||||
for (int i = 0; i < inputPaths.size(); i++) {
|
||||
Path p = inputPaths.get(i);
|
||||
PdfDocument doc = PdfDocument.open(p);
|
||||
docs.add(doc);
|
||||
pageCounts[i] = doc.pageCount();
|
||||
pageOffsets[i] = runningOffset;
|
||||
sourceBookmarks.add(doc.bookmarks());
|
||||
runningOffset += pageCounts[i];
|
||||
}
|
||||
|
||||
BookmarkTree combinedTree =
|
||||
buildCombinedBookmarkTree(files, pageOffsets, sourceBookmarks, generateToc);
|
||||
|
||||
try (PdfDocument merged = PdfMerge.merge(docs)) {
|
||||
if (combinedTree.entries().isEmpty()) {
|
||||
merged.save(outputPath);
|
||||
} else {
|
||||
PdfBookmarkEditor.setBookmarks(merged, combinedTree, outputPath);
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
throw new IOException("JPDFium merge failed", e);
|
||||
} finally {
|
||||
for (PdfDocument doc : docs) {
|
||||
try {
|
||||
doc.close();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
return pageCounts;
|
||||
}
|
||||
|
||||
private BookmarkTree buildCombinedBookmarkTree(
|
||||
MultipartFile[] files,
|
||||
int[] pageOffsets,
|
||||
List<List<Bookmark>> sourceBookmarks,
|
||||
boolean generateToc) {
|
||||
BookmarkTree.Builder builder = BookmarkTree.builder();
|
||||
|
||||
if (generateToc) {
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
String filename = files[i].getOriginalFilename();
|
||||
String title = GeneralUtils.removeExtension(filename);
|
||||
if (title == null || title.isBlank()) {
|
||||
title = "Document " + (i + 1);
|
||||
}
|
||||
builder.add(title, pageOffsets[i]);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < sourceBookmarks.size(); i++) {
|
||||
int offset = pageOffsets[i];
|
||||
for (Bookmark bm : sourceBookmarks.get(i)) {
|
||||
addBookmarkFlat(builder, bm, offset);
|
||||
}
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private void addBookmarkFlat(BookmarkTree.Builder builder, Bookmark root, int offset) {
|
||||
final int maxNodes = 100_000;
|
||||
java.util.Deque<Bookmark> stack = new java.util.ArrayDeque<>();
|
||||
java.util.Set<Bookmark> visited =
|
||||
java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>());
|
||||
stack.push(root);
|
||||
int processed = 0;
|
||||
while (!stack.isEmpty() && processed < maxNodes) {
|
||||
Bookmark bm = stack.pop();
|
||||
if (!visited.add(bm)) {
|
||||
continue;
|
||||
}
|
||||
processed++;
|
||||
if (bm.isInternal() && bm.title() != null) {
|
||||
builder.add(bm.title(), offset + bm.pageIndex());
|
||||
}
|
||||
if (bm.hasChildren()) {
|
||||
List<Bookmark> children = bm.children();
|
||||
for (int i = children.size() - 1; i >= 0; i--) {
|
||||
stack.push(children.get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (processed >= maxNodes) {
|
||||
log.warn(
|
||||
"Source bookmark traversal hit {}-node cap; remaining bookmarks dropped",
|
||||
maxNodes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+33
-26
@@ -3,6 +3,7 @@ package stirling.software.SPDF.controller.api;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -30,12 +31,13 @@ import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.GeneralApi;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfSplit;
|
||||
|
||||
@GeneralApi
|
||||
@Slf4j
|
||||
@@ -91,10 +93,14 @@ public class SplitPDFController {
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(outputTempFile.getPath()))) {
|
||||
if (hasForm) {
|
||||
writeSplitsViaReload(
|
||||
// JPDFium's FPDF_ImportPagesByIndex drops the AcroForm dictionary, which
|
||||
// breaks Fill Forms downstream. Fall back to the PDFBox load-and-remove
|
||||
// path so the AcroForm (with only fields whose widgets remain on kept
|
||||
// pages) is preserved.
|
||||
writeSplitsViaPdfBox(
|
||||
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
|
||||
} else {
|
||||
writeSplitsViaSharedSource(
|
||||
writeSplitsViaJpdfium(
|
||||
sourceTempFile.getFile(), pageNumbers, baseFilename, zipOut);
|
||||
}
|
||||
}
|
||||
@@ -109,7 +115,26 @@ public class SplitPDFController {
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSplitsViaReload(
|
||||
private void writeSplitsViaJpdfium(
|
||||
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
|
||||
throws IOException {
|
||||
try (PdfDocument sourceDoc = PdfDocument.open(source.toPath())) {
|
||||
int previousPageNumber = 0;
|
||||
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
|
||||
int splitPoint = pageNumbers.get(splitIndex);
|
||||
try (TempFile splitTemp = new TempFile(tempFileManager, ".pdf")) {
|
||||
try (PdfDocument splitDoc =
|
||||
PdfSplit.extractPageRange(sourceDoc, previousPageNumber, splitPoint)) {
|
||||
splitDoc.save(splitTemp.getPath());
|
||||
}
|
||||
writeEntry(zipOut, baseFilename, splitIndex + 1, splitTemp.getPath());
|
||||
}
|
||||
previousPageNumber = splitPoint + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSplitsViaPdfBox(
|
||||
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
|
||||
throws IOException {
|
||||
int previousPageNumber = 0;
|
||||
@@ -129,33 +154,15 @@ public class SplitPDFController {
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(splitDoc);
|
||||
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeSplitsViaSharedSource(
|
||||
File source, List<Integer> pageNumbers, String baseFilename, ZipOutputStream zipOut)
|
||||
private void writeEntry(ZipOutputStream zipOut, String baseFilename, int index, Path pdfPath)
|
||||
throws IOException {
|
||||
try (PDDocument sourceDoc = pdfDocumentFactory.load(source)) {
|
||||
int previousPageNumber = 0;
|
||||
for (int splitIndex = 0; splitIndex < pageNumbers.size(); splitIndex++) {
|
||||
int splitPoint = pageNumbers.get(splitIndex);
|
||||
try (PDDocument splitDoc =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDoc)) {
|
||||
for (int i = previousPageNumber; i <= splitPoint; i++) {
|
||||
splitDoc.addPage(sourceDoc.getPage(i));
|
||||
}
|
||||
previousPageNumber = splitPoint + 1;
|
||||
writeEntry(zipOut, baseFilename, splitIndex + 1, splitDoc);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + index + ".pdf"));
|
||||
Files.copy(pdfPath, zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
|
||||
private void writeEntry(ZipOutputStream zipOut, String baseFilename, int index, PDDocument doc)
|
||||
|
||||
+196
-128
@@ -1,6 +1,10 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -8,9 +12,7 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocumentOutline;
|
||||
import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -35,10 +37,13 @@ import stirling.software.common.model.PdfMetadata;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.service.PdfMetadataService;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfSplit;
|
||||
|
||||
@GeneralApi
|
||||
@Slf4j
|
||||
@@ -51,72 +56,36 @@ public class SplitPdfByChaptersController {
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
private static List<Bookmark> extractOutlineItems(
|
||||
PDDocument sourceDocument,
|
||||
PDOutlineItem current,
|
||||
List<Bookmark> bookmarks,
|
||||
PDOutlineItem nextParent,
|
||||
private static void collectBookmarks(
|
||||
List<stirling.software.jpdfium.doc.Bookmark> source,
|
||||
List<Bookmark> out,
|
||||
int level,
|
||||
int maxLevel)
|
||||
throws Exception {
|
||||
|
||||
while (current != null) {
|
||||
|
||||
String currentTitle = current.getTitle().replace("/", "");
|
||||
int firstPage =
|
||||
sourceDocument.getPages().indexOf(current.findDestinationPage(sourceDocument));
|
||||
PDOutlineItem child = current.getFirstChild();
|
||||
PDOutlineItem nextSibling = current.getNextSibling();
|
||||
int endPage;
|
||||
if (child != null && level < maxLevel) {
|
||||
endPage =
|
||||
sourceDocument
|
||||
.getPages()
|
||||
.indexOf(child.findDestinationPage(sourceDocument));
|
||||
} else if (nextSibling != null) {
|
||||
endPage =
|
||||
sourceDocument
|
||||
.getPages()
|
||||
.indexOf(nextSibling.findDestinationPage(sourceDocument));
|
||||
} else if (nextParent != null) {
|
||||
|
||||
endPage =
|
||||
sourceDocument
|
||||
.getPages()
|
||||
.indexOf(nextParent.findDestinationPage(sourceDocument));
|
||||
} else {
|
||||
endPage = -2;
|
||||
/*
|
||||
happens when we have something like this:
|
||||
Outline Item 2
|
||||
Outline Item 2.1
|
||||
Outline Item 2.1.1
|
||||
Outline Item 2.2
|
||||
Outline 2.2.1
|
||||
Outline 2.2.2 <--- this item neither has an immediate next parent nor an immediate next sibling
|
||||
Outline Item 3
|
||||
*/
|
||||
int maxLevel) {
|
||||
for (stirling.software.jpdfium.doc.Bookmark bm : source) {
|
||||
if (!bm.isInternal()) {
|
||||
continue;
|
||||
}
|
||||
if (!bookmarks.isEmpty()
|
||||
&& bookmarks.get(bookmarks.size() - 1).getEndPage() == -2
|
||||
&& firstPage
|
||||
>= bookmarks
|
||||
.get(bookmarks.size() - 1)
|
||||
.getStartPage()) { // for handling the above-mentioned case
|
||||
Bookmark previousBookmark = bookmarks.get(bookmarks.size() - 1);
|
||||
previousBookmark.setEndPage(firstPage);
|
||||
String title = bm.title() == null ? "" : bm.title().replace("/", "");
|
||||
int firstPage = Math.max(0, bm.pageIndex());
|
||||
out.add(new Bookmark(title, firstPage, -2));
|
||||
if (bm.hasChildren() && level < maxLevel) {
|
||||
collectBookmarks(bm.children(), out, level + 1, maxLevel);
|
||||
}
|
||||
bookmarks.add(new Bookmark(currentTitle, firstPage, endPage));
|
||||
|
||||
// Recursively process children
|
||||
if (child != null && level < maxLevel) {
|
||||
extractOutlineItems(
|
||||
sourceDocument, child, bookmarks, nextSibling, level + 1, maxLevel);
|
||||
}
|
||||
|
||||
current = nextSibling;
|
||||
}
|
||||
return bookmarks;
|
||||
}
|
||||
|
||||
private static void assignEndPages(List<Bookmark> bookmarks, int totalPages) {
|
||||
for (int i = 0; i < bookmarks.size(); i++) {
|
||||
Bookmark current = bookmarks.get(i);
|
||||
int next = -1;
|
||||
for (int j = i + 1; j < bookmarks.size(); j++) {
|
||||
if (bookmarks.get(j).getStartPage() >= current.getStartPage()) {
|
||||
next = bookmarks.get(j).getStartPage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
current.setEndPage(next == -1 ? totalPages : next);
|
||||
}
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
@@ -134,46 +103,41 @@ public class SplitPdfByChaptersController {
|
||||
MultipartFile file = request.getFileInput();
|
||||
|
||||
boolean includeMetadata = Boolean.TRUE.equals(request.getIncludeMetadata());
|
||||
Integer bookmarkLevel =
|
||||
request.getBookmarkLevel(); // levels start from 0 (top most bookmarks)
|
||||
Integer bookmarkLevel = request.getBookmarkLevel();
|
||||
if (bookmarkLevel < 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", "bookmark level");
|
||||
}
|
||||
|
||||
try (PDDocument sourceDocument = pdfDocumentFactory.load(file)) {
|
||||
PDDocumentOutline outline = sourceDocument.getDocumentCatalog().getDocumentOutline();
|
||||
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf")) {
|
||||
Files.copy(
|
||||
file.getInputStream(),
|
||||
sourceTempFile.getPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
if (outline == null) {
|
||||
log.warn("No outline found for {}", file.getOriginalFilename());
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.pdfBookmarksNotFound", "No PDF bookmarks/outline found in document");
|
||||
}
|
||||
List<Bookmark> bookmarks = new ArrayList<>();
|
||||
try {
|
||||
bookmarks =
|
||||
extractOutlineItems(
|
||||
sourceDocument,
|
||||
outline.getFirstChild(),
|
||||
bookmarks,
|
||||
outline.getFirstChild().getNextSibling(),
|
||||
0,
|
||||
bookmarkLevel);
|
||||
// to handle last page edge case
|
||||
bookmarks.get(bookmarks.size() - 1).setEndPage(sourceDocument.getNumberOfPages());
|
||||
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("outline extraction", e);
|
||||
throw e;
|
||||
int totalPages;
|
||||
try (PdfDocument sourceDocument = PdfDocument.open(sourceTempFile.getPath())) {
|
||||
totalPages = sourceDocument.pageCount();
|
||||
List<stirling.software.jpdfium.doc.Bookmark> roots = sourceDocument.bookmarks();
|
||||
if (roots == null || roots.isEmpty()) {
|
||||
log.warn("No outline found for {}", file.getOriginalFilename());
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.pdfBookmarksNotFound",
|
||||
"No PDF bookmarks/outline found in document");
|
||||
}
|
||||
collectBookmarks(roots, bookmarks, 0, bookmarkLevel);
|
||||
if (bookmarks.isEmpty()) {
|
||||
log.warn("No outline found for {}", file.getOriginalFilename());
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.pdfBookmarksNotFound",
|
||||
"No PDF bookmarks/outline found in document");
|
||||
}
|
||||
assignEndPages(bookmarks, totalPages);
|
||||
}
|
||||
|
||||
boolean allowDuplicates = Boolean.TRUE.equals(request.getAllowDuplicates());
|
||||
if (!allowDuplicates) {
|
||||
/*
|
||||
duplicates are generated when multiple bookmarks correspond to the same page,
|
||||
if the user doesn't want duplicates mergeBookmarksThatCorrespondToSamePage() method will merge the titles of all
|
||||
the bookmarks that correspond to the same page, and treat them as a single bookmark
|
||||
*/
|
||||
bookmarks = mergeBookmarksThatCorrespondToSamePage(bookmarks);
|
||||
}
|
||||
for (Bookmark bookmark : bookmarks) {
|
||||
@@ -184,7 +148,23 @@ public class SplitPdfByChaptersController {
|
||||
bookmark.getEndPage());
|
||||
}
|
||||
|
||||
TempFile zipTempFile = createZipFile(sourceDocument, bookmarks, includeMetadata);
|
||||
PdfMetadata metadata = null;
|
||||
boolean hasForm = false;
|
||||
if (includeMetadata) {
|
||||
try (PDDocument metaDoc = pdfDocumentFactory.load(sourceTempFile.getFile())) {
|
||||
metadata = pdfMetadataService.extractMetadataFromPdf(metaDoc);
|
||||
PDAcroForm acroForm = metaDoc.getDocumentCatalog().getAcroForm(null);
|
||||
hasForm = acroForm != null;
|
||||
}
|
||||
} else {
|
||||
try (PDDocument acroDoc = pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
|
||||
hasForm = acroDoc.getDocumentCatalog().getAcroForm(null) != null;
|
||||
}
|
||||
}
|
||||
|
||||
TempFile zipTempFile =
|
||||
createZipFile(
|
||||
sourceTempFile.getFile(), bookmarks, metadata, totalPages, hasForm);
|
||||
String filename = GeneralUtils.generateFilename(file.getOriginalFilename(), "");
|
||||
return WebResponseUtils.zipFileToWebResponse(zipTempFile, filename + ".zip");
|
||||
}
|
||||
@@ -216,43 +196,40 @@ public class SplitPdfByChaptersController {
|
||||
}
|
||||
|
||||
private TempFile createZipFile(
|
||||
PDDocument sourceDocument, List<Bookmark> bookmarks, boolean includeMetadata)
|
||||
File sourceFile,
|
||||
List<Bookmark> bookmarks,
|
||||
PdfMetadata metadata,
|
||||
int totalPages,
|
||||
boolean hasForm)
|
||||
throws Exception {
|
||||
PdfMetadata metadata =
|
||||
includeMetadata ? pdfMetadataService.extractMetadataFromPdf(sourceDocument) : null;
|
||||
String fileNumberFormatter = "%0" + (Integer.toString(bookmarks.size()).length()) + "d ";
|
||||
TempFile zipTempFile = new TempFile(tempFileManager, ".zip");
|
||||
try {
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
|
||||
if (hasForm) {
|
||||
// JPDFium's FPDF_ImportPagesByIndex drops the AcroForm dictionary. For form
|
||||
// PDFs, do the per-chapter extract via PDFBox so form fields survive the split.
|
||||
for (int i = 0; i < bookmarks.size(); i++) {
|
||||
Bookmark bookmark = bookmarks.get(i);
|
||||
try (PDDocument splitDocument = new PDDocument()) {
|
||||
boolean isSinglePage = (bookmark.getStartPage() == bookmark.getEndPage());
|
||||
|
||||
for (int pg = bookmark.getStartPage();
|
||||
pg < bookmark.getEndPage() + (isSinglePage ? 1 : 0);
|
||||
pg++) {
|
||||
PDPage page = sourceDocument.getPage(pg);
|
||||
splitDocument.addPage(page);
|
||||
log.debug("Adding page {} to split document", pg);
|
||||
}
|
||||
if (includeMetadata) {
|
||||
pdfMetadataService.setMetadataToPdf(splitDocument, metadata);
|
||||
}
|
||||
|
||||
// split files will be named as "[FILE_NUMBER] [BOOKMARK_TITLE].pdf"
|
||||
String fileName =
|
||||
String.format(Locale.ROOT, fileNumberFormatter, i)
|
||||
+ bookmark.getTitle()
|
||||
+ ".pdf";
|
||||
zipOut.putNextEntry(new ZipEntry(fileName));
|
||||
splitDocument.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
log.debug("Wrote split document {} to zip file", fileName);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("document splitting and saving", e);
|
||||
throw e;
|
||||
writeChapterViaPdfBox(
|
||||
sourceFile,
|
||||
bookmarks.get(i),
|
||||
i,
|
||||
fileNumberFormatter,
|
||||
metadata,
|
||||
zipOut,
|
||||
totalPages);
|
||||
}
|
||||
} else {
|
||||
try (PdfDocument sourceDocument = PdfDocument.open(sourceFile.toPath())) {
|
||||
for (int i = 0; i < bookmarks.size(); i++) {
|
||||
writeChapterViaJpdfium(
|
||||
sourceDocument,
|
||||
bookmarks.get(i),
|
||||
i,
|
||||
fileNumberFormatter,
|
||||
metadata,
|
||||
zipOut,
|
||||
totalPages);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -265,6 +242,97 @@ public class SplitPdfByChaptersController {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void writeChapterViaJpdfium(
|
||||
PdfDocument sourceDocument,
|
||||
Bookmark bookmark,
|
||||
int index,
|
||||
String fileNumberFormatter,
|
||||
PdfMetadata metadata,
|
||||
ZipOutputStream zipOut,
|
||||
int totalPages)
|
||||
throws Exception {
|
||||
int[] range = clampRange(bookmark, totalPages);
|
||||
int from = range[0];
|
||||
int to = range[1];
|
||||
try (TempFile splitTemp = new TempFile(tempFileManager, ".pdf")) {
|
||||
try (PdfDocument splitDoc = PdfSplit.extractPageRange(sourceDocument, from, to)) {
|
||||
splitDoc.save(splitTemp.getPath());
|
||||
}
|
||||
Path finalPath = splitTemp.getPath();
|
||||
TempFile metaTemp = null;
|
||||
try {
|
||||
if (metadata != null) {
|
||||
metaTemp = new TempFile(tempFileManager, ".pdf");
|
||||
try (PDDocument doc = pdfDocumentFactory.load(splitTemp.getFile())) {
|
||||
pdfMetadataService.setMetadataToPdf(doc, metadata);
|
||||
doc.save(metaTemp.getFile());
|
||||
}
|
||||
finalPath = metaTemp.getPath();
|
||||
}
|
||||
writeZipEntry(zipOut, fileNumberFormatter, index, bookmark.getTitle(), finalPath);
|
||||
} finally {
|
||||
if (metaTemp != null) {
|
||||
metaTemp.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeChapterViaPdfBox(
|
||||
File sourceFile,
|
||||
Bookmark bookmark,
|
||||
int index,
|
||||
String fileNumberFormatter,
|
||||
PdfMetadata metadata,
|
||||
ZipOutputStream zipOut,
|
||||
int totalPages)
|
||||
throws Exception {
|
||||
int[] range = clampRange(bookmark, totalPages);
|
||||
int from = range[0];
|
||||
int to = range[1];
|
||||
try (PDDocument doc = pdfDocumentFactory.load(sourceFile)) {
|
||||
for (int p = doc.getNumberOfPages() - 1; p >= 0; p--) {
|
||||
if (p < from || p > to) {
|
||||
doc.removePage(p);
|
||||
}
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(doc);
|
||||
if (metadata != null) {
|
||||
pdfMetadataService.setMetadataToPdf(doc, metadata);
|
||||
}
|
||||
String fileName =
|
||||
String.format(Locale.ROOT, fileNumberFormatter, index)
|
||||
+ bookmark.getTitle()
|
||||
+ ".pdf";
|
||||
zipOut.putNextEntry(new ZipEntry(fileName));
|
||||
doc.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
log.debug("Wrote split document {} to zip file", fileName);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeZipEntry(
|
||||
ZipOutputStream zipOut,
|
||||
String fileNumberFormatter,
|
||||
int index,
|
||||
String title,
|
||||
Path pdfPath)
|
||||
throws IOException {
|
||||
String fileName = String.format(Locale.ROOT, fileNumberFormatter, index) + title + ".pdf";
|
||||
zipOut.putNextEntry(new ZipEntry(fileName));
|
||||
Files.copy(pdfPath, zipOut);
|
||||
zipOut.closeEntry();
|
||||
log.debug("Wrote split document {} to zip file", fileName);
|
||||
}
|
||||
|
||||
private static int[] clampRange(Bookmark bookmark, int totalPages) {
|
||||
boolean isSinglePage = bookmark.getStartPage() == bookmark.getEndPage();
|
||||
int from = Math.min(Math.max(0, bookmark.getStartPage()), totalPages - 1);
|
||||
int rawEnd = isSinglePage ? bookmark.getEndPage() : bookmark.getEndPage() - 1;
|
||||
int to = Math.min(Math.max(from, rawEnd), totalPages - 1);
|
||||
return new int[] {from, to};
|
||||
}
|
||||
}
|
||||
|
||||
@Data
|
||||
|
||||
+142
-128
@@ -1,9 +1,9 @@
|
||||
package stirling.software.SPDF.controller.api;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
@@ -13,7 +13,6 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@@ -37,6 +36,8 @@ import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.jpdfium.PdfDocument;
|
||||
import stirling.software.jpdfium.PdfSplit;
|
||||
|
||||
@GeneralApi
|
||||
@Slf4j
|
||||
@@ -75,23 +76,27 @@ public class SplitPdfBySizeController {
|
||||
sourceTempFile.getPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
|
||||
try (PDDocument sourceDocument =
|
||||
pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
|
||||
boolean hasForm = sourceDocument.getDocumentCatalog().getAcroForm(null) != null;
|
||||
List<List<Integer>> ranges = computeRanges(request, sourceDocument);
|
||||
boolean hasForm;
|
||||
try (PDDocument acroDoc = pdfDocumentFactory.load(sourceTempFile.getFile(), true)) {
|
||||
hasForm = acroDoc.getDocumentCatalog().getAcroForm(null) != null;
|
||||
}
|
||||
|
||||
try (PdfDocument sourceDocument = PdfDocument.open(sourceTempFile.getPath())) {
|
||||
List<int[]> ranges = computeRanges(request, sourceDocument);
|
||||
|
||||
int fileIndex = 1;
|
||||
for (List<Integer> range : ranges) {
|
||||
if (range.isEmpty()) {
|
||||
for (int[] range : ranges) {
|
||||
if (range.length == 0) {
|
||||
continue;
|
||||
}
|
||||
if (hasForm) {
|
||||
writeRangeViaReload(
|
||||
sourceTempFile.getFile(), range, zipOut, filename, fileIndex++);
|
||||
} else {
|
||||
writeRangeViaSharedSource(
|
||||
sourceDocument, range, zipOut, filename, fileIndex++);
|
||||
}
|
||||
writeRange(
|
||||
sourceDocument,
|
||||
sourceTempFile.getFile(),
|
||||
range,
|
||||
zipOut,
|
||||
filename,
|
||||
fileIndex++,
|
||||
hasForm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,29 +109,54 @@ public class SplitPdfBySizeController {
|
||||
}
|
||||
}
|
||||
|
||||
private List<List<Integer>> computeRanges(
|
||||
SplitPdfBySizeOrCountRequest request, PDDocument sourceDocument) throws IOException {
|
||||
private List<int[]> computeRanges(SplitPdfBySizeOrCountRequest request, PdfDocument sourceDoc)
|
||||
throws IOException {
|
||||
int type = request.getSplitType();
|
||||
String value = request.getSplitValue();
|
||||
if (type == 0) {
|
||||
return computeSizeRanges(sourceDocument, GeneralUtils.convertSizeToBytes(value));
|
||||
return computeSizeRanges(sourceDoc, GeneralUtils.convertSizeToBytes(value));
|
||||
} else if (type == 1) {
|
||||
return computePageCountRanges(sourceDocument, Integer.parseInt(value));
|
||||
return computePageCountRanges(sourceDoc, Integer.parseInt(value));
|
||||
} else if (type == 2) {
|
||||
return computeDocCountRanges(sourceDocument, Integer.parseInt(value));
|
||||
return computeDocCountRanges(sourceDoc, Integer.parseInt(value));
|
||||
}
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", "split type: " + type);
|
||||
}
|
||||
|
||||
private void writeRangeViaReload(
|
||||
private void writeRange(
|
||||
PdfDocument sourceDoc,
|
||||
File sourceFile,
|
||||
List<Integer> keepIndices,
|
||||
int[] range,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename,
|
||||
int fileIndex,
|
||||
boolean hasForm)
|
||||
throws IOException {
|
||||
if (hasForm) {
|
||||
// JPDFium's FPDF_ImportPagesByIndex drops the AcroForm dictionary, breaking form
|
||||
// fields downstream. For form-bearing PDFs, do the extract via PDFBox so the
|
||||
// AcroForm survives (pruneOrphanedFormFields removes references to dropped pages).
|
||||
writeRangeViaPdfBox(sourceFile, range, zipOut, baseFilename, fileIndex);
|
||||
} else {
|
||||
try (TempFile splitTemp = new TempFile(tempFileManager, ".pdf")) {
|
||||
extractRangeToFile(sourceDoc, range, splitTemp.getPath());
|
||||
writeEntry(zipOut, baseFilename, fileIndex, splitTemp.getPath());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRangeViaPdfBox(
|
||||
File sourceFile,
|
||||
int[] range,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename,
|
||||
int fileIndex)
|
||||
throws IOException {
|
||||
Set<Integer> keep = new HashSet<>(keepIndices);
|
||||
Set<Integer> keep = new HashSet<>();
|
||||
for (int p : range) {
|
||||
keep.add(p);
|
||||
}
|
||||
try (PDDocument doc = pdfDocumentFactory.load(sourceFile)) {
|
||||
for (int i = doc.getNumberOfPages() - 1; i >= 0; i--) {
|
||||
if (!keep.contains(i)) {
|
||||
@@ -134,50 +164,41 @@ public class SplitPdfBySizeController {
|
||||
}
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(doc);
|
||||
writeEntry(zipOut, baseFilename, fileIndex, doc);
|
||||
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + fileIndex + ".pdf"));
|
||||
doc.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private void writeRangeViaSharedSource(
|
||||
PDDocument sourceDocument,
|
||||
List<Integer> keepIndices,
|
||||
ZipOutputStream zipOut,
|
||||
String baseFilename,
|
||||
int fileIndex)
|
||||
private void extractRangeToFile(PdfDocument sourceDoc, int[] range, Path outputPath)
|
||||
throws IOException {
|
||||
try (PDDocument doc =
|
||||
pdfDocumentFactory.createNewDocumentBasedOnOldDocument(sourceDocument)) {
|
||||
for (int p : keepIndices) {
|
||||
doc.addPage(sourceDocument.getPage(p));
|
||||
}
|
||||
writeEntry(zipOut, baseFilename, fileIndex, doc);
|
||||
int from = range[0];
|
||||
int to = range[range.length - 1];
|
||||
try (PdfDocument split = PdfSplit.extractPageRange(sourceDoc, from, to)) {
|
||||
split.save(outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeEntry(
|
||||
ZipOutputStream zipOut, String baseFilename, int fileIndex, PDDocument doc)
|
||||
ZipOutputStream zipOut, String baseFilename, int fileIndex, Path pdfPath)
|
||||
throws IOException {
|
||||
zipOut.putNextEntry(new ZipEntry(baseFilename + "_" + fileIndex + ".pdf"));
|
||||
doc.save(zipOut);
|
||||
Files.copy(pdfPath, zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
|
||||
/** Page-index ranges each output should contain. AcroForm overhead isn't modeled. */
|
||||
private List<List<Integer>> computeSizeRanges(PDDocument sourceDocument, long maxBytes)
|
||||
throws IOException {
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
List<Integer> currentRange = new ArrayList<>();
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
/** Returns contiguous page-index ranges fitting within {@code maxBytes}. */
|
||||
private List<int[]> computeSizeRanges(PdfDocument sourceDoc, long maxBytes) throws IOException {
|
||||
List<int[]> ranges = new ArrayList<>();
|
||||
int totalPages = sourceDoc.pageCount();
|
||||
int baseCheckFrequency = 5;
|
||||
|
||||
PDDocument scratch = new PDDocument();
|
||||
try {
|
||||
int rangeStart = 0;
|
||||
int rangeEnd = -1;
|
||||
try (TempFile probe = new TempFile(tempFileManager, ".pdf")) {
|
||||
File probeFile = probe.getFile();
|
||||
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
|
||||
PDPage page = sourceDocument.getPage(pageIndex);
|
||||
scratch.addPage(new PDPage(page.getCOSObject()));
|
||||
currentRange.add(pageIndex);
|
||||
|
||||
int pageAdded = currentRange.size();
|
||||
rangeEnd = pageIndex;
|
||||
int pageAdded = rangeEnd - rangeStart + 1;
|
||||
boolean shouldCheckSize =
|
||||
(pageAdded % baseCheckFrequency == 0)
|
||||
|| (pageIndex == totalPages - 1)
|
||||
@@ -185,117 +206,110 @@ public class SplitPdfBySizeController {
|
||||
if (!shouldCheckSize) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long actualSize;
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
scratch.save(out);
|
||||
actualSize = out.size();
|
||||
}
|
||||
long actualSize = saveRange(sourceDoc, rangeStart, rangeEnd, probeFile);
|
||||
|
||||
if (actualSize > maxBytes) {
|
||||
if (scratch.getNumberOfPages() > 1) {
|
||||
scratch.removePage(scratch.getNumberOfPages() - 1);
|
||||
currentRange.remove(currentRange.size() - 1);
|
||||
pageIndex--; // retry this page in the next chunk
|
||||
if (pageAdded > 1) {
|
||||
rangeEnd = pageIndex - 1;
|
||||
pageIndex--;
|
||||
}
|
||||
ranges.add(new ArrayList<>(currentRange));
|
||||
currentRange.clear();
|
||||
scratch.close();
|
||||
scratch = new PDDocument();
|
||||
ranges.add(buildRange(rangeStart, rangeEnd));
|
||||
rangeStart = rangeEnd + 1;
|
||||
rangeEnd = rangeStart - 1;
|
||||
} else if (pageIndex < totalPages - 1 && actualSize < maxBytes * 0.75) {
|
||||
int extraPagesAdded =
|
||||
lookAheadFit(scratch, sourceDocument, pageIndex, maxBytes);
|
||||
for (int i = 0; i < extraPagesAdded; i++) {
|
||||
int extra = pageIndex + 1 + i;
|
||||
scratch.addPage(new PDPage(sourceDocument.getPage(extra).getCOSObject()));
|
||||
currentRange.add(extra);
|
||||
}
|
||||
pageIndex += extraPagesAdded;
|
||||
int extra =
|
||||
lookAheadFit(
|
||||
sourceDoc,
|
||||
rangeStart,
|
||||
pageIndex,
|
||||
maxBytes,
|
||||
totalPages,
|
||||
probeFile);
|
||||
pageIndex += extra;
|
||||
rangeEnd = pageIndex;
|
||||
}
|
||||
}
|
||||
|
||||
if (!currentRange.isEmpty()) {
|
||||
ranges.add(new ArrayList<>(currentRange));
|
||||
}
|
||||
} finally {
|
||||
scratch.close();
|
||||
}
|
||||
if (rangeEnd >= rangeStart) {
|
||||
ranges.add(buildRange(rangeStart, rangeEnd));
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
/** Speculatively tries up to 5 next pages; returns how many fit under {@code maxBytes}. */
|
||||
private int lookAheadFit(PDDocument scratch, PDDocument source, int pageIndex, long maxBytes)
|
||||
private long saveRange(PdfDocument sourceDoc, int from, int to, File output)
|
||||
throws IOException {
|
||||
int totalPages = source.getNumberOfPages();
|
||||
int pagesToLookAhead = Math.min(5, totalPages - pageIndex - 1);
|
||||
if (pagesToLookAhead == 0) {
|
||||
return 0;
|
||||
try (PdfDocument split = PdfSplit.extractPageRange(sourceDoc, from, to)) {
|
||||
split.save(output.toPath());
|
||||
}
|
||||
|
||||
int extraPagesAdded = 0;
|
||||
try (PDDocument testDoc = new PDDocument()) {
|
||||
for (int i = 0; i < scratch.getNumberOfPages(); i++) {
|
||||
testDoc.addPage(new PDPage(scratch.getPage(i).getCOSObject()));
|
||||
}
|
||||
for (int i = 0; i < pagesToLookAhead; i++) {
|
||||
testDoc.addPage(new PDPage(source.getPage(pageIndex + 1 + i).getCOSObject()));
|
||||
long testSize;
|
||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
|
||||
testDoc.save(out);
|
||||
testSize = out.size();
|
||||
}
|
||||
if (testSize > maxBytes) {
|
||||
break;
|
||||
}
|
||||
extraPagesAdded++;
|
||||
}
|
||||
}
|
||||
return extraPagesAdded;
|
||||
return output.length();
|
||||
}
|
||||
|
||||
private List<List<Integer>> computePageCountRanges(PDDocument sourceDocument, int pageCount) {
|
||||
private int lookAheadFit(
|
||||
PdfDocument sourceDoc,
|
||||
int rangeStart,
|
||||
int currentEnd,
|
||||
long maxBytes,
|
||||
int totalPages,
|
||||
File probeFile)
|
||||
throws IOException {
|
||||
int pagesToLookAhead = Math.min(5, totalPages - currentEnd - 1);
|
||||
int extra = 0;
|
||||
for (int i = 0; i < pagesToLookAhead; i++) {
|
||||
int trialEnd = currentEnd + 1 + i;
|
||||
long size = saveRange(sourceDoc, rangeStart, trialEnd, probeFile);
|
||||
if (size > maxBytes) {
|
||||
break;
|
||||
}
|
||||
extra++;
|
||||
}
|
||||
return extra;
|
||||
}
|
||||
|
||||
private List<int[]> computePageCountRanges(PdfDocument sourceDoc, int pageCount) {
|
||||
if (pageCount <= 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", "page count: " + pageCount);
|
||||
}
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
List<Integer> current = new ArrayList<>(pageCount);
|
||||
for (int i = 0; i < totalPages; i++) {
|
||||
current.add(i);
|
||||
if (current.size() == pageCount) {
|
||||
ranges.add(current);
|
||||
current = new ArrayList<>(pageCount);
|
||||
}
|
||||
}
|
||||
if (!current.isEmpty()) {
|
||||
ranges.add(current);
|
||||
int totalPages = sourceDoc.pageCount();
|
||||
List<int[]> ranges = new ArrayList<>();
|
||||
int start = 0;
|
||||
while (start < totalPages) {
|
||||
int end = Math.min(start + pageCount - 1, totalPages - 1);
|
||||
ranges.add(buildRange(start, end));
|
||||
start = end + 1;
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
private List<List<Integer>> computeDocCountRanges(
|
||||
PDDocument sourceDocument, int documentCount) {
|
||||
private List<int[]> computeDocCountRanges(PdfDocument sourceDoc, int documentCount) {
|
||||
if (documentCount <= 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.invalidArgument",
|
||||
"Invalid argument: {0}",
|
||||
"document count: " + documentCount);
|
||||
}
|
||||
int totalPages = sourceDocument.getNumberOfPages();
|
||||
int totalPages = sourceDoc.pageCount();
|
||||
int pagesPerDocument = totalPages / documentCount;
|
||||
int extraPages = totalPages % documentCount;
|
||||
|
||||
List<List<Integer>> ranges = new ArrayList<>();
|
||||
List<int[]> ranges = new ArrayList<>();
|
||||
int cursor = 0;
|
||||
for (int i = 0; i < documentCount; i++) {
|
||||
int pagesToAdd = pagesPerDocument + (i < extraPages ? 1 : 0);
|
||||
List<Integer> range = new ArrayList<>(pagesToAdd);
|
||||
for (int j = 0; j < pagesToAdd; j++) {
|
||||
range.add(cursor++);
|
||||
if (pagesToAdd == 0) {
|
||||
continue;
|
||||
}
|
||||
ranges.add(range);
|
||||
int end = cursor + pagesToAdd - 1;
|
||||
ranges.add(buildRange(cursor, end));
|
||||
cursor = end + 1;
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
private static int[] buildRange(int start, int end) {
|
||||
int[] range = new int[end - start + 1];
|
||||
for (int i = 0; i < range.length; i++) {
|
||||
range[i] = start + i;
|
||||
}
|
||||
return range;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,6 +330,22 @@ aiEngine:
|
||||
url: http://localhost:5001 # URL of the Python AI engine
|
||||
timeoutSeconds: 120 # Timeout in seconds for AI engine requests
|
||||
|
||||
# Cluster configuration. NOT YET ENABLED - scaffolding for later work. Leave at defaults.
|
||||
cluster:
|
||||
enabled: false # Master switch. 'false' (default) wires the in-process backplane and skips all cluster checks. Single-instance installs do not need to change anything here.
|
||||
backplane: inprocess # Backplane implementation: 'inprocess' (single JVM only) or 'valkey' (multi-node via Valkey/Redis)
|
||||
artifactStore: local # Transient cluster job-artifact backend: 'local' (per-node disk; single-node only) or 's3' (shared object store; required for multi-node). Distinct from 'storage.provider' which controls persistent user uploads - when both are 's3' they share the storage.s3.* credentials block. Multi-node deployments MUST set this to 's3'.
|
||||
valkey:
|
||||
url: "" # Valkey/Redis URL, e.g. 'redis://valkey:6379' or 'rediss://...' for TLS. Required when enabled=true and backplane=valkey.
|
||||
tls:
|
||||
skipCertVerification: false # set to 'true' to skip TLS certificate verification on Valkey connections (dev/test only)
|
||||
node:
|
||||
id: "" # Optional explicit node id. Blank = auto-generated UUID at startup.
|
||||
role: both # 'web' (serves HTTP), 'worker' (runs jobs), or 'both' (default)
|
||||
internalAddress: "" # host:port advertised in the instance registry for peer-to-peer cluster traffic. Blank = derived at startup.
|
||||
scheme: http # 'http' or 'https' - scheme peers use to call this node's /internal/cluster/** endpoints
|
||||
heartbeatIntervalMs: 5000 # Heartbeat publish interval for the instance registry (ms)
|
||||
|
||||
pdfEditor:
|
||||
fallback-font: classpath:/static/fonts/NotoSans-Regular.ttf # Override to point at a custom fallback font
|
||||
cache:
|
||||
|
||||
+142
-38
@@ -4,17 +4,28 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationWidget;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDAcroForm;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDField;
|
||||
import org.apache.pdfbox.pdmodel.interactive.form.PDTextField;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -23,8 +34,12 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.SplitPagesRequest;
|
||||
@@ -32,6 +47,7 @@ import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class SplitPDFControllerTest {
|
||||
|
||||
@TempDir Path tempDir;
|
||||
@@ -47,6 +63,12 @@ class SplitPDFControllerTest {
|
||||
String suffix = invocation.getArgument(0);
|
||||
return Files.createTempFile(tempDir, "test", suffix).toFile();
|
||||
});
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(File.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
}
|
||||
|
||||
private byte[] createPdf(int numPages) throws IOException {
|
||||
@@ -60,17 +82,79 @@ class SplitPDFControllerTest {
|
||||
}
|
||||
}
|
||||
|
||||
private void setupFactory() throws IOException {
|
||||
when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.load(any(File.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
when(pdfDocumentFactory.createNewDocumentBasedOnOldDocument(any(PDDocument.class)))
|
||||
.thenAnswer(inv -> new PDDocument());
|
||||
private byte[] createPdfWithForm(int numPages) throws IOException {
|
||||
try (PDDocument doc = new PDDocument()) {
|
||||
PDAcroForm acroForm = new PDAcroForm(doc);
|
||||
doc.getDocumentCatalog().setAcroForm(acroForm);
|
||||
for (int i = 0; i < numPages; i++) {
|
||||
PDPage page = new PDPage(PDRectangle.A4);
|
||||
doc.addPage(page);
|
||||
PDTextField field = new PDTextField(acroForm);
|
||||
field.setPartialName("text_p" + (i + 1));
|
||||
PDAnnotationWidget widget = new PDAnnotationWidget();
|
||||
widget.setRectangle(new PDRectangle(100, 700, 200, 20));
|
||||
widget.setPage(page);
|
||||
field.setWidgets(java.util.List.of(widget));
|
||||
page.getAnnotations().add(widget);
|
||||
acroForm.getFields().add(field);
|
||||
}
|
||||
Path pdfPath = tempDir.resolve("input.pdf");
|
||||
doc.save(pdfPath.toFile());
|
||||
return Files.readAllBytes(pdfPath);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> fieldNamesOf(byte[] pdfBytes) throws IOException {
|
||||
List<String> names = new ArrayList<>();
|
||||
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
|
||||
PDAcroForm acroForm = doc.getDocumentCatalog().getAcroForm(null);
|
||||
if (acroForm == null) {
|
||||
return names;
|
||||
}
|
||||
for (PDField field : acroForm.getFields()) {
|
||||
names.add(field.getFullyQualifiedName());
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
private int widgetCountOnPage(byte[] pdfBytes, int pageIndex) throws IOException {
|
||||
try (PDDocument doc = Loader.loadPDF(pdfBytes)) {
|
||||
int count = 0;
|
||||
for (PDAnnotation a : doc.getPage(pageIndex).getAnnotations()) {
|
||||
if (a instanceof PDAnnotationWidget) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
private List<byte[]> unzip(Resource zipResource) throws IOException {
|
||||
List<byte[]> entries = new ArrayList<>();
|
||||
try (ZipInputStream zis =
|
||||
new ZipInputStream(new ByteArrayInputStream(zipResource.getContentAsByteArray()))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zis.getNextEntry()) != null) {
|
||||
entries.add(zis.readAllBytes());
|
||||
zis.closeEntry();
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private int[] pageCountsOf(List<byte[]> entries) throws IOException {
|
||||
int[] counts = new int[entries.size()];
|
||||
for (int i = 0; i < entries.size(); i++) {
|
||||
try (PDDocument doc = Loader.loadPDF(entries.get(i))) {
|
||||
counts[i] = doc.getNumberOfPages();
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should split 6-page PDF at page 3")
|
||||
@DisplayName("Should split 6-page PDF at page 3 into 2 parts")
|
||||
void shouldSplitAtPage3() throws Exception {
|
||||
byte[] pdfBytes = createPdf(6);
|
||||
MockMultipartFile file =
|
||||
@@ -81,11 +165,12 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("3");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(2);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(3, 3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -100,11 +185,12 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("1,2,3");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(3);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(1, 1, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -119,15 +205,16 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("1");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(1);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should split with range notation")
|
||||
@DisplayName("Should split with multiple split points")
|
||||
void shouldSplitWithRange() throws Exception {
|
||||
byte[] pdfBytes = createPdf(10);
|
||||
MockMultipartFile file =
|
||||
@@ -138,11 +225,12 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("3,7");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(3);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(3, 4, 3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -157,13 +245,14 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("2");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
assertThat(response.getHeaders().getContentType())
|
||||
.isEqualTo(MediaType.APPLICATION_OCTET_STREAM);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(2);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(2, 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -178,11 +267,12 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("5");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(1);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -197,29 +287,43 @@ class SplitPDFControllerTest {
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("all");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(3);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(1, 1, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should handle file without extension in original name")
|
||||
void shouldHandleFileWithoutExtension() throws Exception {
|
||||
byte[] pdfBytes = createPdf(2);
|
||||
@DisplayName("Should preserve AcroForm and per-page widgets when splitting form PDF")
|
||||
void shouldSplitFormPdf() throws Exception {
|
||||
byte[] pdfBytes = createPdfWithForm(4);
|
||||
MockMultipartFile file =
|
||||
new MockMultipartFile(
|
||||
"fileInput", "no_extension", MediaType.APPLICATION_PDF_VALUE, pdfBytes);
|
||||
"fileInput", "input.pdf", MediaType.APPLICATION_PDF_VALUE, pdfBytes);
|
||||
|
||||
SplitPagesRequest request = new SplitPagesRequest();
|
||||
request.setFileInput(file);
|
||||
request.setPageNumbers("1");
|
||||
request.setPageNumbers("2");
|
||||
|
||||
setupFactory();
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(2);
|
||||
assertThat(pageCountsOf(outputs)).containsExactly(2, 2);
|
||||
|
||||
assertThat(fieldNamesOf(outputs.get(0)))
|
||||
.as("first split keeps fields whose widgets are on pages 1-2")
|
||||
.containsExactlyInAnyOrder("text_p1", "text_p2");
|
||||
assertThat(fieldNamesOf(outputs.get(1)))
|
||||
.as("second split keeps fields whose widgets are on pages 3-4")
|
||||
.containsExactlyInAnyOrder("text_p3", "text_p4");
|
||||
|
||||
assertThat(widgetCountOnPage(outputs.get(0), 0)).isEqualTo(1);
|
||||
assertThat(widgetCountOnPage(outputs.get(0), 1)).isEqualTo(1);
|
||||
assertThat(widgetCountOnPage(outputs.get(1), 0)).isEqualTo(1);
|
||||
assertThat(widgetCountOnPage(outputs.get(1), 1)).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
+64
-29
@@ -4,11 +4,19 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
@@ -27,10 +35,11 @@ import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import stirling.software.SPDF.model.api.SplitPdfByChaptersRequest;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
@@ -55,6 +64,12 @@ class SplitPdfByChaptersControllerTest {
|
||||
String suffix = inv.getArgument(0);
|
||||
return Files.createTempFile(tempDir, "test", suffix).toFile();
|
||||
});
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(File.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
lenient()
|
||||
.when(pdfDocumentFactory.load(any(File.class), eq(true)))
|
||||
.thenAnswer(inv -> Loader.loadPDF((File) inv.getArgument(0)));
|
||||
}
|
||||
|
||||
private byte[] createPdfWithBookmarks(int numPages, String... chapterNames) throws IOException {
|
||||
@@ -83,6 +98,29 @@ class SplitPdfByChaptersControllerTest {
|
||||
}
|
||||
}
|
||||
|
||||
private List<byte[]> unzip(Resource zipResource) throws IOException {
|
||||
List<byte[]> entries = new ArrayList<>();
|
||||
try (ZipInputStream zis =
|
||||
new ZipInputStream(new ByteArrayInputStream(zipResource.getContentAsByteArray()))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zis.getNextEntry()) != null) {
|
||||
entries.add(zis.readAllBytes());
|
||||
zis.closeEntry();
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private int totalPagesOf(List<byte[]> entries) throws IOException {
|
||||
int total = 0;
|
||||
for (byte[] data : entries) {
|
||||
try (PDDocument doc = Loader.loadPDF(data)) {
|
||||
total += doc.getNumberOfPages();
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should split PDF by chapters")
|
||||
void shouldSplitByChapters() throws Exception {
|
||||
@@ -97,12 +135,12 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(false);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(3);
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(6);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -119,12 +157,12 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(true);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(2);
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -163,10 +201,6 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(false);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(
|
||||
inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> controller.splitPdf(request));
|
||||
}
|
||||
}
|
||||
@@ -185,12 +219,12 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(false);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(1);
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -207,14 +241,15 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(true);
|
||||
request.setAllowDuplicates(false);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
when(pdfMetadataService.extractMetadataFromPdf(any(PDDocument.class)))
|
||||
lenient()
|
||||
.when(pdfMetadataService.extractMetadataFromPdf(any(PDDocument.class)))
|
||||
.thenReturn(new stirling.software.common.model.PdfMetadata());
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -231,12 +266,12 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(false);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(3);
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(6);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -253,11 +288,11 @@ class SplitPdfByChaptersControllerTest {
|
||||
request.setIncludeMetadata(false);
|
||||
request.setAllowDuplicates(true);
|
||||
|
||||
when(pdfDocumentFactory.load(any(MultipartFile.class)))
|
||||
.thenAnswer(inv -> Loader.loadPDF(((MultipartFile) inv.getArgument(0)).getBytes()));
|
||||
|
||||
var response = controller.splitPdf(request);
|
||||
ResponseEntity<Resource> response = controller.splitPdf(request);
|
||||
|
||||
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
List<byte[]> outputs = unzip(response.getBody());
|
||||
assertThat(outputs).hasSize(5);
|
||||
assertThat(totalPagesOf(outputs)).isEqualTo(10);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user