mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 21:30:14 +03:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Anthony Stirling
|
7ce98d29f2 | ||
|
Anthony Stirling
|
65c01e8078 | ||
|
Anthony Stirling
|
4700542c75 |
@@ -35,7 +35,7 @@ jobs:
|
||||
pr_ref: ${{ steps.resolve.outputs.ref }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -184,7 +184,7 @@ jobs:
|
||||
fetch-depth: 0 # Fetch full history for commit hash detection
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
@@ -193,7 +193,7 @@ jobs:
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -320,7 +320,7 @@ jobs:
|
||||
# under the exact same access rules as the app preview.
|
||||
- name: Detect Storybook changes
|
||||
id: sb-changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
@@ -456,7 +456,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -233,10 +233,10 @@ jobs:
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -474,7 +474,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
is_fork: ${{ steps.decide.outputs.is_fork }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
- name: AI PR Title Analysis
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: ai-title-analysis
|
||||
uses: actions/ai-inference@a7805884c80886efc241e94a5351df715968a0ad # v2.1.1
|
||||
uses: actions/ai-inference@17ff458cb182449bbb2e43701fcd98f6af8f6570 # v2.1.0
|
||||
with:
|
||||
model: openai/gpt-4o
|
||||
system-prompt-file: ".github/config/system-prompt.txt"
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
flavor: [core, proprietary, saas]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -295,70 +295,3 @@ jobs:
|
||||
name: playwright-report-enterprise-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
# Multi-node regression: builds + seeds the clustered stack (testing/compose/docker-compose-multinode.yml)
|
||||
# and runs behave features/multinode. Licence-gated, so it runs after the Playwright job (not in parallel).
|
||||
multinode-e2e:
|
||||
needs: [pick, playwright-e2e-enterprise]
|
||||
# Nightly cron + manual dispatch only (heavy build), fork-gated for the licence secret.
|
||||
if: >-
|
||||
always() && needs.pick.outputs.is_fork != 'true'
|
||||
&& (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
PREMIUM_KEY: ${{ secrets.PREMIUM_KEY_ENTERPRISE }}
|
||||
PREMIUM_ENABLED: "true"
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
MN_COMPOSE: docker-compose-multinode.yml
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
cache-dependency-path: ./testing/cucumber/requirements.txt
|
||||
- name: Install behave test deps
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
|
||||
- name: Build the multi-node image
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" build
|
||||
- name: Bring up the cluster and wait for both nodes healthy
|
||||
working-directory: testing/compose
|
||||
run: |
|
||||
docker compose -f "$MN_COMPOSE" up -d
|
||||
for i in $(seq 1 90); do
|
||||
h1=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-1 2>/dev/null || echo starting)
|
||||
h2=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-2 2>/dev/null || echo starting)
|
||||
if [ "$h1" = healthy ] && [ "$h2" = healthy ]; then echo "both nodes healthy"; exit 0; fi
|
||||
sleep 5
|
||||
done
|
||||
echo "::error::nodes did not become healthy"
|
||||
docker compose -f "$MN_COMPOSE" logs --tail=200 stirling-1 stirling-2
|
||||
exit 1
|
||||
- name: Seed the cluster (teams, users, S3 connection, policy)
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" --profile seed run --rm seed
|
||||
- name: Run multi-node regression (implemented guarantees)
|
||||
working-directory: testing/cucumber
|
||||
# -e overrides behave.ini's exclusion of features/multinode; ~@known_gap skips any tracked-gap scenarios.
|
||||
run: python -m behave features/multinode -e "features/enterprise" --tags="~@known_gap ~@destructive" --no-capture -f plain
|
||||
- name: Run multi-node failover (destructive)
|
||||
working-directory: testing/cucumber
|
||||
run: python -m behave features/multinode -e "features/enterprise" --tags="@destructive ~@known_gap" --no-capture -f plain
|
||||
- name: Dump node logs on failure
|
||||
if: failure()
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" logs --tail=400 stirling-1 stirling-2
|
||||
- name: Tear down
|
||||
if: always()
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" --profile seed down -v --remove-orphans
|
||||
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: .github/config/.files.yaml
|
||||
@@ -267,7 +267,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get commit hashes for frontend and backend
|
||||
id: commit-hashes
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# runtime token isn't exposed) since the docker driver can't use it.
|
||||
- name: Set up Docker Buildx
|
||||
if: inputs.docker-base-changed != 'true'
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Expose ACTIONS_RUNTIME_TOKEN / ACTIONS_RESULTS_URL for docker buildx type=gha cache backend.
|
||||
- name: Expose GitHub runtime for Buildx cache
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: .github/config/.files.yaml
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -326,7 +326,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
file_suffix: "-server"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
allowed-endpoints: >
|
||||
@@ -637,7 +637,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
@@ -13,6 +13,11 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
build_engine:
|
||||
description: "Build & push the stirling-pdf-engine image (plus the -docparse addon variant)."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
force_unoserver_rebuild:
|
||||
description: "Rebuild stirling-unoserver even if its source hash is unchanged."
|
||||
required: false
|
||||
@@ -51,9 +56,11 @@ jobs:
|
||||
env:
|
||||
RUN_MAIN_APP: ${{ github.event_name != 'workflow_dispatch' || inputs.build_main_app }}
|
||||
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
|
||||
# Engine images are dispatch-only for now; flip the default once the addon stabilises.
|
||||
RUN_ENGINE: ${{ github.event_name == 'workflow_dispatch' && inputs.build_engine }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -82,7 +89,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
@@ -107,13 +114,13 @@ jobs:
|
||||
cosign-release: "v2.4.1"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -219,6 +226,62 @@ jobs:
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
|
||||
- name: Generate tags for engine
|
||||
id: meta-engine
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
if: env.RUN_ENGINE == 'true'
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-engine
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf-engine
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push engine image
|
||||
id: build-push-engine
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_ENGINE == 'true' && steps.meta-engine.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: ./engine
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-pdf-engine
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-engine
|
||||
tags: ${{ steps.meta-engine.outputs.tags }}
|
||||
labels: ${{ steps.meta-engine.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Generate tags for engine docparse addon
|
||||
id: meta-engine-docparse
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
if: env.RUN_ENGINE == 'true'
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-engine
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf-engine
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-docparse
|
||||
type=raw,value=latest-docparse
|
||||
|
||||
- name: Build and push engine docparse addon image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_ENGINE == 'true' && steps.meta-engine-docparse.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: ./engine
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-pdf-engine-docparse
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-engine-docparse
|
||||
tags: ${{ steps.meta-engine-docparse.outputs.tags }}
|
||||
labels: ${{ steps.meta-engine-docparse.outputs.labels }}
|
||||
build-args: DOCPARSE=true
|
||||
platforms: linux/amd64
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Generate tags for ultra-lite
|
||||
id: meta-lite
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -26,13 +26,13 @@ jobs:
|
||||
uses: imjasonh/setup-crane@31b88afe9de28ae0ffa220711af4b60be9435f6e # v0.4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -420,9 +420,7 @@ jobs:
|
||||
# Linux: build deb+rpm only here (deb-only on minimal smoke builds).
|
||||
# AppImage runs in its own continue-on-error step below so its
|
||||
# persistent linuxdeploy failure (#6127 onwards) does not tank uploads.
|
||||
args: >-
|
||||
${{ matrix.platform == 'ubuntu-22.04' && (inputs.minimal && '--bundles deb' || '--bundles deb,rpm') || matrix.args }}
|
||||
--config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
args: ${{ matrix.platform == 'ubuntu-22.04' && (inputs.minimal && '--bundles deb' || '--bundles deb,rpm') || matrix.args }}
|
||||
|
||||
# AppImage is decoupled so its linuxdeploy run gets a fresh process
|
||||
# (rpm scratch state torn down) and its failure can't tank deb/rpm.
|
||||
@@ -615,19 +613,12 @@ jobs:
|
||||
pr-comment:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
# Fork and Dependabot pull_request runs receive a read-only GITHUB_TOKEN,
|
||||
# so the API cannot create or update PR comments there. The artifacts are
|
||||
# still uploaded and remain available from the Actions run page.
|
||||
if: >-
|
||||
github.event_name == 'pull_request' &&
|
||||
needs.build.result == 'success' &&
|
||||
!github.event.pull_request.head.repo.fork &&
|
||||
github.actor != 'dependabot[bot]'
|
||||
if: github.event_name == 'pull_request' && needs.build.result == 'success'
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -718,7 +709,7 @@ jobs:
|
||||
if: always()
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
cache-scope: stirling-pdf-fat
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build base image locally (PR base change only)
|
||||
if: github.event_name == 'pull_request' && inputs.docker-base-changed == 'true'
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -131,14 +131,14 @@ jobs:
|
||||
frontend: ${{ steps.changes.outputs.frontend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: ".github/config/.files.yaml"
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -263,16 +263,6 @@ tasks:
|
||||
- task: lint:eslint
|
||||
- task: lint:dpdm
|
||||
- task: lint:colors
|
||||
- task: lint:css
|
||||
|
||||
lint:css:
|
||||
desc: "Lint stylesheets for duplicate selectors"
|
||||
deps: [install]
|
||||
cmds:
|
||||
# Covers the whole editor tree, including the portal/processor layer and
|
||||
# public/css. Vendored CSS and build output are excluded via ignoreFiles
|
||||
# in stylelint.config.mjs.
|
||||
- npx stylelint "editor/**/*.css"
|
||||
|
||||
lint:colors:
|
||||
desc: "Enforce theme tokens — no hardcoded colours or raw primitives in components"
|
||||
@@ -398,13 +388,6 @@ tasks:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/portal/tsconfig.json }
|
||||
|
||||
typecheck:storybook:
|
||||
desc: "Typecheck Storybook config and stories"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: .storybook/tsconfig.json }
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
cmds:
|
||||
@@ -416,7 +399,6 @@ tasks:
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
- task: typecheck:storybook
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
|
||||
@@ -338,7 +338,6 @@ public class EndpointConfiguration {
|
||||
addEndpointToGroup("PageOps", "split-pages");
|
||||
addEndpointToGroup("PageOps", "rearrange-pages");
|
||||
addEndpointToGroup("PageOps", "rotate-pdf");
|
||||
addEndpointToGroup("PageOps", "auto-rotate-pdf");
|
||||
addEndpointToGroup("PageOps", "multi-page-layout");
|
||||
addEndpointToGroup("PageOps", "booklet-imposition");
|
||||
addEndpointToGroup("PageOps", "scale-pages");
|
||||
@@ -434,6 +433,10 @@ public class EndpointConfiguration {
|
||||
addEndpointToGroup("Automation", "automate"); // Alias for handleData (user-friendly name)
|
||||
addEndpointToGroup("Automation", "pipeline");
|
||||
|
||||
// Adding endpoints to "DocParse" group (ingestion: chunk + index + export)
|
||||
addEndpointToGroup("DocParse", "rag-ingest");
|
||||
addEndpointToGroup("DocParse", "extract-tables");
|
||||
|
||||
// Adding endpoints to "DeveloperTools" group
|
||||
addEndpointToGroup("DeveloperTools", "show-javascript");
|
||||
|
||||
|
||||
+19
-14
@@ -77,6 +77,7 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private Docparse docparse = new Docparse();
|
||||
private Mcp mcp = new Mcp();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
@@ -425,6 +426,24 @@ public class ApplicationProperties {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DocParse settings (top-level {@code docparse.*}): document understanding for ingestion
|
||||
* pipelines. The basic tier (text layer) always works; the advanced tier lives in the engine's
|
||||
* docparse addon.
|
||||
*/
|
||||
@Data
|
||||
public static class Docparse {
|
||||
|
||||
/** Master switch; hides the DocParse endpoints when false. */
|
||||
private boolean enabled = true;
|
||||
|
||||
/** Requested tier: 'auto', 'basic', or 'advanced'. 'auto' resolves per document. */
|
||||
private String mode = "auto";
|
||||
|
||||
/** Mirrors DOCPARSE_AUTO_INSTALL for the engine's boot-time addon install script. */
|
||||
private boolean autoInstall = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model Context Protocol (MCP) server configuration. All keys live under the top-level {@code
|
||||
* mcp.*} prefix. {@link #enabled} defaults to {@code false}: when off, no MCP beans are wired,
|
||||
@@ -1076,20 +1095,6 @@ public class ApplicationProperties {
|
||||
private Quotas quotas = new Quotas();
|
||||
private Sharing sharing = new Sharing();
|
||||
private Signing signing = new Signing();
|
||||
private Encryption encryption = new Encryption();
|
||||
|
||||
/**
|
||||
* Encryption at rest for stored files (Pro/Enterprise). Enabling encrypts new writes;
|
||||
* disabling later only stops encrypting new writes — existing encrypted files keep
|
||||
* decrypting as long as the key material is present. The master key is resolved like the
|
||||
* credential key: {@code stirling.security.fileEncryptionKey} property, {@code
|
||||
* STIRLING_FILE_ENCRYPTION_KEY} env var, or an auto-generated {@code file-encryption.key}
|
||||
* in the config directory.
|
||||
*/
|
||||
@Data
|
||||
public static class Encryption {
|
||||
private boolean enabled = false;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Local {
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
/**
|
||||
* View of the engine's DocParse capability for modules that cannot see the proprietary
|
||||
* implementation (e.g. ConfigController in core). Implemented by the proprietary
|
||||
* DocparseCapabilityService; absent when the proprietary module is not loaded.
|
||||
*/
|
||||
public interface DocparseCapabilityServiceInterface {
|
||||
|
||||
/**
|
||||
* Whether the engine reports the docparse addon (advanced tier) as installed. Must be cheap and
|
||||
* non-blocking: returns the cached probe result, {@code false} when the engine is disabled,
|
||||
* unreachable, or not yet probed.
|
||||
*/
|
||||
boolean isAdvancedInstalled();
|
||||
}
|
||||
@@ -53,7 +53,7 @@ public class InternalApiClient {
|
||||
// ApiConnectionResolver.
|
||||
private static final Pattern ALLOWED_ENDPOINT_PATH =
|
||||
Pattern.compile(
|
||||
"^/api/v1/(general|misc|security|convert|filter|integration)(/[A-Za-z0-9_-]+)+$"
|
||||
"^/api/v1/(general|misc|security|convert|filter|integration|docparse)(/[A-Za-z0-9_-]+)+$"
|
||||
+ "|^/api/v1/ai/tools(/[A-Za-z0-9_-]+)+$");
|
||||
|
||||
/**
|
||||
|
||||
@@ -50,7 +50,7 @@ dependencies {
|
||||
implementation "org.apache.pdfbox:preflight:$pdfboxVersion"
|
||||
implementation "org.apache.pdfbox:xmpbox:$pdfboxVersion"
|
||||
|
||||
implementation 'org.verapdf:validation-model:1.30.2'
|
||||
implementation 'org.verapdf:validation-model:1.28.2'
|
||||
// CVE-2025-66453: Explicit rhino 1.7.15 to override verapdf's 1.7.13
|
||||
implementation "org.mozilla:rhino:${rhinoVersion}"
|
||||
|
||||
|
||||
-385
@@ -1,385 +0,0 @@
|
||||
package stirling.software.SPDF.controller.api.misc;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult.PageResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotatePdfRequest;
|
||||
import stirling.software.SPDF.model.api.misc.PageRotation;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.OsdResult;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.TextDirection;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.MiscApi;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.ProcessExecutor;
|
||||
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
|
||||
import stirling.software.common.util.TempDirectory;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
|
||||
@MiscApi
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class AutoRotateController {
|
||||
|
||||
// OSD decides orientation from script shape, not character identity, so it does not need
|
||||
// OCR-grade resolution. 150 DPI is ample for that and a quarter of the pixels of 300.
|
||||
private static final int OSD_RENDER_DPI = 150;
|
||||
|
||||
private static final String METHOD_TEXT = "text";
|
||||
private static final String METHOD_OSD = "osd";
|
||||
private static final String METHOD_INFERRED = "inferred";
|
||||
private static final String METHOD_NONE = "none";
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final EndpointConfiguration endpointConfiguration;
|
||||
private final RuntimePathConfig runtimePathConfig;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/auto-rotate-pdf",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Detect and fix the orientation of every page",
|
||||
description =
|
||||
"Detects each page's orientation (embedded-text direction first, Tesseract OSD"
|
||||
+ " for scanned pages) and sets the page rotation so the content"
|
||||
+ " displays upright. With dryRun=true, returns a JSON per-page report"
|
||||
+ " instead of the PDF. With pageRotations set, applies the given"
|
||||
+ " corrections without running detection."
|
||||
+ " Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<?> autoRotatePdf(@Valid @ModelAttribute AutoRotatePdfRequest request)
|
||||
throws IOException, InterruptedException {
|
||||
String mode =
|
||||
request.getDetectionMode() == null
|
||||
? "auto"
|
||||
: request.getDetectionMode().toLowerCase(Locale.ROOT);
|
||||
if (!"auto".equals(mode) && !"text".equals(mode) && !"osd".equals(mode)) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"detectionMode",
|
||||
"must be one of auto, text, osd");
|
||||
}
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(request)) {
|
||||
if (request.getPageRotations() != null && !request.getPageRotations().isEmpty()) {
|
||||
applyExplicitRotations(document, request.getPageRotations());
|
||||
return pdfResponse(document, request);
|
||||
}
|
||||
|
||||
AutoRotateAnalysisResult analysis = analyse(document, mode, request);
|
||||
if (request.isDryRun()) {
|
||||
return ResponseEntity.ok(analysis);
|
||||
}
|
||||
for (PageResult pageResult : analysis.getPages()) {
|
||||
if (pageResult.isApply()) {
|
||||
PDPage page = document.getPage(pageResult.getPageNumber() - 1);
|
||||
page.setRotation(
|
||||
Math.floorMod(page.getRotation() + pageResult.getCorrection(), 360));
|
||||
}
|
||||
}
|
||||
return pdfResponse(document, request);
|
||||
}
|
||||
}
|
||||
|
||||
private AutoRotateAnalysisResult analyse(
|
||||
PDDocument document, String mode, AutoRotatePdfRequest request)
|
||||
throws IOException, InterruptedException {
|
||||
double threshold =
|
||||
request.getConfidenceThreshold() == null ? 14.0 : request.getConfidenceThreshold();
|
||||
boolean tesseractAvailable = endpointConfiguration.isGroupEnabled("tesseract");
|
||||
boolean useText = !"osd".equals(mode);
|
||||
boolean useOsd = !"text".equals(mode);
|
||||
|
||||
List<PageResult> results = new ArrayList<>();
|
||||
List<Integer> osdCandidates = new ArrayList<>();
|
||||
|
||||
int pageCount = document.getNumberOfPages();
|
||||
// One walk of the document for all pages, rather than one walk per page.
|
||||
List<TextDirection> textDirections =
|
||||
useText ? AutoRotateDetection.detectTextDirections(document) : List.of();
|
||||
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
int currentRotation = Math.floorMod(document.getPage(i).getRotation(), 360);
|
||||
PageResult result =
|
||||
PageResult.builder()
|
||||
.pageNumber(i + 1)
|
||||
.currentRotation(currentRotation)
|
||||
.method(METHOD_NONE)
|
||||
.build();
|
||||
|
||||
if (useText) {
|
||||
TextDirection direction = textDirections.get(i);
|
||||
if (direction.isConclusive()) {
|
||||
int correction =
|
||||
AutoRotateDetection.correctionFromTextDirection(
|
||||
direction.dominantDirection(), currentRotation);
|
||||
result.setMethod(METHOD_TEXT);
|
||||
result.setCorrection(correction);
|
||||
result.setConfidence(direction.dominance() * 100);
|
||||
result.setApply(correction != 0);
|
||||
} else if (!useOsd) {
|
||||
result.setNote(
|
||||
direction.glyphCount() < AutoRotateDetection.MIN_GLYPHS
|
||||
? "tooFewGlyphs"
|
||||
: "noDominantDirection");
|
||||
}
|
||||
}
|
||||
|
||||
if (useOsd && METHOD_NONE.equals(result.getMethod())) {
|
||||
if (tesseractAvailable) {
|
||||
osdCandidates.add(i);
|
||||
} else {
|
||||
result.setNote("tesseractUnavailable");
|
||||
}
|
||||
}
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
if (!osdCandidates.isEmpty()) {
|
||||
runOsdOnPages(document, osdCandidates, results, threshold);
|
||||
}
|
||||
|
||||
if (request.isInferUndetected()) {
|
||||
inferUndetectedPages(results);
|
||||
}
|
||||
|
||||
return summarise(results, pageCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill in pages that no signal could decide, using the pages that could. When every decided
|
||||
* page sharing an undecided page's current rotation agrees on one correction, that correction
|
||||
* is the document's consensus for that rotation and is applied to the undecided page. This is
|
||||
* the common "whole document rotated uniformly, but a cover or near-blank page has too little
|
||||
* text to detect on its own" case. If decided pages disagree, nothing is inferred.
|
||||
*/
|
||||
private void inferUndetectedPages(List<PageResult> results) {
|
||||
// rotation -> the single agreed correction, or null once a conflict is seen
|
||||
Map<Integer, Integer> consensus = new HashMap<>();
|
||||
Set<Integer> conflicted = new HashSet<>();
|
||||
for (PageResult result : results) {
|
||||
if (METHOD_NONE.equals(result.getMethod())) {
|
||||
continue;
|
||||
}
|
||||
int rotation = result.getCurrentRotation();
|
||||
if (conflicted.contains(rotation)) {
|
||||
continue;
|
||||
}
|
||||
Integer existing = consensus.get(rotation);
|
||||
if (existing == null) {
|
||||
consensus.put(rotation, result.getCorrection());
|
||||
} else if (existing != result.getCorrection()) {
|
||||
conflicted.add(rotation);
|
||||
consensus.remove(rotation);
|
||||
}
|
||||
}
|
||||
|
||||
for (PageResult result : results) {
|
||||
if (!METHOD_NONE.equals(result.getMethod())) {
|
||||
continue;
|
||||
}
|
||||
Integer correction = consensus.get(result.getCurrentRotation());
|
||||
if (correction == null) {
|
||||
continue;
|
||||
}
|
||||
result.setMethod(METHOD_INFERRED);
|
||||
result.setCorrection(correction);
|
||||
result.setConfidence(null);
|
||||
result.setApply(correction != 0);
|
||||
result.setNote("inferredFromDocument");
|
||||
}
|
||||
}
|
||||
|
||||
private void runOsdOnPages(
|
||||
PDDocument document,
|
||||
List<Integer> pageIndexes,
|
||||
List<PageResult> results,
|
||||
double threshold)
|
||||
throws IOException, InterruptedException {
|
||||
String tessDataPath = runtimePathConfig.getTessDataPath();
|
||||
boolean haveOsdData =
|
||||
tessDataPath != null && new File(tessDataPath, "osd.traineddata").exists();
|
||||
|
||||
int dpi = OSD_RENDER_DPI;
|
||||
if (applicationProperties != null && applicationProperties.getSystem() != null) {
|
||||
dpi = Math.min(OSD_RENDER_DPI, applicationProperties.getSystem().getMaxDPI());
|
||||
}
|
||||
final int renderDpi = dpi;
|
||||
|
||||
try (TempDirectory tempDir = new TempDirectory(tempFileManager)) {
|
||||
PDFRenderer renderer = new PDFRenderer(document);
|
||||
renderer.setSubsamplingAllowed(true);
|
||||
// One reused path, deleted after every page: the images are throwaway input for
|
||||
// Tesseract, so a long document must not accumulate one file per page on disk.
|
||||
File imageFile = new File(tempDir.getPath().toFile(), "osd-page.bmp");
|
||||
|
||||
for (int pageIndex : pageIndexes) {
|
||||
PageResult result = results.get(pageIndex);
|
||||
try {
|
||||
// Rendering honours the page's current /Rotate, so OSD sees the page exactly
|
||||
// as a viewer would and its verdict is always an additive correction.
|
||||
BufferedImage image =
|
||||
ExceptionUtils.handleOomRendering(
|
||||
pageIndex + 1,
|
||||
renderDpi,
|
||||
() ->
|
||||
renderer.renderImageWithDPI(
|
||||
pageIndex, renderDpi, ImageType.GRAY));
|
||||
|
||||
if (AutoRotateDetection.isBlankRender(image)) {
|
||||
// Nothing for OSD to read; skip the process spawn entirely.
|
||||
result.setNote("blankPage");
|
||||
continue;
|
||||
}
|
||||
|
||||
// BMP, not PNG: the file is deleted straight after Tesseract reads it, so
|
||||
// paying for compression only to discard the result is wasted work.
|
||||
ImageIO.write(image, "bmp", imageFile);
|
||||
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add("tesseract");
|
||||
command.add(imageFile.getAbsolutePath());
|
||||
command.add("stdout");
|
||||
command.add("--psm");
|
||||
command.add("0");
|
||||
if (haveOsdData) {
|
||||
command.add("--tessdata-dir");
|
||||
command.add(tessDataPath);
|
||||
}
|
||||
|
||||
ProcessExecutorResult processResult =
|
||||
ProcessExecutor.getInstance(ProcessExecutor.Processes.TESSERACT)
|
||||
.runCommandWithOutputHandling(command);
|
||||
|
||||
Optional<OsdResult> osd =
|
||||
AutoRotateDetection.parseOsd(processResult.getMessages());
|
||||
if (osd.isEmpty()) {
|
||||
result.setNote("osdNoVerdict");
|
||||
continue;
|
||||
}
|
||||
result.setConfidence(osd.get().confidence());
|
||||
result.setCorrection(osd.get().rotate());
|
||||
if (osd.get().confidence() >= threshold) {
|
||||
result.setMethod(METHOD_OSD);
|
||||
result.setApply(osd.get().rotate() != 0);
|
||||
} else {
|
||||
result.setNote("belowThreshold");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Textless pages make Tesseract exit non-zero; skip, never guess.
|
||||
log.debug("OSD failed for page {}: {}", pageIndex + 1, e.getMessage());
|
||||
result.setNote("osdFailed");
|
||||
} finally {
|
||||
Files.deleteIfExists(imageFile.toPath());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void applyExplicitRotations(PDDocument document, List<PageRotation> rotations) {
|
||||
int pageCount = document.getNumberOfPages();
|
||||
Set<Integer> seen = new HashSet<>();
|
||||
for (PageRotation entry : rotations) {
|
||||
Integer pageNumber = entry.getPageNumber();
|
||||
Integer angle = entry.getRotation();
|
||||
if (pageNumber == null
|
||||
|| angle == null
|
||||
|| pageNumber < 1
|
||||
|| pageNumber > pageCount
|
||||
|| angle % 90 != 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"pageRotations",
|
||||
"page numbers must exist and rotations must be multiples of 90");
|
||||
}
|
||||
// Rotations are additive, so a repeated page would be turned twice; reject rather
|
||||
// than silently pick a winner.
|
||||
if (!seen.add(pageNumber)) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"pageRotations",
|
||||
"page " + pageNumber + " is listed more than once");
|
||||
}
|
||||
PDPage page = document.getPage(pageNumber - 1);
|
||||
page.setRotation(Math.floorMod(page.getRotation() + angle, 360));
|
||||
}
|
||||
}
|
||||
|
||||
private AutoRotateAnalysisResult summarise(List<PageResult> results, int pageCount) {
|
||||
int toRotate = 0;
|
||||
int byText = 0;
|
||||
int byOsd = 0;
|
||||
int byInference = 0;
|
||||
int undetected = 0;
|
||||
for (PageResult result : results) {
|
||||
if (result.isApply()) {
|
||||
toRotate++;
|
||||
}
|
||||
switch (result.getMethod()) {
|
||||
case METHOD_TEXT -> byText++;
|
||||
case METHOD_OSD -> byOsd++;
|
||||
case METHOD_INFERRED -> byInference++;
|
||||
default -> undetected++;
|
||||
}
|
||||
}
|
||||
return AutoRotateAnalysisResult.builder()
|
||||
.pages(results)
|
||||
.totalPages(pageCount)
|
||||
.pagesToRotate(toRotate)
|
||||
.detectedByText(byText)
|
||||
.detectedByOsd(byOsd)
|
||||
.inferred(byInference)
|
||||
.undetected(undetected)
|
||||
.build();
|
||||
}
|
||||
|
||||
private ResponseEntity<?> pdfResponse(PDDocument document, AutoRotatePdfRequest request)
|
||||
throws IOException {
|
||||
String originalName =
|
||||
request.getFileInput() != null
|
||||
? request.getFileInput().getOriginalFilename()
|
||||
: "document.pdf";
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
GeneralUtils.generateFilename(originalName, "_auto_rotated.pdf"),
|
||||
tempFileManager);
|
||||
}
|
||||
}
|
||||
+2
-6
@@ -140,15 +140,11 @@ public class ValidateSignatureController {
|
||||
// covers the TSTInfo, which can never match.
|
||||
boolean isDocTimeStamp = SUBFILTER_RFC3161.equals(sig.getSubFilter());
|
||||
CMSSignedData signedData;
|
||||
// Parse from a stream: /Contents is zero-padded to its reserved length and the
|
||||
// byte[] constructors reject those trailing bytes since BC 1.85.
|
||||
if (isDocTimeStamp) {
|
||||
signedData = new CMSSignedData(new ByteArrayInputStream(signatureBytes));
|
||||
signedData = new CMSSignedData(signatureBytes);
|
||||
} else {
|
||||
CMSProcessable content = new CMSProcessableByteArray(signedContent);
|
||||
signedData =
|
||||
new CMSSignedData(
|
||||
content, new ByteArrayInputStream(signatureBytes));
|
||||
signedData = new CMSSignedData(content, signatureBytes);
|
||||
}
|
||||
|
||||
// What actually binds a timestamp to this document: the TSTInfo's message
|
||||
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** Per-page orientation report returned by auto-rotate-pdf when dryRun is set. */
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AutoRotateAnalysisResult {
|
||||
|
||||
private List<PageResult> pages;
|
||||
|
||||
private int totalPages;
|
||||
|
||||
@Schema(description = "Number of pages a correction would be applied to")
|
||||
private int pagesToRotate;
|
||||
|
||||
private int detectedByText;
|
||||
|
||||
private int detectedByOsd;
|
||||
|
||||
@Schema(description = "Pages whose correction was inherited from the document consensus")
|
||||
private int inferred;
|
||||
|
||||
private int undetected;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class PageResult {
|
||||
|
||||
@Schema(description = "1-based page number")
|
||||
private int pageNumber;
|
||||
|
||||
@Schema(description = "The page's current /Rotate value, normalised to 0-270")
|
||||
private int currentRotation;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detected additional clockwise rotation that would make the page upright."
|
||||
+ " Reported for diagnostics even when it is not used (for example"
|
||||
+ " an OSD verdict below the confidence threshold); 'apply' is the"
|
||||
+ " authority on whether it is actually applied")
|
||||
private int correction;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detection confidence: percentage of glyphs sharing the dominant direction"
|
||||
+ " for method 'text', Tesseract orientation confidence for method"
|
||||
+ " 'osd', absent when nothing was detected")
|
||||
private Double confidence;
|
||||
|
||||
@Schema(
|
||||
description = "How the orientation was determined",
|
||||
allowableValues = {"text", "osd", "inferred", "none"})
|
||||
private String method;
|
||||
|
||||
@Schema(description = "Whether the correction will be (or was) applied")
|
||||
private boolean apply;
|
||||
|
||||
@Schema(description = "Machine-readable reason when no correction is applied")
|
||||
private String note;
|
||||
}
|
||||
}
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Min;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class AutoRotatePdfRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detection method. 'auto' tries embedded-text direction first and falls back"
|
||||
+ " to Tesseract OSD for pages without usable text; 'text' uses only"
|
||||
+ " embedded-text direction; 'osd' forces Tesseract OSD for every page",
|
||||
allowableValues = {"auto", "text", "osd"},
|
||||
defaultValue = "auto")
|
||||
private String detectionMode = "auto";
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Minimum Tesseract OSD orientation confidence required before a correction is"
|
||||
+ " applied. Matches OCRmyPDF's --rotate-pages-threshold scale",
|
||||
minimum = "0",
|
||||
defaultValue = "14.0")
|
||||
@Min(value = 0, message = "Confidence threshold must be non-negative")
|
||||
private Double confidenceThreshold = 14.0;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"If true, no rotation is applied; returns a JSON report of the per-page"
|
||||
+ " detection results instead of a PDF")
|
||||
private boolean dryRun;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"When a page cannot be decided on its own but the pages that could be decided"
|
||||
+ " agree on a single correction for that same current rotation, apply"
|
||||
+ " that shared correction to the undecided page. Handles documents"
|
||||
+ " rotated uniformly where some pages are too sparse to detect alone",
|
||||
defaultValue = "true")
|
||||
private boolean inferUndetected = true;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Optional pre-computed corrections to apply without running detection. Pages"
|
||||
+ " not listed are left unchanged, and a page may only appear once")
|
||||
@Valid
|
||||
private List<PageRotation> pageRotations;
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** A pre-computed rotation for one page, used by auto-rotate-pdf's apply-only path. */
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class PageRotation {
|
||||
|
||||
@Schema(
|
||||
description = "1-based page number to rotate",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example = "1")
|
||||
private Integer pageNumber;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Additional clockwise rotation to add to the page's current rotation, in"
|
||||
+ " degrees. Must be a multiple of 90",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example = "90")
|
||||
private Integer rotation;
|
||||
}
|
||||
@@ -200,8 +200,8 @@ public class VeraPDFService {
|
||||
|
||||
try (PDFAParser detectionParser =
|
||||
Foundries.defaultInstance().createParser(new ByteArrayInputStream(pdfBytes))) {
|
||||
detectedFlavours = detectedFlavours(detectionParser);
|
||||
declaredFlavour = firstFlavour(detectedFlavours);
|
||||
declaredFlavour = detectionParser.getFlavour();
|
||||
detectedFlavours = detectionParser.getFlavours();
|
||||
}
|
||||
|
||||
// For PDF/A flavours, we need to validate first to check if PDF/A identification exists in
|
||||
@@ -299,7 +299,7 @@ public class VeraPDFService {
|
||||
Foundries.defaultInstance()
|
||||
.createParser(new ByteArrayInputStream(pdfBytes), flavour)) {
|
||||
|
||||
PDFAFlavour parserDeclared = firstFlavour(detectedFlavours(parser));
|
||||
PDFAFlavour parserDeclared = parser.getFlavour();
|
||||
PDFAValidator validator =
|
||||
Foundries.defaultInstance().createValidator(flavour, false);
|
||||
ValidationResult result = validator.validate(parser);
|
||||
@@ -322,19 +322,7 @@ public class VeraPDFService {
|
||||
}
|
||||
|
||||
private static boolean isPdfaFlavour(PDFAFlavour flavour) {
|
||||
return flavour != null
|
||||
&& PDFFlavours.isFlavourFamily(flavour, PDFAFlavour.SpecificationFamily.PDF_A);
|
||||
}
|
||||
|
||||
// veraPDF 1.30+ returns an empty flavour list for non-PDF/A files, where getFlavour() throws
|
||||
private static List<PDFAFlavour> detectedFlavours(PDFAParser parser) {
|
||||
List<PDFAFlavour> flavours = parser.getFlavours();
|
||||
return flavours != null ? flavours : List.of();
|
||||
}
|
||||
|
||||
// null means "no PDF/A flavour detected" rather than an error
|
||||
private static PDFAFlavour firstFlavour(List<PDFAFlavour> flavours) {
|
||||
return flavours.isEmpty() ? null : flavours.get(0);
|
||||
return PDFFlavours.isFlavourFamily(flavour, PDFAFlavour.SpecificationFamily.PDF_A);
|
||||
}
|
||||
|
||||
private static String formatStandardDisplay(
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
package stirling.software.SPDF.utils;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Page-orientation detection primitives for the auto-rotate tool.
|
||||
*
|
||||
* <p>Two independent signals are supported: the dominant direction of a page's embedded text
|
||||
* (cheap, digital PDFs only) and Tesseract's orientation-and-script-detection output (works on
|
||||
* scans, requires the external binary). Both express their result as the additional clockwise
|
||||
* /Rotate correction that makes the page display upright.
|
||||
*/
|
||||
public final class AutoRotateDetection {
|
||||
|
||||
private AutoRotateDetection() {}
|
||||
|
||||
/** Glyphs needed to trust the text signal at the ordinary dominance bar. */
|
||||
public static final int MIN_GLYPHS = 30;
|
||||
|
||||
/** Fraction of glyphs that must share one direction at the ordinary bar. */
|
||||
public static final double MIN_DOMINANCE = 0.95;
|
||||
|
||||
/**
|
||||
* Glyphs needed to trust the text signal when the glyphs are near-unanimous. Lets sparse pages
|
||||
* (a header, a single line, a rotated URL) be decided from their own text instead of falling
|
||||
* through to OSD, as long as effectively every glyph agrees on the direction.
|
||||
*/
|
||||
public static final int MIN_GLYPHS_UNANIMOUS = 8;
|
||||
|
||||
/** Dominance required for the sparse-page path — essentially total agreement. */
|
||||
public static final double UNANIMOUS_DOMINANCE = 0.99;
|
||||
|
||||
/**
|
||||
* Dominant embedded-text direction of one page.
|
||||
*
|
||||
* @param dominantDirection glyph direction in page space, degrees CCW (0/90/180/270)
|
||||
* @param dominance fraction of counted glyphs sharing the dominant direction (0..1)
|
||||
* @param glyphCount number of non-whitespace glyphs counted
|
||||
*/
|
||||
public record TextDirection(int dominantDirection, double dominance, int glyphCount) {
|
||||
|
||||
public boolean isConclusive() {
|
||||
if (glyphCount >= MIN_GLYPHS && dominance >= MIN_DOMINANCE) {
|
||||
return true;
|
||||
}
|
||||
return glyphCount >= MIN_GLYPHS_UNANIMOUS && dominance >= UNANIMOUS_DOMINANCE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed Tesseract OSD verdict.
|
||||
*
|
||||
* @param rotate clockwise degrees to rotate the rendered page so text is upright
|
||||
* @param confidence Tesseract's orientation confidence (same scale OCRmyPDF thresholds on)
|
||||
*/
|
||||
public record OsdResult(int rotate, double confidence) {}
|
||||
|
||||
private static final Pattern OSD_ROTATE =
|
||||
Pattern.compile("^Rotate:\\s*(\\d+)", Pattern.MULTILINE);
|
||||
private static final Pattern OSD_CONFIDENCE =
|
||||
Pattern.compile("^Orientation confidence:\\s*([0-9.]+)", Pattern.MULTILINE);
|
||||
|
||||
/**
|
||||
* Counts non-whitespace glyph directions for every page in one pass. A stripper per page would
|
||||
* re-walk the document once per page, which is quadratic on long documents; this walks it once
|
||||
* and buckets glyphs by the page being processed.
|
||||
*
|
||||
* @return one entry per page, in page order
|
||||
*/
|
||||
public static List<TextDirection> detectTextDirections(PDDocument document) throws IOException {
|
||||
int pageCount = document.getNumberOfPages();
|
||||
DirectionCountingStripper stripper = new DirectionCountingStripper(pageCount);
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(pageCount);
|
||||
stripper.getText(document);
|
||||
|
||||
List<TextDirection> directions = new ArrayList<>(pageCount);
|
||||
for (int page = 0; page < pageCount; page++) {
|
||||
int[] counts = stripper.counts[page];
|
||||
int total = 0;
|
||||
int bestIndex = 0;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
total += counts[i];
|
||||
if (counts[i] > counts[bestIndex]) {
|
||||
bestIndex = i;
|
||||
}
|
||||
}
|
||||
double dominance = total == 0 ? 0 : (double) counts[bestIndex] / total;
|
||||
directions.add(new TextDirection(bestIndex * 90, dominance, total));
|
||||
}
|
||||
return directions;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a rendered page carries no ink worth analysing. Checked after rendering but before
|
||||
* spawning Tesseract, since the process spawn costs far more than the pixel scan and this
|
||||
* catches both empty generated pages and scanned blanks (the back of a duplex sheet).
|
||||
*/
|
||||
public static boolean isBlankRender(BufferedImage image) {
|
||||
final int darkThreshold = 200; // 8-bit grey; anything lighter counts as paper
|
||||
final int step = 4; // subsample: blank pages are uniform, no need for every pixel
|
||||
long sampled = 0;
|
||||
long dark = 0;
|
||||
for (int y = 0; y < image.getHeight(); y += step) {
|
||||
for (int x = 0; x < image.getWidth(); x += step) {
|
||||
sampled++;
|
||||
if ((image.getRGB(x, y) & 0xFF) < darkThreshold) {
|
||||
dark++;
|
||||
// A page needs a meaningful amount of ink before OSD can do anything;
|
||||
// bail out as soon as we know there is enough.
|
||||
if (dark > sampled / 1000 + 20) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clockwise /Rotate correction for a page whose dominant glyph direction (page space, CCW) is
|
||||
* {@code dominantDirection} and whose current /Rotate is {@code pageRotation}. Derivation: the
|
||||
* on-screen text angle is (direction - rotation) CCW, and adding d to /Rotate turns the display
|
||||
* a further d clockwise, so the correction that zeroes the screen angle is their difference.
|
||||
*/
|
||||
public static int correctionFromTextDirection(int dominantDirection, int pageRotation) {
|
||||
return Math.floorMod(dominantDirection - pageRotation, 360);
|
||||
}
|
||||
|
||||
/** Extracts rotation and confidence from `tesseract <img> stdout --psm 0` output. */
|
||||
public static Optional<OsdResult> parseOsd(String tesseractOutput) {
|
||||
if (tesseractOutput == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Matcher rotate = OSD_ROTATE.matcher(tesseractOutput);
|
||||
Matcher confidence = OSD_CONFIDENCE.matcher(tesseractOutput);
|
||||
if (!rotate.find() || !confidence.find()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
try {
|
||||
return Optional.of(
|
||||
new OsdResult(
|
||||
Integer.parseInt(rotate.group(1)),
|
||||
Double.parseDouble(confidence.group(1))));
|
||||
} catch (NumberFormatException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
private static class DirectionCountingStripper extends PDFTextStripper {
|
||||
|
||||
// counts[page][i] holds glyphs on that page whose direction is i * 90 degrees
|
||||
final int[][] counts;
|
||||
|
||||
DirectionCountingStripper(int pageCount) throws IOException {
|
||||
this.counts = new int[pageCount][4];
|
||||
}
|
||||
|
||||
/**
|
||||
* PDFBox snaps glyph direction to a quadrant, so getDir() only ever yields 0/90/180/270 —
|
||||
* obliquely drawn text (30, 45, 135 degrees) is reported as 0 rather than as its true
|
||||
* angle. Skew is therefore invisible to this signal by construction, which is consistent
|
||||
* with skew being out of scope here: only 90-degree orientation is corrected.
|
||||
*/
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
String unicode = text.getUnicode();
|
||||
if (unicode == null || unicode.isBlank()) {
|
||||
return;
|
||||
}
|
||||
int page = getCurrentPageNo() - 1;
|
||||
if (page < 0 || page >= counts.length) {
|
||||
return;
|
||||
}
|
||||
counts[page][Math.floorMod(Math.round(text.getDir()), 360) / 90]++;
|
||||
// super is intentionally not called: we only count, no text assembly needed
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -292,26 +292,6 @@ storage:
|
||||
linkExpirationDays: 3 # Number of days before share links expire
|
||||
signing:
|
||||
enabled: false # set to 'true' to enable group signing workflow (requires storage.enabled) [ALPHA]
|
||||
# ====================================================================================
|
||||
# ENCRYPTION AT REST - PRO / ENTERPRISE LICENSE REQUIRED TO ENABLE
|
||||
# ====================================================================================
|
||||
# Encrypts stored files (AES-256 envelope encryption, per-team keys). The master key is
|
||||
# resolved in this order:
|
||||
# 1. stirling.security.fileEncryptionKey property
|
||||
# 2. STIRLING_FILE_ENCRYPTION_KEY environment variable
|
||||
# 3. an auto-generated configs/file-encryption.key (single-node only; cluster mode
|
||||
# requires an explicitly shared key on every node)
|
||||
# Generate a key with: openssl rand -base64 32
|
||||
#
|
||||
# *** BACK UP THE MASTER KEY. Losing it makes every encrypted stored file ***
|
||||
# *** permanently unrecoverable. Verify backups against the key fingerprint logged ***
|
||||
# *** at startup. ***
|
||||
#
|
||||
# Enabling encrypts new writes only (existing files stay readable as plaintext).
|
||||
# Disabling later only stops encrypting new writes - existing encrypted files remain
|
||||
# readable as long as the key material is present.
|
||||
encryption:
|
||||
enabled: false # set to 'true' to encrypt stored files at rest
|
||||
userListScope: org # Signing user-picker scope: 'org' (default) = whole instance, else caller's team only.
|
||||
autoPipeline:
|
||||
outputFolder: "" # Output folder for processed pipeline files (leave empty for default)
|
||||
@@ -416,6 +396,14 @@ aiEngine:
|
||||
pdfComment: true # AI-authored PDF comments/annotations
|
||||
classify: true # Automatic document classification/labelling
|
||||
|
||||
# DocParse: document understanding for ingestion pipelines (chunking + knowledge-base
|
||||
# indexing). The basic tier (text layer) always works; the advanced tier (layout parsing)
|
||||
# requires the engine's docparse addon. Env overrides: DOCPARSE_ENABLED, DOCPARSE_MODE.
|
||||
docparse:
|
||||
enabled: true # Master switch; hides the DocParse endpoints when false
|
||||
mode: auto # Tier selection: 'auto' (best available), 'basic', or 'advanced'
|
||||
autoInstall: false # Mirrors DOCPARSE_AUTO_INSTALL for the engine's boot-time addon install script
|
||||
|
||||
policies:
|
||||
# Folder automations can read from and write to the directories you allow here, so treat this as a
|
||||
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs,
|
||||
|
||||
-342
@@ -1,342 +0,0 @@
|
||||
package stirling.software.SPDF.controller.api.misc;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotatePdfRequest;
|
||||
import stirling.software.SPDF.model.api.misc.PageRotation;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AutoRotateControllerTest {
|
||||
|
||||
private static final String SAMPLE_TEXT =
|
||||
"The quick brown fox jumps over the lazy dog again and again";
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private EndpointConfiguration endpointConfiguration;
|
||||
@Mock private RuntimePathConfig runtimePathConfig;
|
||||
@Mock private ApplicationProperties applicationProperties;
|
||||
|
||||
@InjectMocks private AutoRotateController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile("test", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
lenient().when(endpointConfiguration.isGroupEnabled("tesseract")).thenReturn(false);
|
||||
}
|
||||
|
||||
private static PDDocument docWithUprightText(int... pageRotations) throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
for (int rotation : pageRotations) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(72, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(rotation);
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
private AutoRotatePdfRequest request(PDDocument document) throws IOException {
|
||||
AutoRotatePdfRequest request = new AutoRotatePdfRequest();
|
||||
request.setFileInput(
|
||||
new MockMultipartFile(
|
||||
"fileInput",
|
||||
"test.pdf",
|
||||
MediaType.APPLICATION_PDF_VALUE,
|
||||
new byte[] {1, 2, 3}));
|
||||
when(pdfDocumentFactory.load(request)).thenReturn(document);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static PDDocument reload(ResponseEntity<?> response) throws IOException {
|
||||
Resource resource = (Resource) response.getBody();
|
||||
return Loader.loadPDF(resource.getContentAsByteArray());
|
||||
}
|
||||
|
||||
private static PDDocument docWithTextAt(int textAngleDegrees, int pageRotation)
|
||||
throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(textAngleDegrees), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(pageRotation);
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the dominant glyph direction straight from a document, independently of the production
|
||||
* detection code, so the round-trip assertion below validates the result rather than restating
|
||||
* the formula under test.
|
||||
*/
|
||||
private static int dominantGlyphDirection(PDDocument document) throws IOException {
|
||||
int[] counts = new int[4];
|
||||
PDFTextStripper stripper =
|
||||
new PDFTextStripper() {
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
if (!text.getUnicode().isBlank()) {
|
||||
counts[Math.floorMod(Math.round(text.getDir()), 360) / 90]++;
|
||||
}
|
||||
}
|
||||
};
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(1);
|
||||
stripper.getText(document);
|
||||
int best = 0;
|
||||
for (int i = 1; i < 4; i++) {
|
||||
if (counts[i] > counts[best]) {
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best * 90;
|
||||
}
|
||||
|
||||
/**
|
||||
* End-to-end round trip: build a page whose text is drawn at a known angle under a known
|
||||
* /Rotate, run the real controller, then assert the output actually displays upright. Upright
|
||||
* means the glyph direction and the page rotation cancel — computed here in the test, not via
|
||||
* the production helper.
|
||||
*/
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
"0, 0",
|
||||
"0, 90",
|
||||
"0, 180",
|
||||
"0, 270",
|
||||
"90, 0",
|
||||
"90, 90",
|
||||
"180, 0",
|
||||
"180, 270",
|
||||
"270, 90",
|
||||
})
|
||||
void roundTripLeavesPageUpright(int textAngle, int pageRotation) throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithTextAt(textAngle, pageRotation));
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
int glyphDirection = dominantGlyphDirection(corrected);
|
||||
int finalRotation = Math.floorMod(corrected.getPage(0).getRotation(), 360);
|
||||
assertThat(Math.floorMod(glyphDirection - finalRotation, 360))
|
||||
.as(
|
||||
"text drawn at %d under /Rotate %d should display upright, got glyph"
|
||||
+ " direction %d with /Rotate %d",
|
||||
textAngle, pageRotation, glyphDirection, finalRotation)
|
||||
.isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void dryRunReportsTextDetection() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(90, 0));
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
assertThat(result.getTotalPages()).isEqualTo(2);
|
||||
assertThat(result.getPagesToRotate()).isEqualTo(1);
|
||||
assertThat(result.getDetectedByText()).isEqualTo(2);
|
||||
|
||||
AutoRotateAnalysisResult.PageResult first = result.getPages().get(0);
|
||||
assertThat(first.getMethod()).isEqualTo("text");
|
||||
assertThat(first.getCorrection()).isEqualTo(270);
|
||||
assertThat(first.isApply()).isTrue();
|
||||
assertThat(first.getConfidence()).isEqualTo(100.0);
|
||||
|
||||
AutoRotateAnalysisResult.PageResult second = result.getPages().get(1);
|
||||
assertThat(second.getCorrection()).isZero();
|
||||
assertThat(second.isApply()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesDetectedCorrections() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(90, 0));
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
assertThat(corrected.getPage(0).getRotation()).isZero();
|
||||
assertThat(corrected.getPage(1).getRotation()).isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesExplicitPageRotations() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0, 0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 90)));
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
assertThat(corrected.getPage(0).getRotation()).isEqualTo(90);
|
||||
assertThat(corrected.getPage(1).getRotation()).isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportsTesseractUnavailableForTextlessPages() throws Exception {
|
||||
PDDocument document = new PDDocument();
|
||||
document.addPage(new PDPage(PDRectangle.LETTER));
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page = result.getPages().get(0);
|
||||
assertThat(page.getMethod()).isEqualTo("none");
|
||||
assertThat(page.getNote()).isEqualTo("tesseractUnavailable");
|
||||
assertThat(result.getUndetected()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void infersUndetectedPageFromDocumentConsensus() throws Exception {
|
||||
// Page 1 has body text and is rotated 90 (-> 270 correction); page 2 is blank and shares
|
||||
// the same rotation. With OSD unavailable, page 2 can't be detected on its own, so it
|
||||
// should inherit page 1's 270 correction.
|
||||
PDDocument document = docWithUprightText(90);
|
||||
PDPage blank = new PDPage(PDRectangle.LETTER);
|
||||
blank.setRotation(90);
|
||||
document.addPage(blank);
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page2 = result.getPages().get(1);
|
||||
assertThat(page2.getMethod()).isEqualTo("inferred");
|
||||
assertThat(page2.getCorrection()).isEqualTo(270);
|
||||
assertThat(page2.isApply()).isTrue();
|
||||
assertThat(page2.getNote()).isEqualTo("inferredFromDocument");
|
||||
assertThat(result.getInferred()).isEqualTo(1);
|
||||
assertThat(result.getPagesToRotate()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotInferWhenDisabled() throws Exception {
|
||||
PDDocument document = docWithUprightText(90);
|
||||
PDPage blank = new PDPage(PDRectangle.LETTER);
|
||||
blank.setRotation(90);
|
||||
document.addPage(blank);
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
request.setInferUndetected(false);
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page2 = result.getPages().get(1);
|
||||
assertThat(page2.getMethod()).isEqualTo("none");
|
||||
assertThat(page2.isApply()).isFalse();
|
||||
assertThat(result.getInferred()).isZero();
|
||||
assertThat(result.getUndetected()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsInvalidDetectionMode() {
|
||||
AutoRotatePdfRequest request = new AutoRotatePdfRequest();
|
||||
request.setDetectionMode("magic");
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsRotationThatIsNotAMultipleOf90() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 45)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsPageRotationOutsideTheDocument() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0));
|
||||
request.setPageRotations(List.of(new PageRotation(5, 90)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsDuplicatePageInRotations() throws Exception {
|
||||
// Rotations are additive, so applying the same page twice would over-rotate it.
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0, 0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 90), new PageRotation(1, 90)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
-22
@@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
@@ -292,27 +291,6 @@ class ValidateSignatureControllerMoreTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Padded signature contents")
|
||||
class PaddedContentsTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("Zero-padded /Contents still parses as CMS")
|
||||
void zeroPaddedContentsParsesAsCms() throws Exception {
|
||||
byte[] contents;
|
||||
try (PDDocument doc = Loader.loadPDF(signedPdfBytes)) {
|
||||
PDSignature sig = doc.getSignatureDictionaries().get(0);
|
||||
contents = sig.getContents(new ByteArrayInputStream(signedPdfBytes));
|
||||
}
|
||||
|
||||
// /Contents is zero-filled up to its reserved length. Since BC 1.85 the byte[]
|
||||
// constructors reject that tail ("Extra data detected in stream"), streams do not.
|
||||
assertThat(contents[contents.length - 1]).isZero();
|
||||
CMSSignedData parsed = new CMSSignedData(new ByteArrayInputStream(contents));
|
||||
assertThat(parsed.getSignerInfos().size()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Error and edge handling")
|
||||
class ErrorHandlingTests {
|
||||
|
||||
-197
@@ -1,197 +0,0 @@
|
||||
package stirling.software.SPDF.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.verapdf.gf.foundry.VeraGreenfieldFoundryProvider;
|
||||
import org.verapdf.pdfa.Foundries;
|
||||
import org.verapdf.pdfa.PDFAParser;
|
||||
import org.verapdf.pdfa.PDFAValidator;
|
||||
import org.verapdf.pdfa.flavours.PDFAFlavour;
|
||||
import org.verapdf.pdfa.results.ValidationResult;
|
||||
|
||||
import stirling.software.SPDF.model.api.security.PDFVerificationResult;
|
||||
|
||||
/**
|
||||
* Exercises {@link VeraPDFService} against real PDF/A files. Fixtures were produced by Ghostscript
|
||||
* with the same flags as ConvertPDFToPDFA and independently confirmed conformant by veraPDF.
|
||||
*/
|
||||
class VeraPDFServicePdfaFixtureTest {
|
||||
|
||||
private static final String VALID_1B = "valid-pdfa-1b.pdf";
|
||||
private static final String VALID_2B = "valid-pdfa-2b.pdf";
|
||||
private static final String DECLARED_BUT_INVALID_1B = "declared-pdfa-1b-no-outputintent.pdf";
|
||||
|
||||
private VeraPDFService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new VeraPDFService();
|
||||
service.initialize();
|
||||
}
|
||||
|
||||
@Test
|
||||
void fixtures_areGenuinePdfaAccordingToVeraPdfItself() throws Exception {
|
||||
assertVeraPdfVerdict(VALID_1B, PDFAFlavour.PDFA_1_B, true);
|
||||
assertVeraPdfVerdict(VALID_2B, PDFAFlavour.PDFA_2_B, true);
|
||||
assertVeraPdfVerdict(DECLARED_BUT_INVALID_1B, PDFAFlavour.PDFA_1_B, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_realPdfa1b_reportsCompliantPdfa1b() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(VALID_1B));
|
||||
|
||||
assertEquals("1b", result.getStandard());
|
||||
assertEquals("1b", result.getValidationProfile());
|
||||
assertTrue(result.isDeclaredPdfa(), "Genuine PDF/A-1b must be reported as declared PDF/A");
|
||||
assertTrue(result.isCompliant(), "Genuine PDF/A-1b must validate as compliant");
|
||||
assertEquals(
|
||||
0, result.getTotalFailures(), () -> "Unexpected failures: " + messages(result));
|
||||
assertTrue(
|
||||
result.getStandardName().startsWith("PDF/A-"),
|
||||
"Display name should name the PDF/A standard, got: " + result.getStandardName());
|
||||
assertTrue(
|
||||
result.getStandardName().endsWith(" compliant"),
|
||||
"Display name should read as compliant, got: " + result.getStandardName());
|
||||
assertEquals(result.getStandardName(), result.getComplianceSummary());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_realPdfa2b_reportsPdfa2bAndNotPdfa1b() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(VALID_2B));
|
||||
|
||||
// Proves firstFlavour() returns the flavour actually declared, not just a non-null one
|
||||
assertEquals("2b", result.getStandard());
|
||||
assertEquals("2b", result.getValidationProfile());
|
||||
assertNotEquals("1b", result.getStandard());
|
||||
assertTrue(result.isDeclaredPdfa());
|
||||
assertTrue(result.isCompliant(), () -> "Unexpected failures: " + messages(result));
|
||||
assertEquals(0, result.getTotalFailures());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_plainPdf_reportsNotPdfaAndDoesNotThrowIndexOutOfBounds() throws Exception {
|
||||
byte[] pdfBytes = createSimplePdf();
|
||||
|
||||
// veraPDF 1.30 returns an empty flavour list here where 1.28 returned [1b]; get(0) threw
|
||||
List<PDFVerificationResult> results =
|
||||
assertDoesNotThrow(
|
||||
() -> service.validatePDF(new ByteArrayInputStream(pdfBytes)),
|
||||
"Empty veraPDF flavour list must not surface as IndexOutOfBoundsException");
|
||||
|
||||
assertEquals(1, results.size());
|
||||
PDFVerificationResult result = results.get(0);
|
||||
assertEquals("not-pdfa", result.getStandard());
|
||||
assertFalse(result.isDeclaredPdfa());
|
||||
assertFalse(result.isCompliant());
|
||||
assertEquals("Not PDF/A (no PDF/A identification metadata)", result.getStandardName());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_declaresPdfaButNotConformant_reportsFlavourWithFailures() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(DECLARED_BUT_INVALID_1B));
|
||||
|
||||
// "declares PDF/A but broken" must stay distinct from "not PDF/A at all"
|
||||
assertEquals("1b", result.getStandard());
|
||||
assertNotEquals("not-pdfa", result.getStandard());
|
||||
assertTrue(result.isDeclaredPdfa(), "XMP still declares pdfaid:part=1");
|
||||
assertFalse(result.isCompliant(), "Stripped OutputIntent must fail conformance");
|
||||
assertTrue(result.getTotalFailures() > 0, "Non-conformance must be reported as issues");
|
||||
assertTrue(
|
||||
result.getStandardName().endsWith(" with errors"),
|
||||
"Display name should flag errors, got: " + result.getStandardName());
|
||||
assertTrue(
|
||||
messages(result).contains("OutputIntent"),
|
||||
"Expected the missing OutputIntent to be reported, got: " + messages(result));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstFlavour_withEmptyList_returnsNullInsteadOfThrowing() throws Exception {
|
||||
Method method = VeraPDFService.class.getDeclaredMethod("firstFlavour", List.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
assertNull(method.invoke(null, List.of()));
|
||||
assertEquals(PDFAFlavour.PDFA_2_B, method.invoke(null, List.of(PDFAFlavour.PDFA_2_B)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectedFlavours_withNullFlavourList_returnsEmptyList() throws Exception {
|
||||
Method method =
|
||||
VeraPDFService.class.getDeclaredMethod("detectedFlavours", PDFAParser.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
PDFAParser parser = mock(PDFAParser.class);
|
||||
when(parser.getFlavours()).thenReturn(null);
|
||||
|
||||
assertEquals(List.of(), method.invoke(null, parser));
|
||||
}
|
||||
|
||||
private static void assertVeraPdfVerdict(
|
||||
String fixtureName, PDFAFlavour expectedFlavour, boolean expectedCompliant)
|
||||
throws Exception {
|
||||
VeraGreenfieldFoundryProvider.initialise();
|
||||
byte[] bytes = fixture(fixtureName);
|
||||
|
||||
List<PDFAFlavour> flavours;
|
||||
try (PDFAParser parser =
|
||||
Foundries.defaultInstance().createParser(new ByteArrayInputStream(bytes))) {
|
||||
flavours = parser.getFlavours();
|
||||
}
|
||||
assertEquals(List.of(expectedFlavour), flavours, fixtureName + " declared flavours");
|
||||
|
||||
try (PDFAParser parser =
|
||||
Foundries.defaultInstance()
|
||||
.createParser(new ByteArrayInputStream(bytes), expectedFlavour)) {
|
||||
PDFAValidator validator =
|
||||
Foundries.defaultInstance().createValidator(expectedFlavour, false);
|
||||
ValidationResult result = validator.validate(parser);
|
||||
assertEquals(
|
||||
expectedCompliant, result.isCompliant(), fixtureName + " veraPDF compliance");
|
||||
}
|
||||
}
|
||||
|
||||
private PDFVerificationResult onlyResult(byte[] pdfBytes) throws Exception {
|
||||
List<PDFVerificationResult> results =
|
||||
service.validatePDF(new ByteArrayInputStream(pdfBytes));
|
||||
|
||||
assertNotNull(results);
|
||||
assertEquals(1, results.size(), () -> "Expected a single result, got: " + results);
|
||||
return results.get(0);
|
||||
}
|
||||
|
||||
private static String messages(PDFVerificationResult result) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (PDFVerificationResult.ValidationIssue issue : result.getFailures()) {
|
||||
builder.append(issue.getMessage()).append(" | ");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private static byte[] fixture(String name) throws IOException {
|
||||
try (InputStream in =
|
||||
VeraPDFServicePdfaFixtureTest.class.getResourceAsStream("/pdfa/" + name)) {
|
||||
assertNotNull(in, "Missing test fixture /pdfa/" + name);
|
||||
return in.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] createSimplePdf() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
document.addPage(new PDPage());
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package stirling.software.SPDF.utils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.OsdResult;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.TextDirection;
|
||||
|
||||
class AutoRotateDetectionTest {
|
||||
|
||||
private static final String SAMPLE_TEXT =
|
||||
"The quick brown fox jumps over the lazy dog again and again";
|
||||
|
||||
private PDDocument docWithText(int textAngleDegrees, int pageRotation) throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(textAngleDegrees), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(pageRotation);
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ground truth per the PDF spec: /Rotate R displays the page R degrees clockwise, so upright
|
||||
* page-space text under /Rotate R needs a further (360 - R) % 360 to display upright again;
|
||||
* text drawn rotated T degrees CCW in page space needs T clockwise to correct. Combined, the
|
||||
* expected correction is (T - R) mod 360.
|
||||
*/
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
// textAngle, pageRotation, expectedCorrection
|
||||
"0, 0, 0",
|
||||
"0, 90, 270",
|
||||
"0, 180, 180",
|
||||
"0, 270, 90",
|
||||
"90, 0, 90",
|
||||
"180, 0, 180",
|
||||
"270, 0, 270",
|
||||
"90, 90, 0",
|
||||
"180, 90, 90",
|
||||
})
|
||||
void detectsCorrectionForRotatedTextAndPages(
|
||||
int textAngle, int pageRotation, int expectedCorrection) throws IOException {
|
||||
try (PDDocument document = docWithText(textAngle, pageRotation)) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
|
||||
assertThat(direction.isConclusive())
|
||||
.as(
|
||||
"direction should be conclusive, glyphs=%d dominance=%s",
|
||||
direction.glyphCount(), direction.dominance())
|
||||
.isTrue();
|
||||
assertThat(
|
||||
AutoRotateDetection.correctionFromTextDirection(
|
||||
direction.dominantDirection(),
|
||||
Math.floorMod(pageRotation, 360)))
|
||||
.isEqualTo(expectedCorrection);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedDirectionsAreNotConclusive() throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(100, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(90), 300, 200));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyPageIsNotConclusive() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
document.addPage(new PDPage(PDRectangle.LETTER));
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.glyphCount()).isZero();
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shortTextIsNotConclusive() throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(100, 400));
|
||||
content.showText("Short");
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unanimousShortTextIsConclusive() throws IOException {
|
||||
// Between MIN_GLYPHS_UNANIMOUS (8) and MIN_GLYPHS (30): trusted only because every
|
||||
// glyph agrees on direction, the sparse-page path (e.g. a lone header or URL line).
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(90), 300, 200));
|
||||
content.showText("york.gov.uk/pay");
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.glyphCount())
|
||||
.isBetween(
|
||||
AutoRotateDetection.MIN_GLYPHS_UNANIMOUS,
|
||||
AutoRotateDetection.MIN_GLYPHS - 1);
|
||||
assertThat(direction.dominance()).isEqualTo(1.0);
|
||||
assertThat(direction.isConclusive()).isTrue();
|
||||
assertThat(direction.dominantDirection()).isEqualTo(90);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void bucketsGlyphsPerPageInOneWalk() throws IOException {
|
||||
// Each page carries text at a different angle; the single-pass walk must attribute
|
||||
// glyphs to the right page rather than pooling them.
|
||||
int[] angles = {0, 90, 180, 270};
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
for (int angle : angles) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(angle), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
}
|
||||
|
||||
List<TextDirection> directions = AutoRotateDetection.detectTextDirections(document);
|
||||
|
||||
assertThat(directions).hasSize(angles.length);
|
||||
for (int i = 0; i < angles.length; i++) {
|
||||
assertThat(directions.get(i).isConclusive()).as("page %d", i + 1).isTrue();
|
||||
assertThat(directions.get(i).dominantDirection())
|
||||
.as("page %d direction", i + 1)
|
||||
.isEqualTo(angles[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsBlankAndInkedRenders() {
|
||||
BufferedImage blank = new BufferedImage(200, 200, BufferedImage.TYPE_BYTE_GRAY);
|
||||
Graphics2D g = blank.createGraphics();
|
||||
g.setColor(Color.WHITE);
|
||||
g.fillRect(0, 0, 200, 200);
|
||||
g.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(blank)).isTrue();
|
||||
|
||||
BufferedImage speck = copyOf(blank);
|
||||
Graphics2D specked = speck.createGraphics();
|
||||
specked.setColor(Color.BLACK);
|
||||
specked.fillRect(0, 0, 2, 2); // a dust speck must not count as content
|
||||
specked.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(speck)).isTrue();
|
||||
|
||||
BufferedImage inked = copyOf(blank);
|
||||
Graphics2D inkedG = inked.createGraphics();
|
||||
inkedG.setColor(Color.BLACK);
|
||||
inkedG.fillRect(20, 20, 120, 60);
|
||||
inkedG.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(inked)).isFalse();
|
||||
}
|
||||
|
||||
private static BufferedImage copyOf(BufferedImage source) {
|
||||
BufferedImage copy =
|
||||
new BufferedImage(source.getWidth(), source.getHeight(), source.getType());
|
||||
Graphics2D g = copy.createGraphics();
|
||||
g.drawImage(source, 0, 0, null);
|
||||
g.dispose();
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesTypicalOsdOutput() {
|
||||
String output =
|
||||
"""
|
||||
Estimating resolution as 336
|
||||
Page number: 0
|
||||
Orientation in degrees: 180
|
||||
Rotate: 180
|
||||
Orientation confidence: 9.15
|
||||
Script: Latin
|
||||
Script confidence: 4.43
|
||||
""";
|
||||
Optional<OsdResult> result = AutoRotateDetection.parseOsd(output);
|
||||
assertThat(result).isPresent();
|
||||
assertThat(result.get().rotate()).isEqualTo(180);
|
||||
assertThat(result.get().confidence()).isEqualTo(9.15);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseOsdRejectsIncompleteOutput() {
|
||||
assertThat(AutoRotateDetection.parseOsd("Too few characters. Skipping this page"))
|
||||
.isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd("Rotate: 90")).isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd(null)).isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd("")).isEmpty();
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -67,9 +67,6 @@ dependencies {
|
||||
implementation "software.amazon.awssdk:s3:${awsSdkVersion}"
|
||||
implementation "software.amazon.awssdk:url-connection-client:${awsSdkVersion}"
|
||||
|
||||
// Streaming AEAD (AES-GCM-HKDF segments) for storage encryption at rest. Apache-2.0.
|
||||
implementation "com.google.crypto.tink:tink:${tinkVersion}"
|
||||
|
||||
// @DataJpaTest slice (Boot 4 ships test slices as separate starters, like webmvc-test at the
|
||||
// root) so policy.source repositories can be exercised against embedded H2.
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test'
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.model.api.docparse.ExtractTablesApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.RagIngestApiRequest;
|
||||
import stirling.software.proprietary.model.docparse.DocChunk;
|
||||
import stirling.software.proprietary.model.docparse.DocTable;
|
||||
import stirling.software.proprietary.model.docparse.DocparseCapabilitiesView;
|
||||
import stirling.software.proprietary.model.docparse.DocparseMode;
|
||||
import stirling.software.proprietary.model.docparse.ExtractTablesResponse;
|
||||
import stirling.software.proprietary.model.docparse.RagIngestResponse;
|
||||
import stirling.software.proprietary.service.AiToolResponseHeaders;
|
||||
import stirling.software.proprietary.service.DocParseService;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Public DocParse ingestion API. Thin HTTP layer over {@link DocParseService}, which owns the
|
||||
* engine wire contract; this class owns the pipeline step shape (report header, export ZIP).
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/docparse")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(
|
||||
name = "DocParse",
|
||||
description =
|
||||
"Document ingestion: chunk, embed, and index documents into the searchable"
|
||||
+ " knowledge base, or export the parsed content (markdown, chunks JSONL)"
|
||||
+ " for external systems.")
|
||||
public class DocParseController {
|
||||
|
||||
private static final MediaType CSV = MediaType.parseMediaType("text/csv");
|
||||
|
||||
private final DocParseService docParseService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/rag-ingest",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Chunk, embed, and index a document into the RAG store (pipeline shape)",
|
||||
description =
|
||||
"Ingests the document into the engine's RAG store under a stable documentId"
|
||||
+ " (default: content hash). Returns the ORIGINAL PDF unchanged as the"
|
||||
+ " body, with the ingest summary JSON in the X-Stirling-Tool-Report"
|
||||
+ " header so policy pipelines pick it up as the step report. With"
|
||||
+ " exportMarkdown/exportChunksJsonl the body becomes a ZIP holding the"
|
||||
+ " original plus the corpus files, ready for delivery to external"
|
||||
+ " systems. Input:PDF Output:PDF/ZIP Type:SISO")
|
||||
public ResponseEntity<Resource> ragIngest(@ModelAttribute RagIngestApiRequest request)
|
||||
throws IOException {
|
||||
MultipartFile file = request.getFileInput();
|
||||
boolean export = request.isExportMarkdown() || request.isExportChunksJsonl();
|
||||
RagIngestResponse result =
|
||||
docParseService.ragIngest(
|
||||
file,
|
||||
request.getDocumentId(),
|
||||
request.getChunkSize(),
|
||||
request.getOverlap(),
|
||||
DocparseMode.fromWire(request.getMode()),
|
||||
request.isIndex(),
|
||||
request.isExportMarkdown(),
|
||||
request.isExportChunksJsonl());
|
||||
|
||||
// The report header must stay small: summary fields only, never the echoed content.
|
||||
ObjectNode report = objectMapper.createObjectNode();
|
||||
report.put("mode", result.mode().wire());
|
||||
report.put("documentId", result.documentId());
|
||||
report.put("chunksIndexed", result.chunksIndexed());
|
||||
report.put("pages", result.pages());
|
||||
report.put("indexed", request.isIndex());
|
||||
|
||||
String fileName = DocParseService.fileName(file);
|
||||
byte[] original = file.getBytes();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set(AiToolResponseHeaders.TOOL_REPORT, objectMapper.writeValueAsString(report));
|
||||
|
||||
if (!export) {
|
||||
headers.setContentType(MediaType.APPLICATION_PDF);
|
||||
headers.setContentDispositionFormData("attachment", fileName);
|
||||
headers.setContentLength(original.length);
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(original));
|
||||
}
|
||||
|
||||
byte[] zip = exportZip(fileName, original, result, request);
|
||||
headers.setContentType(MediaType.parseMediaType("application/zip"));
|
||||
headers.setContentDispositionFormData("attachment", baseName(fileName) + "-ingested.zip");
|
||||
headers.setContentLength(zip.length);
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(zip));
|
||||
}
|
||||
|
||||
@GetMapping("/capabilities")
|
||||
@Operation(
|
||||
summary = "DocParse capability summary",
|
||||
description =
|
||||
"Merged view of the Java settings and the engine's capability probe, so"
|
||||
+ " clients can gate advanced-tier UI.")
|
||||
public ResponseEntity<DocparseCapabilitiesView> capabilities() {
|
||||
return ResponseEntity.ok(docParseService.capabilitiesView());
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/extract-tables",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Extract tables from a document",
|
||||
description =
|
||||
"Extracts table structure and returns CSV (all tables concatenated, blank line"
|
||||
+ " between them) or the structured JSON table list."
|
||||
+ " Input:PDF Output:CSV/JSON Type:SISO")
|
||||
public ResponseEntity<?> extractTables(@ModelAttribute ExtractTablesApiRequest request)
|
||||
throws IOException {
|
||||
ExtractTablesResponse result = docParseService.tables(request.getFileInput());
|
||||
if ("json".equalsIgnoreCase(request.getOutputFormat())) {
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
return WebResponseUtils.bytesToWebResponse(
|
||||
tablesToCsv(result.tables()).getBytes(StandardCharsets.UTF_8),
|
||||
outputName(request.getFileInput(), "_tables.csv"),
|
||||
CSV);
|
||||
}
|
||||
|
||||
/** Original + requested corpus files in one ZIP, so destinations receive them together. */
|
||||
private byte[] exportZip(
|
||||
String fileName, byte[] original, RagIngestResponse result, RagIngestApiRequest request)
|
||||
throws IOException {
|
||||
String base = baseName(fileName);
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zip = new ZipOutputStream(out)) {
|
||||
zip.putNextEntry(new ZipEntry(fileName));
|
||||
zip.write(original);
|
||||
zip.closeEntry();
|
||||
if (request.isExportMarkdown()) {
|
||||
zip.putNextEntry(new ZipEntry(base + ".md"));
|
||||
zip.write(
|
||||
(result.markdown() == null ? "" : result.markdown())
|
||||
.getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
if (request.isExportChunksJsonl()) {
|
||||
zip.putNextEntry(new ZipEntry(base + ".chunks.jsonl"));
|
||||
zip.write(chunksJsonl(result).getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
/** One chunk per line, each self-describing (documentId + source travel on every line). */
|
||||
private String chunksJsonl(RagIngestResponse result) {
|
||||
if (result.chunks() == null) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder lines = new StringBuilder();
|
||||
for (DocChunk chunk : result.chunks()) {
|
||||
ObjectNode line = objectMapper.createObjectNode();
|
||||
line.put("documentId", result.documentId());
|
||||
line.put("index", chunk.index());
|
||||
line.put("text", chunk.text());
|
||||
if (chunk.pageStart() != null) {
|
||||
line.put("pageStart", chunk.pageStart());
|
||||
}
|
||||
if (chunk.pageEnd() != null) {
|
||||
line.put("pageEnd", chunk.pageEnd());
|
||||
}
|
||||
var headings = line.putArray("headingPath");
|
||||
chunk.headingPath().forEach(headings::add);
|
||||
lines.append(objectMapper.writeValueAsString(line)).append('\n');
|
||||
}
|
||||
return lines.toString();
|
||||
}
|
||||
|
||||
private static String baseName(String fileName) {
|
||||
int dot = fileName.lastIndexOf('.');
|
||||
return dot > 0 ? fileName.substring(0, dot) : fileName;
|
||||
}
|
||||
|
||||
private static String tablesToCsv(List<DocTable> tables) throws IOException {
|
||||
CSVFormat format = CSVFormat.EXCEL.builder().setEscape('"').build();
|
||||
StringWriter writer = new StringWriter();
|
||||
try (CSVPrinter printer = format.print(writer)) {
|
||||
boolean first = true;
|
||||
for (DocTable table : tables) {
|
||||
if (!first) {
|
||||
printer.println();
|
||||
}
|
||||
first = false;
|
||||
for (List<String> row : table.cells()) {
|
||||
printer.printRecord(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
return writer.toString();
|
||||
}
|
||||
|
||||
private static String outputName(MultipartFile file, String suffix) {
|
||||
return GeneralUtils.removeExtension(DocParseService.fileName(file)) + suffix;
|
||||
}
|
||||
}
|
||||
-17
@@ -115,7 +115,6 @@ public class ExternalApiCallController {
|
||||
@RequestParam(value = "headers", required = false) String headers,
|
||||
@RequestParam(value = "includeContext", defaultValue = "false") boolean includeContext,
|
||||
@RequestParam(value = "includeFile", defaultValue = "true") boolean includeFile,
|
||||
@RequestParam(value = "maxRequestBytes", defaultValue = "0") long maxRequestBytes,
|
||||
@RequestHeader(value = InternalApiClient.POLICY_NAME_HEADER, required = false)
|
||||
String policyName,
|
||||
@RequestHeader(value = AutomationRunContext.RUN_ID_HEADER, required = false)
|
||||
@@ -139,17 +138,6 @@ public class ExternalApiCallController {
|
||||
: fileInput.getContentType();
|
||||
byte[] content = fileInput.getBytes();
|
||||
|
||||
// Some destinations cap uploads (Discord's varies with Nitro tier), so the operator
|
||||
// sets the limit; we fail clearly here rather than on an opaque vendor rejection.
|
||||
if (maxRequestBytes > 0 && content.length > maxRequestBytes) {
|
||||
throw new IllegalArgumentException(
|
||||
"The document is "
|
||||
+ megabytes(content.length)
|
||||
+ " MB, over the "
|
||||
+ megabytes(maxRequestBytes)
|
||||
+ " MB limit set for this step.");
|
||||
}
|
||||
|
||||
ObjectNode context =
|
||||
DocumentContext.build(fileInput, content, policyName, runId, objectMapper);
|
||||
|
||||
@@ -557,11 +545,6 @@ public class ExternalApiCallController {
|
||||
: oneLine.substring(0, MAX_REPORT_BODY_CHARS) + "…";
|
||||
}
|
||||
|
||||
/** Bytes as MB to one decimal, for a size message an operator reads in the units they set. */
|
||||
private static String megabytes(long bytes) {
|
||||
return String.format(Locale.ROOT, "%.1f", bytes / (1024.0 * 1024.0));
|
||||
}
|
||||
|
||||
private static String safeFileName(String originalFilename) {
|
||||
String name = Filenames.toSimpleFileName(originalFilename);
|
||||
return (name == null || name.isBlank()) ? "document" : name;
|
||||
|
||||
+2
-13
@@ -21,8 +21,6 @@ import tools.jackson.databind.node.StringNode;
|
||||
*/
|
||||
final class Placeholders {
|
||||
|
||||
private static final int MAX_TREE_DEPTH = 64;
|
||||
|
||||
private static final Pattern PLACEHOLDER = Pattern.compile("\\{\\{\\s*([\\w.]+)\\s*}}");
|
||||
|
||||
/** How a resolved value is escaped for the position it lands in. */
|
||||
@@ -70,24 +68,15 @@ final class Placeholders {
|
||||
* documents[0].data} as readily as a flat field - without a connector per vendor.
|
||||
*/
|
||||
static JsonNode resolveTree(JsonNode node, JsonNode context) {
|
||||
return resolveTree(node, context, 0);
|
||||
}
|
||||
|
||||
private static JsonNode resolveTree(JsonNode node, JsonNode context, int depth) {
|
||||
// Deeply nested JSON is left untouched rather than recursed into, so a
|
||||
// pathological template cannot overflow the stack.
|
||||
if (depth > MAX_TREE_DEPTH) {
|
||||
return node;
|
||||
}
|
||||
if (node instanceof ObjectNode object) {
|
||||
for (String name : new java.util.ArrayList<>(object.propertyNames())) {
|
||||
object.set(name, resolveTree(object.get(name), context, depth + 1));
|
||||
object.set(name, resolveTree(object.get(name), context));
|
||||
}
|
||||
return object;
|
||||
}
|
||||
if (node instanceof ArrayNode array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
array.set(i, resolveTree(array.get(i), context, depth + 1));
|
||||
array.set(i, resolveTree(array.get(i), context));
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ExtractTablesApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description = "Response format: CSV text or the structured JSON table list",
|
||||
allowableValues = {"csv", "json"},
|
||||
defaultValue = "csv")
|
||||
private String outputFormat = "csv";
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class RagIngestApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Stable identifier for the ingested document; re-ingesting the same id replaces"
|
||||
+ " its chunks. Defaults to a content hash of the uploaded bytes.")
|
||||
private String documentId;
|
||||
|
||||
@Schema(description = "Target chunk size in characters (64-32768)", defaultValue = "512")
|
||||
private int chunkSize = 512;
|
||||
|
||||
@Schema(
|
||||
description = "Overlap between adjacent chunks in characters (0-4096)",
|
||||
defaultValue = "64")
|
||||
private int overlap = 64;
|
||||
|
||||
@Schema(
|
||||
description = "Tier to use: 'auto' picks per document, or force 'basic'/'advanced'",
|
||||
allowableValues = {"auto", "basic", "advanced"},
|
||||
defaultValue = "auto")
|
||||
private String mode = "auto";
|
||||
|
||||
@Schema(
|
||||
description = "Index the document into the built-in knowledge base",
|
||||
defaultValue = "true")
|
||||
private boolean index = true;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Also return the parsed document as a markdown file, for delivery to external"
|
||||
+ " systems (vector DBs, training corpora)",
|
||||
defaultValue = "false")
|
||||
private boolean exportMarkdown = false;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Also return the chunks as a JSONL file (one chunk per line with page span and"
|
||||
+ " heading breadcrumb), ready for external embedding or indexing",
|
||||
defaultValue = "false")
|
||||
private boolean exportChunksJsonl = false;
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** One RAG chunk with page span and heading breadcrumb. Mirrors {@code docparse.py DocChunk}. */
|
||||
public record DocChunk(
|
||||
int index, String text, Integer pageStart, Integer pageEnd, List<String> headingPath) {
|
||||
|
||||
public DocChunk {
|
||||
headingPath = headingPath == null ? List.of() : headingPath;
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** One extracted table. Mirrors {@code docparse.py DocTable}. */
|
||||
public record DocTable(
|
||||
int page, List<Double> bbox, List<List<String>> cells, String markdown, Double confidence) {
|
||||
|
||||
public DocTable {
|
||||
cells = cells == null ? List.of() : cells;
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* What the engine can actually do right now; Java caches and republishes this. Mirrors {@code
|
||||
* docparse.py DocparseCapabilities}.
|
||||
*/
|
||||
public record DocparseCapabilities(
|
||||
boolean advancedInstalled,
|
||||
String doclingVersion,
|
||||
String torchVersion,
|
||||
boolean modelsAvailable,
|
||||
String modelsPath,
|
||||
List<String> errors) {
|
||||
|
||||
public DocparseCapabilities {
|
||||
errors = errors == null ? List.of() : errors;
|
||||
}
|
||||
|
||||
/** The addon-absent view used when the engine is disabled, unreachable, or probing failed. */
|
||||
public static DocparseCapabilities absent(String reason) {
|
||||
return new DocparseCapabilities(false, null, null, false, null, List.of(reason));
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Merged capability view served by {@code GET /api/v1/docparse/capabilities} (Java side). */
|
||||
public record DocparseCapabilitiesView(
|
||||
boolean enabled,
|
||||
String mode,
|
||||
boolean advancedInstalled,
|
||||
boolean engineReachable,
|
||||
String doclingVersion) {}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* What the caller asked for; {@code AUTO} resolves per request. Wire values are lowercase to match
|
||||
* {@code engine/src/stirling/contracts/docparse.py DocparseMode}.
|
||||
*/
|
||||
public enum DocparseMode {
|
||||
AUTO("auto"),
|
||||
BASIC("basic"),
|
||||
ADVANCED("advanced");
|
||||
|
||||
private final String wire;
|
||||
|
||||
DocparseMode(String wire) {
|
||||
this.wire = wire;
|
||||
}
|
||||
|
||||
@JsonValue
|
||||
public String wire() {
|
||||
return wire;
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public static DocparseMode fromWire(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return AUTO;
|
||||
}
|
||||
return valueOf(value.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Which implementation actually served a request. Wire values are lowercase to match {@code
|
||||
* engine/src/stirling/contracts/docparse.py DocparseTier}.
|
||||
*/
|
||||
public enum DocparseTier {
|
||||
BASIC("basic"),
|
||||
ADVANCED("advanced");
|
||||
|
||||
private final String wire;
|
||||
|
||||
DocparseTier(String wire) {
|
||||
this.wire = wire;
|
||||
}
|
||||
|
||||
@JsonValue
|
||||
public String wire() {
|
||||
return wire;
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public static DocparseTier fromWire(String value) {
|
||||
return valueOf(value.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/docparse/tables}. */
|
||||
public record ExtractTablesRequest(String fileName, String contentBase64) {}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Engine response for {@code POST /api/v1/docparse/tables}. */
|
||||
public record ExtractTablesResponse(DocparseTier mode, List<DocTable> tables) {
|
||||
|
||||
public ExtractTablesResponse {
|
||||
tables = tables == null ? List.of() : tables;
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
|
||||
/**
|
||||
* Engine request for {@code POST /api/v1/docparse/rag-ingest}. Owner semantics mirror {@code POST
|
||||
* /api/v1/documents}: {@code ownerId} is the tenant, {@code readPrincipals} the explicit readers,
|
||||
* and a null {@code expiresAt} keeps the ingested content until an explicit delete. {@code index}
|
||||
* false skips the store (export-only); {@code includeMarkdown}/{@code includeChunks} echo the
|
||||
* parsed content back so the caller can emit corpus files.
|
||||
*/
|
||||
public record RagIngestRequest(
|
||||
String fileName,
|
||||
String documentId,
|
||||
String source,
|
||||
String ownerId,
|
||||
List<String> readPrincipals,
|
||||
Instant expiresAt,
|
||||
List<AiPageText> pages,
|
||||
String contentBase64,
|
||||
int chunkSize,
|
||||
int overlap,
|
||||
DocparseMode mode,
|
||||
boolean index,
|
||||
boolean includeMarkdown,
|
||||
boolean includeChunks) {}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Engine response for {@code POST /api/v1/docparse/rag-ingest}. {@code markdown} and {@code chunks}
|
||||
* are only present when the request asked for them via includeMarkdown/includeChunks.
|
||||
*/
|
||||
public record RagIngestResponse(
|
||||
DocparseTier mode,
|
||||
String documentId,
|
||||
int chunksIndexed,
|
||||
int pages,
|
||||
String markdown,
|
||||
List<DocChunk> chunks) {}
|
||||
+2
-1
@@ -355,7 +355,8 @@ public class PolicyController {
|
||||
policy.name(),
|
||||
owner,
|
||||
policy.enabled(),
|
||||
policy.inputs(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
|
||||
+1
-101
@@ -3,7 +3,6 @@ package stirling.software.proprietary.policy.engine;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -34,7 +33,6 @@ import stirling.software.proprietary.service.AiToolResponseHeaders;
|
||||
import tools.jackson.core.JacksonException;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Runs an ordered chain of tool steps, feeding each step's output files into the next.
|
||||
@@ -51,10 +49,6 @@ public class PolicyExecutor {
|
||||
|
||||
private static final String FILTER_OPERATION_PREFIX = "/api/v1/filter/filter-";
|
||||
|
||||
// Nested list parameters are walked recursively; cap the depth so a pathological
|
||||
// pipeline cannot overflow the stack.
|
||||
private static final int MAX_PARAMETER_DEPTH = 32;
|
||||
|
||||
private final InternalApiClient internalApiClient;
|
||||
private final ToolMetadataService toolMetadataService;
|
||||
private final TempFileManager tempFileManager;
|
||||
@@ -95,10 +89,6 @@ public class PolicyExecutor {
|
||||
// Last non-null report wins: the terminal step defines the output.
|
||||
JsonNode lastReport = null;
|
||||
String lastReportTool = null;
|
||||
// Every step's report, keyed by 1-based position, so a later step can reference an earlier
|
||||
// one's response via {{steps.N...}} - e.g. post the share link an upload step returned.
|
||||
ObjectNode runContext = objectMapper.createObjectNode();
|
||||
ObjectNode stepReports = runContext.putObject("steps");
|
||||
|
||||
for (int i = 0; i < steps.size(); i++) {
|
||||
PipelineStep step = steps.get(i);
|
||||
@@ -108,17 +98,13 @@ public class PolicyExecutor {
|
||||
"Pipeline step " + (i + 1) + " has no operation");
|
||||
}
|
||||
listener.onStepStart(i + 1, steps.size(), operation);
|
||||
// Fill in references to earlier steps' outputs before dispatch; document- and run-scope
|
||||
// placeholders are left for the tool to resolve per document.
|
||||
PipelineStep resolved = resolveStepReferences(step, runContext);
|
||||
StepOutput stepResult =
|
||||
executeStep(resolved, currentFiles, currentOrigins, supportingFiles);
|
||||
executeStep(step, currentFiles, currentOrigins, supportingFiles);
|
||||
currentFiles = stepResult.files();
|
||||
currentOrigins = stepResult.origins();
|
||||
if (stepResult.report() != null) {
|
||||
lastReport = stepResult.report();
|
||||
lastReportTool = operation;
|
||||
stepReports.set(String.valueOf(i + 1), stepResult.report());
|
||||
}
|
||||
listener.onStepComplete(i + 1, steps.size(), operation);
|
||||
}
|
||||
@@ -177,92 +163,6 @@ public class PolicyExecutor {
|
||||
return new StepOutput(files, origins, report);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve {@code {{steps.N...}}} references in a step's string parameters against the reports
|
||||
* earlier steps produced. Returns the step unchanged when it references nothing, so a pipeline
|
||||
* that uses no cross-step values pays nothing and behaves exactly as before.
|
||||
*/
|
||||
private PipelineStep resolveStepReferences(PipelineStep step, JsonNode runContext) {
|
||||
boolean any = step.parameters().values().stream().anyMatch(this::referencesStep);
|
||||
if (!any) {
|
||||
return step;
|
||||
}
|
||||
Map<String, Object> resolved = new LinkedHashMap<>();
|
||||
step.parameters()
|
||||
.forEach((key, value) -> resolved.put(key, resolveValue(value, runContext)));
|
||||
return new PipelineStep(step.operation(), resolved, step.fileParameters());
|
||||
}
|
||||
|
||||
private boolean referencesStep(Object value) {
|
||||
return referencesStep(value, 0);
|
||||
}
|
||||
|
||||
private boolean referencesStep(Object value, int depth) {
|
||||
if (depth > MAX_PARAMETER_DEPTH) {
|
||||
return false;
|
||||
}
|
||||
if (value instanceof String s) {
|
||||
return StepOutputPlaceholders.references(s);
|
||||
}
|
||||
if (value instanceof List<?> list) {
|
||||
return list.stream().anyMatch(item -> referencesStep(item, depth + 1));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Object resolveValue(Object value, JsonNode runContext) {
|
||||
return resolveValue(value, runContext, 0);
|
||||
}
|
||||
|
||||
private Object resolveValue(Object value, JsonNode runContext, int depth) {
|
||||
if (depth > MAX_PARAMETER_DEPTH) {
|
||||
return value;
|
||||
}
|
||||
if (value instanceof String s) {
|
||||
return resolveString(s, runContext);
|
||||
}
|
||||
if (value instanceof List<?> list) {
|
||||
List<Object> out = new ArrayList<>(list.size());
|
||||
for (Object item : list) {
|
||||
out.add(resolveValue(item, runContext, depth + 1));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* A JSON-shaped parameter (bodyTemplate, fields, headers) is resolved inside its parsed tree,
|
||||
* so an earlier step's response can only ever become a value in it. String-level substitution
|
||||
* would let a response like {@code x", "admin": true, "y": "} inject fields into the JSON the
|
||||
* operator wrote; a plain-text parameter keeps the plain substitution.
|
||||
*/
|
||||
private String resolveString(String value, JsonNode runContext) {
|
||||
if (!StepOutputPlaceholders.references(value)) {
|
||||
return value;
|
||||
}
|
||||
JsonNode tree = parseJsonContainer(value);
|
||||
if (tree == null) {
|
||||
return StepOutputPlaceholders.resolve(value, runContext);
|
||||
}
|
||||
return objectMapper.writeValueAsString(
|
||||
StepOutputPlaceholders.resolveTree(tree, runContext));
|
||||
}
|
||||
|
||||
/** The value parsed as a JSON object or array, or null when it is anything else. */
|
||||
private JsonNode parseJsonContainer(String value) {
|
||||
String trimmed = value.trim();
|
||||
if (trimmed.isEmpty() || (trimmed.charAt(0) != '{' && trimmed.charAt(0) != '[')) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JsonNode node = objectMapper.readTree(value);
|
||||
return (node.isObject() || node.isArray()) ? node : null;
|
||||
} catch (JacksonException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Call an endpoint, returning result files and optional report. Response handling: JSON body is
|
||||
* the report with no file; a file body returns the file plus any {@link
|
||||
|
||||
+10
-28
@@ -15,7 +15,6 @@ import stirling.software.proprietary.policy.input.ResolvedInput;
|
||||
import stirling.software.proprietary.policy.ledger.ProcessedLedger;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineDefinition;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyInputs;
|
||||
import stirling.software.proprietary.policy.model.PolicyRun;
|
||||
@@ -43,46 +42,30 @@ public class PolicyRunner {
|
||||
private final SourceDocCounter docCounter;
|
||||
private final ProcessedLedger processedLedger;
|
||||
|
||||
/** Full-listing sweep over every input: resolve each source, then reconcile the ledger. */
|
||||
/** Full-listing sweep: resolve every source, then reconcile the ledger. */
|
||||
public SweepOutcome run(Policy policy) {
|
||||
return run(policy, SweepKind.FULL);
|
||||
}
|
||||
|
||||
/** Sweep every input of the policy at the given listing depth. */
|
||||
public SweepOutcome run(Policy policy, SweepKind sweep) {
|
||||
return run(policy, policy.inputs(), sweep);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire one input binding: a background trigger pulling its own source without touching the
|
||||
* policy's other inputs. Never reconciles the ledger (it sees a single source, so pruning would
|
||||
* wrongly forget the rest); a full-policy sweep handles that.
|
||||
*/
|
||||
public SweepOutcome runInput(Policy policy, PipelineInput input, SweepKind sweep) {
|
||||
return run(policy, List.of(input), sweep);
|
||||
}
|
||||
|
||||
/**
|
||||
* Core sweep: pulls each of the given inputs' sources; each yielded unit becomes its own run so
|
||||
* one failure does not affect the others. No inputs means one run with no input (generator
|
||||
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
|
||||
* one failure does not affect the others. No sources means one run with no input (generator
|
||||
* pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
|
||||
* rest. Presence cleanup only runs when the sweep covered every input of the policy - a
|
||||
* single-binding fire cannot reconcile the whole policy's ledger. Returns the ids of the runs
|
||||
* it started plus what the sweep skipped, so a manual trigger can report which runs to follow
|
||||
* or why nothing ran.
|
||||
* rest. Returns the ids of the runs it started plus what the sweep skipped, so a manual trigger
|
||||
* can report which runs to follow or why nothing ran.
|
||||
*/
|
||||
public SweepOutcome run(Policy policy, List<PipelineInput> inputs, SweepKind sweep) {
|
||||
public SweepOutcome run(Policy policy, SweepKind sweep) {
|
||||
long sweepStart = System.currentTimeMillis();
|
||||
PolicySweep context = new PolicySweep(policy.id(), sweep, processedLedger);
|
||||
List<String> runIds = new ArrayList<>();
|
||||
if (inputs.isEmpty()) {
|
||||
List<String> sourceIds = policy.sourceIds();
|
||||
if (sourceIds.isEmpty()) {
|
||||
// Generator pipeline: one run with no input. Still fall through to the cleanup
|
||||
// below so rows recorded for its folder outputs are pruned like anything else,
|
||||
// instead of accumulating until the policy is deleted.
|
||||
runIds.add(startRun(policy, PolicyInputs.of(List.of()), unused -> {}));
|
||||
}
|
||||
for (PipelineInput input : inputs) {
|
||||
String sourceId = input.sourceId();
|
||||
for (String sourceId : sourceIds) {
|
||||
Source source = sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null) {
|
||||
// No veto: a deleted source's rows should age out via the cleanup below.
|
||||
@@ -101,8 +84,7 @@ public class PolicyRunner {
|
||||
}
|
||||
runIds.addAll(pullAndRun(policy, sourceId, source.toInputSpec(), context));
|
||||
}
|
||||
boolean fullPolicy = inputs.size() == policy.inputs().size();
|
||||
if (fullPolicy && context.cleanupAllowed()) {
|
||||
if (context.cleanupAllowed()) {
|
||||
processedLedger.markSeen(policy.id(), context.presentIdentities());
|
||||
int removed = processedLedger.deleteUnseen(policy.id(), sweepStart);
|
||||
if (removed > 0) {
|
||||
|
||||
+11
-41
@@ -9,7 +9,6 @@ import lombok.RequiredArgsConstructor;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.TriggerConfig;
|
||||
@@ -19,11 +18,10 @@ import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
|
||||
|
||||
/**
|
||||
* Validates a policy at save time by delegating each facet (inputs, their triggers, output) to the
|
||||
* bean that handles its type, so a misconfiguration fails fast rather than at run time. Each
|
||||
* input's {@code sourceId} must resolve to a persisted {@link Source} whose config its {@link
|
||||
* InputSource} bean accepts; its optional trigger must be a known type compatible with that source.
|
||||
* A null trigger is a manual-only input and skips trigger validation.
|
||||
* Validates a policy at save time by delegating each facet (trigger, sources, steps, output) to the
|
||||
* bean that handles its type, so a misconfiguration fails fast rather than at run time. A null
|
||||
* trigger is a manual-only policy and skips trigger validation. Each referenced {@code sourceId}
|
||||
* must resolve to a persisted {@link Source} whose config its {@link InputSource} bean accepts.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -36,31 +34,21 @@ public class PolicyValidator {
|
||||
private final SourceStore sourceStore;
|
||||
|
||||
/**
|
||||
* @throws IllegalArgumentException if the policy has more than one input or output, any facet's
|
||||
* type is unknown, a referenced source does not exist, a trigger is incompatible with its
|
||||
* input's source, or any config is invalid
|
||||
* @throws IllegalArgumentException if any facet's type is unknown, a referenced source does not
|
||||
* exist, or any config is invalid
|
||||
*/
|
||||
public void validate(Policy policy) {
|
||||
// Deliberate product cap, not a model limit: the lists stay lists so multiple
|
||||
// inputs/outputs can be supported later, but today a policy carries at most one of
|
||||
// each (zero of either remains fine - run on demand / inline output).
|
||||
if (policy.inputs().size() > 1) {
|
||||
throw new IllegalArgumentException("a policy supports at most one input");
|
||||
if (policy.trigger() != null) {
|
||||
triggerFor(policy.trigger()).validate(policy);
|
||||
}
|
||||
if (policy.outputIds().size() > 1) {
|
||||
throw new IllegalArgumentException("a policy supports at most one output");
|
||||
}
|
||||
for (PipelineInput input : policy.inputs()) {
|
||||
for (String sourceId : policy.sourceIds()) {
|
||||
Source source =
|
||||
sourceStore
|
||||
.get(input.sourceId())
|
||||
.get(sourceId)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new IllegalArgumentException(
|
||||
"unknown source: " + input.sourceId()));
|
||||
if (input.trigger() != null) {
|
||||
validateTrigger(policy, input, source);
|
||||
}
|
||||
"unknown source: " + sourceId));
|
||||
InputSpec spec = source.toInputSpec();
|
||||
inputSourceFor(spec).validate(spec);
|
||||
}
|
||||
@@ -85,24 +73,6 @@ public class PolicyValidator {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check an input's trigger is a known type whose source constraints its source satisfies (e.g.
|
||||
* folder-watch only on a folder source), then let the trigger validate its own options.
|
||||
*/
|
||||
private void validateTrigger(Policy policy, PipelineInput input, Source source) {
|
||||
PolicyTrigger trigger = triggerFor(input.trigger());
|
||||
if (!trigger.supportedSourceTypes().isEmpty()
|
||||
&& !trigger.supportedSourceTypes().contains(source.type())) {
|
||||
throw new IllegalArgumentException(
|
||||
"trigger '"
|
||||
+ trigger.type()
|
||||
+ "' is not compatible with source type '"
|
||||
+ source.type()
|
||||
+ "'");
|
||||
}
|
||||
trigger.validate(policy, input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an output spec against its sink. Must be called on a request thread (caller's
|
||||
* principal present) so an S3 output's connection is authorization-checked against the caller -
|
||||
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
package stirling.software.proprietary.policy.engine;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
import tools.jackson.databind.node.StringNode;
|
||||
|
||||
/**
|
||||
* Substitutes {@code {{steps.N...}}} references in a step's parameters against the reports the
|
||||
* earlier steps of the same run produced.
|
||||
*
|
||||
* <p>This is pipeline-scope resolution, and it lives here rather than in the tool because only the
|
||||
* executor can see the steps before the current one. Document- and run-scope placeholders ({@code
|
||||
* {{document.*}}}, {@code {{run.*}}}) are deliberately left untouched for the tool to resolve per
|
||||
* document; only {@code steps.*} is touched here, so the two passes never collide and neither has
|
||||
* to know the other's namespace.
|
||||
*
|
||||
* <p>Deliberately not a template language, matching the document-scope resolver: dotted lookup and
|
||||
* nothing else. A reference that names a step or field with no value fails the run rather than
|
||||
* sending an empty value, so a typo or a forward reference surfaces as an error.
|
||||
*/
|
||||
final class StepOutputPlaceholders {
|
||||
|
||||
private static final int MAX_TREE_DEPTH = 64;
|
||||
|
||||
// Only steps.* is matched; a document/run reference is left verbatim for the downstream tool.
|
||||
private static final Pattern STEP_REF = Pattern.compile("\\{\\{\\s*(steps\\.[\\w.]+?)\\s*}}");
|
||||
|
||||
private StepOutputPlaceholders() {}
|
||||
|
||||
/** Whether the text references an earlier step at all, so callers can skip resolving. */
|
||||
static boolean references(String text) {
|
||||
return text != null && STEP_REF.matcher(text).find();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param template text that may contain {@code {{steps...}}} references; null passes through
|
||||
* @param context the run context whose {@code steps} object is keyed by 1-based step number
|
||||
* @throws IllegalArgumentException if a reference names a step or field the context does not
|
||||
* hold, or names a non-scalar, so it cannot be inlined into a string parameter
|
||||
*/
|
||||
static String resolve(String template, JsonNode context) {
|
||||
if (template == null || template.isEmpty()) {
|
||||
return template;
|
||||
}
|
||||
Matcher matcher = STEP_REF.matcher(template);
|
||||
StringBuilder out = new StringBuilder();
|
||||
while (matcher.find()) {
|
||||
String path = matcher.group(1);
|
||||
JsonNode value = lookup(context, path);
|
||||
if (value == null || value.isMissingNode() || !value.isValueNode()) {
|
||||
throw new IllegalArgumentException(
|
||||
"step reference '{{"
|
||||
+ path
|
||||
+ "}}' resolved to nothing; an earlier step must have produced that"
|
||||
+ " value (a later step cannot be referenced, and only scalar"
|
||||
+ " values can be inlined)");
|
||||
}
|
||||
matcher.appendReplacement(out, Matcher.quoteReplacement(value.asString()));
|
||||
}
|
||||
matcher.appendTail(out);
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve every string inside a parsed JSON tree, leaving structure and non-strings alone.
|
||||
* Mirrors the document-scope resolver: a substituted value lands in a text node and is escaped
|
||||
* on serialise, so a response can never inject fields into the JSON the operator wrote.
|
||||
*/
|
||||
static JsonNode resolveTree(JsonNode node, JsonNode context) {
|
||||
return resolveTree(node, context, 0);
|
||||
}
|
||||
|
||||
private static JsonNode resolveTree(JsonNode node, JsonNode context, int depth) {
|
||||
// Deeply nested JSON is left untouched rather than recursed into, so a
|
||||
// pathological template cannot overflow the stack.
|
||||
if (depth > MAX_TREE_DEPTH) {
|
||||
return node;
|
||||
}
|
||||
if (node instanceof ObjectNode object) {
|
||||
for (String name : new java.util.ArrayList<>(object.propertyNames())) {
|
||||
object.set(name, resolveTree(object.get(name), context, depth + 1));
|
||||
}
|
||||
return object;
|
||||
}
|
||||
if (node instanceof ArrayNode array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
array.set(i, resolveTree(array.get(i), context, depth + 1));
|
||||
}
|
||||
return array;
|
||||
}
|
||||
if (node != null && node.isString()) {
|
||||
return StringNode.valueOf(resolve(node.asString(), context));
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
private static JsonNode lookup(JsonNode context, String path) {
|
||||
JsonNode node = context;
|
||||
for (String segment : path.split("\\.")) {
|
||||
if (node == null || !node.isObject()) {
|
||||
return null;
|
||||
}
|
||||
node = node.get(segment);
|
||||
}
|
||||
return node;
|
||||
}
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package stirling.software.proprietary.policy.model;
|
||||
|
||||
/**
|
||||
* One input of a policy: a reference to a persisted {@code Source} paired with the {@link
|
||||
* TriggerConfig} that decides when <em>this</em> source is pulled. The trigger lives on the
|
||||
* binding, not on the source (so one connection can feed many policies on different schedules) and
|
||||
* not on the policy (so a folder input can be watched while an S3 input on the same policy polls).
|
||||
* A {@code null} trigger means this input is pulled only when the policy is run on demand.
|
||||
*/
|
||||
public record PipelineInput(String sourceId, TriggerConfig trigger) {
|
||||
|
||||
/** An input with no automatic trigger: pulled only on a manual run. */
|
||||
public static PipelineInput manual(String sourceId) {
|
||||
return new PipelineInput(sourceId, null);
|
||||
}
|
||||
}
|
||||
+30
-31
@@ -3,31 +3,29 @@ package stirling.software.proprietary.policy.model;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A stored automation: ordered tool steps, input bindings, and output destinations.
|
||||
* A stored automation: ordered tool steps, input sources, and output destinations.
|
||||
*
|
||||
* <p>Always runnable on demand. Each {@link PipelineInput} references a persisted {@code Source}
|
||||
* connection (resolved live at run time) and carries its own optional {@link TriggerConfig}: the
|
||||
* trigger decides when that source is pulled, so one input can be watched while another polls, and
|
||||
* a {@code null} trigger makes that input manual-only. An input with no trigger, or a policy with
|
||||
* no triggered inputs, still runs when the policy is run on demand; a manual run pulls every input.
|
||||
*
|
||||
* <p>{@code outputIds} reference the {@code Source} locations (resolved live) a run's files are
|
||||
* delivered to - a run is delivered to every one; when empty the inline {@link #output} is used
|
||||
* (results returned to the caller), the case for editor and one-off policies.
|
||||
* <p>Always runnable on demand. An optional {@link TriggerConfig} fires it automatically; a {@code
|
||||
* null} trigger means manual-only. Trigger decides when; {@code sourceIds} reference the persisted
|
||||
* {@code Source} locations (resolved live at run time) files come from; a run pulls from every
|
||||
* referenced source. {@code outputIds} reference the {@code Source} locations (resolved live) a
|
||||
* run's files are delivered to - a run is delivered to every one; when empty the inline {@link
|
||||
* #output} is used (results returned to the caller), the case for editor and one-off policies.
|
||||
*/
|
||||
public record Policy(
|
||||
String id,
|
||||
String name,
|
||||
String owner,
|
||||
boolean enabled,
|
||||
List<PipelineInput> inputs,
|
||||
TriggerConfig trigger,
|
||||
List<String> sourceIds,
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output,
|
||||
List<String> outputIds,
|
||||
Long teamId) {
|
||||
|
||||
public Policy {
|
||||
inputs = inputs == null ? List.of() : List.copyOf(inputs);
|
||||
sourceIds = sourceIds == null ? List.of() : List.copyOf(sourceIds);
|
||||
steps = steps == null ? List.of() : steps;
|
||||
output = output == null ? OutputSpec.inline() : output;
|
||||
outputIds = outputIds == null ? List.of() : List.copyOf(outputIds);
|
||||
@@ -43,11 +41,12 @@ public record Policy(
|
||||
String name,
|
||||
String owner,
|
||||
boolean enabled,
|
||||
List<PipelineInput> inputs,
|
||||
TriggerConfig trigger,
|
||||
List<String> sourceIds,
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output,
|
||||
Long teamId) {
|
||||
this(id, name, owner, enabled, inputs, steps, output, List.of(), teamId);
|
||||
this(id, name, owner, enabled, trigger, sourceIds, steps, output, List.of(), teamId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -59,35 +58,35 @@ public record Policy(
|
||||
String name,
|
||||
String owner,
|
||||
boolean enabled,
|
||||
List<PipelineInput> inputs,
|
||||
TriggerConfig trigger,
|
||||
List<String> sourceIds,
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output) {
|
||||
this(id, name, owner, enabled, inputs, steps, output, List.of(), null);
|
||||
this(id, name, owner, enabled, trigger, sourceIds, steps, output, List.of(), null);
|
||||
}
|
||||
|
||||
/** The source ids this policy pulls from, in input order; a derived view for reads. */
|
||||
public List<String> sourceIds() {
|
||||
return inputs.stream().map(PipelineInput::sourceId).toList();
|
||||
}
|
||||
|
||||
/** The distinct trigger types configured across this policy's inputs (manual inputs aside). */
|
||||
public List<String> triggerTypes() {
|
||||
return inputs.stream()
|
||||
.map(PipelineInput::trigger)
|
||||
.filter(trigger -> trigger != null)
|
||||
.map(TriggerConfig::type)
|
||||
.distinct()
|
||||
.toList();
|
||||
/** A policy with no configured sources (a generator, or files supplied directly to a run). */
|
||||
public Policy(
|
||||
String id,
|
||||
String name,
|
||||
String owner,
|
||||
boolean enabled,
|
||||
TriggerConfig trigger,
|
||||
List<PipelineStep> steps,
|
||||
OutputSpec output) {
|
||||
this(id, name, owner, enabled, trigger, List.of(), steps, output, List.of(), null);
|
||||
}
|
||||
|
||||
/** A copy with the inline output replaced (e.g. resolved for the engine, or migrated). */
|
||||
public Policy withOutput(OutputSpec resolved) {
|
||||
return new Policy(id, name, owner, enabled, inputs, steps, resolved, outputIds, teamId);
|
||||
return new Policy(
|
||||
id, name, owner, enabled, trigger, sourceIds, steps, resolved, outputIds, teamId);
|
||||
}
|
||||
|
||||
/** A copy referencing the given saved output destinations. */
|
||||
public Policy withOutputIds(List<String> newOutputIds) {
|
||||
return new Policy(id, name, owner, enabled, inputs, steps, output, newOutputIds, teamId);
|
||||
return new Policy(
|
||||
id, name, owner, enabled, trigger, sourceIds, steps, output, newOutputIds, teamId);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package stirling.software.proprietary.policy.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A policy paired with one of its {@link PipelineInput}s: the unit a background trigger fires. A
|
||||
* policy with two triggered inputs yields two bindings, so each fires independently on its own
|
||||
* trigger and pulls only its own source.
|
||||
*/
|
||||
public record PolicyBinding(Policy policy, PipelineInput input) {
|
||||
|
||||
/**
|
||||
* The bindings across these policies whose input carries a trigger of the given type. Shared by
|
||||
* the {@code PolicyStore} implementations so every backend derives a trigger's bindings the
|
||||
* same way. Callers pass the policies a background trigger should consider (i.e. the enabled
|
||||
* ones).
|
||||
*/
|
||||
public static List<PolicyBinding> matching(List<Policy> policies, String triggerType) {
|
||||
return policies.stream()
|
||||
.flatMap(
|
||||
policy ->
|
||||
policy.inputs().stream()
|
||||
.filter(
|
||||
input ->
|
||||
input.trigger() != null
|
||||
&& triggerType.equals(
|
||||
input.trigger().type()))
|
||||
.map(input -> new PolicyBinding(policy, input)))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
+5
-8
@@ -14,6 +14,7 @@ import stirling.software.proprietary.policy.config.PolicyAccessGuard;
|
||||
import stirling.software.proprietary.policy.model.OutputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineStep;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.TriggerConfig;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceAccessGuard;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
@@ -72,7 +73,7 @@ public class PolicyOverviewService {
|
||||
policy.name(),
|
||||
policy.enabled(),
|
||||
policy.enabled() ? "active" : "paused",
|
||||
triggerSummary(policy),
|
||||
triggerSummary(policy.trigger()),
|
||||
sources,
|
||||
steps,
|
||||
outputSummary(policy, sourceNames),
|
||||
@@ -94,13 +95,9 @@ public class PolicyOverviewService {
|
||||
return outputSummary(policy.output());
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarise a policy's triggers for the overview row: "manual" when no input is triggered,
|
||||
* otherwise the distinct trigger types across its inputs (e.g. "folder-watch, schedule").
|
||||
*/
|
||||
private static String triggerSummary(Policy policy) {
|
||||
List<String> types = policy.triggerTypes();
|
||||
return types.isEmpty() ? "manual" : String.join(", ", types);
|
||||
/** A null trigger is a manual-only policy; otherwise the trigger's type keys the summary. */
|
||||
private static String triggerSummary(TriggerConfig trigger) {
|
||||
return trigger == null ? "manual" : trigger.type();
|
||||
}
|
||||
|
||||
private static String outputSummary(OutputSpec output) {
|
||||
|
||||
+1
@@ -87,6 +87,7 @@ public class DefaultClassificationPolicySeeder {
|
||||
POLICY_NAME,
|
||||
"system",
|
||||
true,
|
||||
null,
|
||||
List.of(),
|
||||
List.of(new PipelineStep(CLASSIFY_ENDPOINT, Map.of())),
|
||||
new OutputSpec("inline", options),
|
||||
|
||||
+8
-5
@@ -9,7 +9,6 @@ import java.util.UUID;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
|
||||
/**
|
||||
* In-memory {@link PolicyStore} for tests and any future no-database mode. {@link JpaPolicyStore}
|
||||
@@ -33,7 +32,8 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
policy.name(),
|
||||
policy.owner(),
|
||||
policy.enabled(),
|
||||
policy.inputs(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
@@ -77,9 +77,12 @@ public class InProcessPolicyStore implements PolicyStore {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PolicyBinding> findBindingsByTriggerType(String triggerType) {
|
||||
List<Policy> enabled = policies.values().stream().filter(Policy::enabled).toList();
|
||||
return PolicyBinding.matching(enabled, triggerType);
|
||||
public List<Policy> findByTriggerType(String triggerType) {
|
||||
return policies.values().stream()
|
||||
.filter(Policy::enabled)
|
||||
.filter(policy -> policy.trigger() != null)
|
||||
.filter(policy -> triggerType.equals(policy.trigger().type()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+9
-45
@@ -12,12 +12,8 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ArrayNode;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Durable {@link PolicyStore} backed by JPA; the runtime store. Policies are persisted as JSON via
|
||||
@@ -44,7 +40,8 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
policy.name(),
|
||||
policy.owner(),
|
||||
policy.enabled(),
|
||||
policy.inputs(),
|
||||
policy.trigger(),
|
||||
policy.sourceIds(),
|
||||
policy.steps(),
|
||||
policy.output(),
|
||||
policy.outputIds(),
|
||||
@@ -55,6 +52,7 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
entity.setName(stored.name());
|
||||
entity.setOwner(stored.owner());
|
||||
entity.setEnabled(stored.enabled());
|
||||
entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type());
|
||||
entity.setTeamId(stored.teamId());
|
||||
// Preserve an existing policy's run-order position; append a new one to the end of its
|
||||
// team's queue (max + 1), so setting up a policy adds it last by default.
|
||||
@@ -121,13 +119,11 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PolicyBinding> findBindingsByTriggerType(String triggerType) {
|
||||
List<Policy> enabled =
|
||||
repository.findByEnabledTrue().stream()
|
||||
.map(this::toPolicy)
|
||||
.flatMap(Optional::stream)
|
||||
.toList();
|
||||
return PolicyBinding.matching(enabled, triggerType);
|
||||
public List<Policy> findByTriggerType(String triggerType) {
|
||||
return repository.findByTriggerTypeAndEnabledTrue(triggerType).stream()
|
||||
.map(this::toPolicy)
|
||||
.flatMap(Optional::stream)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -143,8 +139,7 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
// One unreadable row must never abort a bulk read or crash startup.
|
||||
private Optional<Policy> toPolicy(PolicyEntity entity) {
|
||||
try {
|
||||
JsonNode node = upgradeLegacyShape(objectMapper.readTree(entity.getPolicyJson()));
|
||||
return Optional.of(objectMapper.treeToValue(node, Policy.class));
|
||||
return Optional.of(objectMapper.readValue(entity.getPolicyJson(), Policy.class));
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Skipping unreadable policy id={} name={}: stored JSON could not be parsed"
|
||||
@@ -155,35 +150,4 @@ public class JpaPolicyStore implements PolicyStore {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate a policy JSON blob written before triggers moved onto inputs. The old shape carried a
|
||||
* single policy-level {@code trigger} and a {@code sourceIds} list; pair each source with that
|
||||
* trigger so an upgraded policy keeps firing. A trigger incompatible with a source
|
||||
* (folder-watch on an S3 source) is simply inert at run time, matching the old behaviour where
|
||||
* such a source was never watched. New-shape blobs (already carrying {@code inputs}) are
|
||||
* returned untouched.
|
||||
*/
|
||||
private JsonNode upgradeLegacyShape(JsonNode root) {
|
||||
if (!(root instanceof ObjectNode obj) || obj.has("inputs")) {
|
||||
return root;
|
||||
}
|
||||
JsonNode trigger = obj.get("trigger");
|
||||
JsonNode sourceIds = obj.get("sourceIds");
|
||||
ArrayNode inputs = objectMapper.createArrayNode();
|
||||
if (sourceIds != null && sourceIds.isArray()) {
|
||||
for (JsonNode sourceId : sourceIds) {
|
||||
ObjectNode input = objectMapper.createObjectNode();
|
||||
input.set("sourceId", sourceId);
|
||||
if (trigger != null && !trigger.isNull()) {
|
||||
input.set("trigger", trigger);
|
||||
}
|
||||
inputs.add(input);
|
||||
}
|
||||
}
|
||||
obj.set("inputs", inputs);
|
||||
obj.remove("trigger");
|
||||
obj.remove("sourceIds");
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
+7
-4
@@ -17,10 +17,10 @@ import stirling.software.proprietary.integration.crypto.LegacyDecryptStringConve
|
||||
/**
|
||||
* JPA row for a {@link stirling.software.proprietary.policy.model.Policy}. The whole policy lives
|
||||
* as JSON in {@code policyJson} (authoritative on read); the scalar columns are denormalized copies
|
||||
* for querying, notably {@code enabled} so background triggers can scan the active policies, and
|
||||
* {@code teamId} so the caller's team can be loaded without scanning every team's rows. {@code
|
||||
* owner} and {@code teamId} are plain values, not foreign keys, to stay decoupled from the security
|
||||
* entities.
|
||||
* for querying, notably {@code triggerType} + {@code enabled} so background triggers can fetch
|
||||
* their policies, and {@code teamId} so the caller's team can be loaded without scanning every
|
||||
* team's rows. {@code owner} and {@code teamId} are plain values, not foreign keys, to stay
|
||||
* decoupled from the security entities.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "policies")
|
||||
@@ -44,6 +44,9 @@ public class PolicyEntity implements Serializable {
|
||||
@Column(name = "enabled")
|
||||
private boolean enabled;
|
||||
|
||||
@Column(name = "trigger_type")
|
||||
private String triggerType;
|
||||
|
||||
@Column(name = "team_id")
|
||||
private Long teamId;
|
||||
|
||||
|
||||
+2
-6
@@ -13,12 +13,8 @@ import jakarta.persistence.LockModeType;
|
||||
@Repository
|
||||
public interface PolicyRepository extends JpaRepository<PolicyEntity, String> {
|
||||
|
||||
/**
|
||||
* Enabled policies, for background triggers to scan for inputs of their trigger type. Which
|
||||
* inputs (and their trigger types) a policy carries lives in the JSON blob, so the type filter
|
||||
* is applied after parsing rather than in SQL.
|
||||
*/
|
||||
List<PolicyEntity> findByEnabledTrue();
|
||||
/** Enabled policies of a given trigger type, for background triggers to activate. */
|
||||
List<PolicyEntity> findByTriggerTypeAndEnabledTrue(String triggerType);
|
||||
|
||||
/**
|
||||
* Policies belonging to a team, in run order (ascending {@code sortOrder}; a null order sorts
|
||||
|
||||
+2
-6
@@ -4,7 +4,6 @@ import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
|
||||
/** Stores {@link Policy} definitions. */
|
||||
public interface PolicyStore {
|
||||
@@ -19,11 +18,8 @@ public interface PolicyStore {
|
||||
/** Policies owned by the given team, loaded scoped rather than fetched globally. */
|
||||
List<Policy> findByTeam(Long teamId);
|
||||
|
||||
/**
|
||||
* Enabled inputs with the given trigger type, as {@code (policy, input)} bindings, so a
|
||||
* background trigger fires each input independently and pulls only its own source.
|
||||
*/
|
||||
List<PolicyBinding> findBindingsByTriggerType(String triggerType);
|
||||
/** Enabled policies with the given trigger type, for background triggers. */
|
||||
List<Policy> findByTriggerType(String triggerType);
|
||||
|
||||
/**
|
||||
* Set the team's run order from {@code orderedIds} (position → sortOrder). Only policies that
|
||||
|
||||
+33
-44
@@ -31,9 +31,7 @@ import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.input.InputSource;
|
||||
import stirling.software.proprietary.policy.model.InputSpec;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
@@ -87,10 +85,10 @@ public class FolderWatchTrigger implements PolicyTrigger {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Policy policy, PipelineInput input) {
|
||||
if (watchDirsOf(input).isEmpty()) {
|
||||
public void validate(Policy policy) {
|
||||
if (watchDirsOf(policy).isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"folder-watch trigger requires a watchable (folder) input source");
|
||||
"folder-watch trigger requires at least one watchable (folder) input source");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,30 +185,24 @@ public class FolderWatchTrigger implements PolicyTrigger {
|
||||
return changed;
|
||||
}
|
||||
|
||||
/** Fire every folder-watch input that draws from one of the changed directories. */
|
||||
/** Run every folder-watch policy that draws from one of the changed directories. */
|
||||
void runForChangedDirs(Set<Path> changedDirs) {
|
||||
if (changedDirs.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
List<Path> dirs;
|
||||
try {
|
||||
dirs = watchDirsOf(binding.input());
|
||||
dirs = watchDirsOf(policy);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Folder-watch input {}/{} is misconfigured: {}",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId(),
|
||||
e.getMessage());
|
||||
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
|
||||
continue;
|
||||
}
|
||||
if (dirs.stream().anyMatch(changedDirs::contains)) {
|
||||
log.debug(
|
||||
"Folder-watch input {}/{} saw activity",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId());
|
||||
log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name());
|
||||
// Light: the periodic reconcile does the full sweep.
|
||||
policyRunner.runInput(binding.policy(), binding.input(), SweepKind.LIGHT);
|
||||
policyRunner.run(policy, SweepKind.LIGHT);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -224,16 +216,15 @@ public class FolderWatchTrigger implements PolicyTrigger {
|
||||
}
|
||||
}
|
||||
|
||||
/** Reconcile safety net: run every folder-watch input regardless of watch events. */
|
||||
/** Reconcile safety net: run every folder-watch policy regardless of watch events. */
|
||||
void runAll() {
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
try {
|
||||
policyRunner.runInput(binding.policy(), binding.input(), SweepKind.FULL);
|
||||
policyRunner.run(policy);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Folder-watch reconcile run failed for input {}/{}: {}",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId(),
|
||||
"Folder-watch reconcile run failed for policy {}: {}",
|
||||
policy.id(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
@@ -278,43 +269,41 @@ public class FolderWatchTrigger implements PolicyTrigger {
|
||||
return Set.copyOf(keysByDir.keySet());
|
||||
}
|
||||
|
||||
/** Every existing directory any current folder-watch input wants watched. */
|
||||
/** Every existing directory any current folder-watch policy wants watched. */
|
||||
private Set<Path> desiredDirs() {
|
||||
Set<Path> dirs = new HashSet<>();
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
try {
|
||||
for (Path dir : watchDirsOf(binding.input())) {
|
||||
for (Path dir : watchDirsOf(policy)) {
|
||||
if (Files.isDirectory(dir)) {
|
||||
dirs.add(dir);
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Folder-watch input {}/{} is misconfigured: {}",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId(),
|
||||
e.getMessage());
|
||||
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
|
||||
}
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
// Absolute + normalised so registration keys and event-time matching compare regardless of how
|
||||
// the path was configured. Empty for a non-folder or missing source (that input is never
|
||||
// watched), so a folder-watch trigger paired with an S3 input is simply inert.
|
||||
private List<Path> watchDirsOf(PipelineInput input) {
|
||||
// the path was configured.
|
||||
private List<Path> watchDirsOf(Policy policy) {
|
||||
List<Path> dirs = new ArrayList<>();
|
||||
Source source = sourceStore.get(input.sourceId()).orElse(null);
|
||||
if (source == null) {
|
||||
return dirs;
|
||||
}
|
||||
InputSpec spec = source.toInputSpec();
|
||||
InputSource inputSource = sourceFor(spec);
|
||||
if (inputSource == null) {
|
||||
return dirs;
|
||||
}
|
||||
for (Path dir : inputSource.watchTargets(spec)) {
|
||||
dirs.add(dir.toAbsolutePath().normalize());
|
||||
for (String sourceId : policy.sourceIds()) {
|
||||
Source source = sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null) {
|
||||
continue;
|
||||
}
|
||||
InputSpec spec = source.toInputSpec();
|
||||
InputSource inputSource = sourceFor(spec);
|
||||
if (inputSource == null) {
|
||||
continue;
|
||||
}
|
||||
for (Path dir : inputSource.watchTargets(spec)) {
|
||||
dirs.add(dir.toAbsolutePath().normalize());
|
||||
}
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
+5
-8
@@ -2,13 +2,11 @@ package stirling.software.proprietary.policy.trigger;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
|
||||
/**
|
||||
* Decides <em>when</em> a policy input runs. On firing it hands the binding to {@code
|
||||
* PolicyRunner}, which pulls only that input's source; it never resolves sources itself. New
|
||||
* trigger kinds are just new beans of this type.
|
||||
* Decides <em>when</em> a policy runs. On firing it hands the policy to {@code PolicyRunner}; it
|
||||
* never resolves sources itself. New trigger kinds are just new beans of this type.
|
||||
*/
|
||||
public interface PolicyTrigger {
|
||||
|
||||
@@ -34,11 +32,10 @@ public interface PolicyTrigger {
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate one input's use of this trigger at save time so misconfiguration fails fast, not at
|
||||
* fire time. Receives the owning {@link Policy} and the specific {@link PipelineInput} so a
|
||||
* trigger that depends on the input's source (folder-watch) can check it.
|
||||
* Validate at save time so misconfiguration fails fast, not at fire time. Receives the whole
|
||||
* {@link Policy} so triggers that depend on the policy's sources (folder-watch) can check that.
|
||||
*/
|
||||
default void validate(Policy policy, PipelineInput input) {}
|
||||
default void validate(Policy policy) {}
|
||||
|
||||
default void start() {}
|
||||
|
||||
|
||||
+13
-31
@@ -17,18 +17,14 @@ import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
import stirling.software.proprietary.policy.model.Schedule;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Fires policy inputs on a {@link Schedule}: a fixed-interval sweep pulls each due "schedule"
|
||||
* input, independently of the policy's other inputs.
|
||||
* Fires policies on a {@link Schedule}: a fixed-interval sweep runs each due "schedule" policy.
|
||||
*
|
||||
* <p>Last-fire times are in memory, so this assumes a single node and resets on restart.
|
||||
*/
|
||||
@@ -44,20 +40,17 @@ public class ScheduleTrigger implements PolicyTrigger {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
private final Map<BindingKey, Instant> lastFiredByBinding = new ConcurrentHashMap<>();
|
||||
private final Map<String, Instant> lastFiredByPolicy = new ConcurrentHashMap<>();
|
||||
private volatile ScheduledExecutorService scheduler;
|
||||
|
||||
/** Identifies a schedule binding: one input (by source) of one policy. */
|
||||
private record BindingKey(String policyId, String sourceId) {}
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Policy policy, PipelineInput input) {
|
||||
ScheduleConfig.from(objectMapper, input.trigger().options());
|
||||
public void validate(Policy policy) {
|
||||
ScheduleConfig.from(objectMapper, policy.trigger().options());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -90,26 +83,19 @@ public class ScheduleTrigger implements PolicyTrigger {
|
||||
}
|
||||
}
|
||||
|
||||
/** Fire every scheduled input that is due as of {@code now}. Package-visible for testing. */
|
||||
/** Fire every scheduled policy that is due as of {@code now}. Package-visible for testing. */
|
||||
void sweep(Instant now) {
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
Policy policy = binding.policy();
|
||||
PipelineInput input = binding.input();
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
ScheduleConfig config;
|
||||
try {
|
||||
config = ScheduleConfig.from(objectMapper, input.trigger().options());
|
||||
config = ScheduleConfig.from(objectMapper, policy.trigger().options());
|
||||
} catch (IllegalArgumentException e) {
|
||||
log.warn(
|
||||
"Scheduled input {}/{} is misconfigured: {}",
|
||||
policy.id(),
|
||||
input.sourceId(),
|
||||
e.getMessage());
|
||||
log.warn("Scheduled policy {} is misconfigured: {}", policy.id(), e.getMessage());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Baseline a newly-seen binding to now so it does not fire immediately.
|
||||
BindingKey key = new BindingKey(policy.id(), input.sourceId());
|
||||
Instant last = lastFiredByBinding.computeIfAbsent(key, id -> now);
|
||||
// Baseline a newly-seen policy to now so it does not fire immediately.
|
||||
Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now);
|
||||
ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone()));
|
||||
if (next.toInstant().isAfter(now)) {
|
||||
continue;
|
||||
@@ -119,13 +105,9 @@ public class ScheduleTrigger implements PolicyTrigger {
|
||||
next = later;
|
||||
later = config.schedule().nextAfter(later);
|
||||
}
|
||||
lastFiredByBinding.put(key, next.toInstant());
|
||||
log.info(
|
||||
"Scheduled input {}/{} ({}) is due",
|
||||
policy.id(),
|
||||
input.sourceId(),
|
||||
policy.name());
|
||||
policyRunner.runInput(policy, input, SweepKind.FULL);
|
||||
lastFiredByPolicy.put(policy.id(), next.toInstant());
|
||||
log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name());
|
||||
policyRunner.run(policy);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+29
-35
@@ -13,9 +13,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.proprietary.policy.engine.PolicyRunner;
|
||||
import stirling.software.proprietary.policy.engine.SweepKind;
|
||||
import stirling.software.proprietary.policy.model.PipelineInput;
|
||||
import stirling.software.proprietary.policy.model.Policy;
|
||||
import stirling.software.proprietary.policy.model.PolicyBinding;
|
||||
import stirling.software.proprietary.policy.source.Source;
|
||||
import stirling.software.proprietary.policy.source.SourceStore;
|
||||
import stirling.software.proprietary.policy.store.PolicyStore;
|
||||
@@ -52,14 +50,15 @@ public class WebhookTrigger implements PolicyTrigger {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Policy policy, PipelineInput input) {
|
||||
boolean isWebhookSource =
|
||||
sourceStore
|
||||
.get(input.sourceId())
|
||||
.filter(source -> WEBHOOK_SOURCE_TYPE.equals(source.type()))
|
||||
.isPresent();
|
||||
if (!isWebhookSource) {
|
||||
throw new IllegalArgumentException("webhook trigger requires a webhook input source");
|
||||
public void validate(Policy policy) {
|
||||
boolean hasWebhookSource =
|
||||
policy.sourceIds().stream()
|
||||
.map(sourceStore::get)
|
||||
.flatMap(java.util.Optional::stream)
|
||||
.anyMatch(source -> WEBHOOK_SOURCE_TYPE.equals(source.type()));
|
||||
if (!hasWebhookSource) {
|
||||
throw new IllegalArgumentException(
|
||||
"webhook trigger requires at least one webhook input source");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,38 +83,29 @@ public class WebhookTrigger implements PolicyTrigger {
|
||||
}
|
||||
}
|
||||
|
||||
/** Fire every webhook input fed by this webhook, pulling only that input's source. */
|
||||
public void fireForWebhook(String webhookId) {
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
if (!referencesWebhook(binding.input(), webhookId)) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
if (!referencesWebhook(policy, webhookId)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
log.debug(
|
||||
"Webhook input {}/{} saw a delivery",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId());
|
||||
policyRunner.runInput(binding.policy(), binding.input(), SweepKind.LIGHT);
|
||||
log.debug("Webhook policy {} ({}) saw a delivery", policy.id(), policy.name());
|
||||
policyRunner.run(policy, SweepKind.LIGHT);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Webhook run failed for input {}/{}: {}",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId(),
|
||||
e.getMessage());
|
||||
log.warn("Webhook run failed for policy {}: {}", policy.id(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void safeReconcile() {
|
||||
try {
|
||||
for (PolicyBinding binding : policyStore.findBindingsByTriggerType(TYPE)) {
|
||||
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
|
||||
try {
|
||||
policyRunner.runInput(binding.policy(), binding.input(), SweepKind.FULL);
|
||||
policyRunner.run(policy);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn(
|
||||
"Webhook reconcile run failed for input {}/{}: {}",
|
||||
binding.policy().id(),
|
||||
binding.input().sourceId(),
|
||||
"Webhook reconcile run failed for policy {}: {}",
|
||||
policy.id(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
@@ -124,13 +114,17 @@ public class WebhookTrigger implements PolicyTrigger {
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this input draws from the webhook source the delivery arrived on. */
|
||||
private boolean referencesWebhook(PipelineInput input, String webhookId) {
|
||||
Source source = sourceStore.get(input.sourceId()).orElse(null);
|
||||
if (source == null || !WEBHOOK_SOURCE_TYPE.equals(source.type())) {
|
||||
return false;
|
||||
private boolean referencesWebhook(Policy policy, String webhookId) {
|
||||
for (String sourceId : policy.sourceIds()) {
|
||||
Source source = sourceStore.get(sourceId).orElse(null);
|
||||
if (source == null || !WEBHOOK_SOURCE_TYPE.equals(source.type())) {
|
||||
continue;
|
||||
}
|
||||
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
|
||||
if (configured != null && configured.toString().equals(webhookId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Object configured = source.options().get(WebhookConfig.WEBHOOK_ID_OPTION);
|
||||
return configured != null && configured.toString().equals(webhookId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package stirling.software.proprietary.security.filter;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.slf4j.MDC;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.filter.OncePerRequestFilter;
|
||||
|
||||
import jakarta.servlet.FilterChain;
|
||||
import jakarta.servlet.ServletException;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* Stamps the authenticated principal into MDC for every request. Async job workers resolve the
|
||||
* caller via UserService's MDC fallback; without this the fallback only worked when the audit
|
||||
* aspect (a pro feature) happened to populate it.
|
||||
*/
|
||||
@Component
|
||||
@Order(Ordered.LOWEST_PRECEDENCE)
|
||||
public class PrincipalMdcFilter extends OncePerRequestFilter {
|
||||
|
||||
static final String MDC_KEY = "auditPrincipal";
|
||||
|
||||
@Override
|
||||
protected void doFilterInternal(
|
||||
HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||
throws ServletException, IOException {
|
||||
String previous = MDC.get(MDC_KEY);
|
||||
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
|
||||
boolean stamped = false;
|
||||
if (previous == null
|
||||
&& authentication != null
|
||||
&& authentication.isAuthenticated()
|
||||
&& !"anonymousUser".equals(authentication.getPrincipal())) {
|
||||
MDC.put(MDC_KEY, authentication.getName());
|
||||
stamped = true;
|
||||
}
|
||||
try {
|
||||
filterChain.doFilter(request, response);
|
||||
} finally {
|
||||
if (stamped) {
|
||||
MDC.remove(MDC_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+7
@@ -254,6 +254,13 @@ public class AiEngineClient {
|
||||
|
||||
private void checkResponseStatus(HttpResponse<String> response) {
|
||||
int status = response.statusCode();
|
||||
// 501 = capability not implemented (e.g. docparse addon missing); keep the status and
|
||||
// body so callers can surface the machine-readable addonRequired detail.
|
||||
if (status == 501) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.NOT_IMPLEMENTED,
|
||||
"AI engine capability not implemented: " + response.body());
|
||||
}
|
||||
if (status >= 500) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.BAD_GATEWAY, "AI engine returned error: " + status);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user