mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Anthony Stirling
|
0d70133e64 | ||
|
Anthony Stirling
|
07de12746c | ||
|
Anthony Stirling
|
469e7c499c | ||
|
Anthony Stirling
|
40ccbc15cc | ||
|
Anthony Stirling
|
62944d7423 | ||
|
Anthony Stirling
|
7ce98d29f2 | ||
|
Anthony Stirling
|
65c01e8078 | ||
|
Anthony Stirling
|
4700542c75 |
@@ -35,7 +35,7 @@ jobs:
|
||||
pr_ref: ${{ steps.resolve.outputs.ref }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -184,7 +184,7 @@ jobs:
|
||||
fetch-depth: 0 # Fetch full history for commit hash detection
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
@@ -193,7 +193,7 @@ jobs:
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -320,7 +320,7 @@ jobs:
|
||||
# under the exact same access rules as the app preview.
|
||||
- name: Detect Storybook changes
|
||||
id: sb-changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
@@ -456,7 +456,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
enable_prototypes: ${{ steps.check-prototypes-flag.outputs.enable_prototypes }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -233,10 +233,10 @@ jobs:
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -474,7 +474,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
is_fork: ${{ steps.decide.outputs.is_fork }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
- name: AI PR Title Analysis
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: ai-title-analysis
|
||||
uses: actions/ai-inference@a7805884c80886efc241e94a5351df715968a0ad # v2.1.1
|
||||
uses: actions/ai-inference@17ff458cb182449bbb2e43701fcd98f6af8f6570 # v2.1.0
|
||||
with:
|
||||
model: openai/gpt-4o
|
||||
system-prompt-file: ".github/config/system-prompt.txt"
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
flavor: [core, proprietary, saas]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
@@ -295,70 +295,3 @@ jobs:
|
||||
name: playwright-report-enterprise-${{ github.run_id }}
|
||||
path: frontend/playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
# Multi-node regression: builds + seeds the clustered stack (testing/compose/docker-compose-multinode.yml)
|
||||
# and runs behave features/multinode. Licence-gated, so it runs after the Playwright job (not in parallel).
|
||||
multinode-e2e:
|
||||
needs: [pick, playwright-e2e-enterprise]
|
||||
# Nightly cron + manual dispatch only (heavy build), fork-gated for the licence secret.
|
||||
if: >-
|
||||
always() && needs.pick.outputs.is_fork != 'true'
|
||||
&& (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
|
||||
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || format('depot-ubuntu-24.04-{0}', inputs.depot_cores || '8') }}
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
PREMIUM_KEY: ${{ secrets.PREMIUM_KEY_ENTERPRISE }}
|
||||
PREMIUM_ENABLED: "true"
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
MN_COMPOSE: docker-compose-multinode.yml
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
cache-dependency-path: ./testing/cucumber/requirements.txt
|
||||
- name: Install behave test deps
|
||||
run: |
|
||||
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
|
||||
- name: Build the multi-node image
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" build
|
||||
- name: Bring up the cluster and wait for both nodes healthy
|
||||
working-directory: testing/compose
|
||||
run: |
|
||||
docker compose -f "$MN_COMPOSE" up -d
|
||||
for i in $(seq 1 90); do
|
||||
h1=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-1 2>/dev/null || echo starting)
|
||||
h2=$(docker inspect -f '{{.State.Health.Status}}' multinode-stirling-2 2>/dev/null || echo starting)
|
||||
if [ "$h1" = healthy ] && [ "$h2" = healthy ]; then echo "both nodes healthy"; exit 0; fi
|
||||
sleep 5
|
||||
done
|
||||
echo "::error::nodes did not become healthy"
|
||||
docker compose -f "$MN_COMPOSE" logs --tail=200 stirling-1 stirling-2
|
||||
exit 1
|
||||
- name: Seed the cluster (teams, users, S3 connection, policy)
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" --profile seed run --rm seed
|
||||
- name: Run multi-node regression (implemented guarantees)
|
||||
working-directory: testing/cucumber
|
||||
# -e overrides behave.ini's exclusion of features/multinode; ~@known_gap skips any tracked-gap scenarios.
|
||||
run: python -m behave features/multinode -e "features/enterprise" --tags="~@known_gap ~@destructive" --no-capture -f plain
|
||||
- name: Run multi-node failover (destructive)
|
||||
working-directory: testing/cucumber
|
||||
run: python -m behave features/multinode -e "features/enterprise" --tags="@destructive ~@known_gap" --no-capture -f plain
|
||||
- name: Dump node logs on failure
|
||||
if: failure()
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" logs --tail=400 stirling-1 stirling-2
|
||||
- name: Tear down
|
||||
if: always()
|
||||
working-directory: testing/compose
|
||||
run: docker compose -f "$MN_COMPOSE" --profile seed down -v --remove-orphans
|
||||
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
proprietary: ${{ steps.changes.outputs.proprietary }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: .github/config/.files.yaml
|
||||
@@ -267,7 +267,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get commit hashes for frontend and backend
|
||||
id: commit-hashes
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
# runtime token isn't exposed) since the docker driver can't use it.
|
||||
- name: Set up Docker Buildx
|
||||
if: inputs.docker-base-changed != 'true'
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Expose ACTIONS_RUNTIME_TOKEN / ACTIONS_RESULTS_URL for docker buildx type=gha cache backend.
|
||||
- name: Expose GitHub runtime for Buildx cache
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
licenses-backend: ${{ steps.changes.outputs.licenses-backend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: .github/config/.files.yaml
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -326,7 +326,7 @@ jobs:
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
file_suffix: "-server"
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
allowed-endpoints: >
|
||||
@@ -637,7 +637,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
jar_sha256: ${{ steps.hashes.outputs.jar_sha256 }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -43,20 +43,20 @@ jobs:
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
|
||||
@@ -13,6 +13,11 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
build_engine:
|
||||
description: "Build & push the stirling-pdf-engine image (plus the -docparse addon variant)."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
force_unoserver_rebuild:
|
||||
description: "Rebuild stirling-unoserver even if its source hash is unchanged."
|
||||
required: false
|
||||
@@ -51,9 +56,11 @@ jobs:
|
||||
env:
|
||||
RUN_MAIN_APP: ${{ github.event_name != 'workflow_dispatch' || inputs.build_main_app }}
|
||||
RUN_UNOSERVER: ${{ github.event_name != 'workflow_dispatch' || inputs.build_unoserver }}
|
||||
# Engine images are dispatch-only for now; flip the default once the addon stabilises.
|
||||
RUN_ENGINE: ${{ github.event_name == 'workflow_dispatch' && inputs.build_engine }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -82,7 +89,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Install Task
|
||||
uses: go-task/setup-task@01a4adf9db2d14c1de7a560f09170b6e0df736aa # v2.1.0
|
||||
@@ -107,13 +114,13 @@ jobs:
|
||||
cosign-release: "v2.4.1"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -219,6 +226,62 @@ jobs:
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
|
||||
- name: Generate tags for engine
|
||||
id: meta-engine
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
if: env.RUN_ENGINE == 'true'
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-engine
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf-engine
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push engine image
|
||||
id: build-push-engine
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_ENGINE == 'true' && steps.meta-engine.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: ./engine
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-pdf-engine
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-engine
|
||||
tags: ${{ steps.meta-engine.outputs.tags }}
|
||||
labels: ${{ steps.meta-engine.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Generate tags for engine docparse addon
|
||||
id: meta-engine-docparse
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
if: env.RUN_ENGINE == 'true'
|
||||
with:
|
||||
images: |
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-engine
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf-engine
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-docparse
|
||||
type=raw,value=latest-docparse
|
||||
|
||||
- name: Build and push engine docparse addon image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
if: env.RUN_ENGINE == 'true' && steps.meta-engine-docparse.outputs.tags != ''
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: ./engine
|
||||
push: true
|
||||
cache-from: type=gha,scope=stirling-pdf-engine-docparse
|
||||
cache-to: type=gha,mode=max,scope=stirling-pdf-engine-docparse
|
||||
tags: ${{ steps.meta-engine-docparse.outputs.tags }}
|
||||
labels: ${{ steps.meta-engine-docparse.outputs.labels }}
|
||||
build-args: DOCPARSE=true
|
||||
platforms: linux/amd64
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Generate tags for ultra-lite
|
||||
id: meta-lite
|
||||
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
packages: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -26,13 +26,13 @@ jobs:
|
||||
uses: imjasonh/setup-crane@31b88afe9de28ae0ffa220711af4b60be9435f6e # v0.4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
RELEASE_GPG_PRIVATE_KEY: ${{ secrets.RELEASE_GPG_PRIVATE_KEY }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -420,9 +420,7 @@ jobs:
|
||||
# Linux: build deb+rpm only here (deb-only on minimal smoke builds).
|
||||
# AppImage runs in its own continue-on-error step below so its
|
||||
# persistent linuxdeploy failure (#6127 onwards) does not tank uploads.
|
||||
args: >-
|
||||
${{ matrix.platform == 'ubuntu-22.04' && (inputs.minimal && '--bundles deb' || '--bundles deb,rpm') || matrix.args }}
|
||||
--config '{"bundle":{"createUpdaterArtifacts":false}}'
|
||||
args: ${{ matrix.platform == 'ubuntu-22.04' && (inputs.minimal && '--bundles deb' || '--bundles deb,rpm') || matrix.args }}
|
||||
|
||||
# AppImage is decoupled so its linuxdeploy run gets a fresh process
|
||||
# (rpm scratch state torn down) and its failure can't tank deb/rpm.
|
||||
@@ -615,19 +613,12 @@ jobs:
|
||||
pr-comment:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
# Fork and Dependabot pull_request runs receive a read-only GITHUB_TOKEN,
|
||||
# so the API cannot create or update PR comments there. The artifacts are
|
||||
# still uploaded and remain available from the Actions run page.
|
||||
if: >-
|
||||
github.event_name == 'pull_request' &&
|
||||
needs.build.result == 'success' &&
|
||||
!github.event.pull_request.head.repo.fork &&
|
||||
github.actor != 'dependabot[bot]'
|
||||
if: github.event_name == 'pull_request' && needs.build.result == 'success'
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden the runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -718,7 +709,7 @@ jobs:
|
||||
if: always()
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
cache-scope: stirling-pdf-fat
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build base image locally (PR base change only)
|
||||
if: github.event_name == 'pull_request' && inputs.docker-base-changed == 'true'
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build docker/unoserver/Dockerfile
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
@@ -131,14 +131,14 @@ jobs:
|
||||
frontend: ${{ steps.changes.outputs.frontend }}
|
||||
steps:
|
||||
- name: Harden the runner (Audit all outbound calls)
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: changes
|
||||
with:
|
||||
filters: ".github/config/.files.yaml"
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0
|
||||
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
|
||||
@@ -263,16 +263,6 @@ tasks:
|
||||
- task: lint:eslint
|
||||
- task: lint:dpdm
|
||||
- task: lint:colors
|
||||
- task: lint:css
|
||||
|
||||
lint:css:
|
||||
desc: "Lint stylesheets for duplicate selectors"
|
||||
deps: [install]
|
||||
cmds:
|
||||
# Covers the whole editor tree, including the portal/processor layer and
|
||||
# public/css. Vendored CSS and build output are excluded via ignoreFiles
|
||||
# in stylelint.config.mjs.
|
||||
- npx stylelint "editor/**/*.css"
|
||||
|
||||
lint:colors:
|
||||
desc: "Enforce theme tokens — no hardcoded colours or raw primitives in components"
|
||||
@@ -398,13 +388,6 @@ tasks:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: editor/src/portal/tsconfig.json }
|
||||
|
||||
typecheck:storybook:
|
||||
desc: "Typecheck Storybook config and stories"
|
||||
deps: [prepare]
|
||||
cmds:
|
||||
- task: typecheck:_run
|
||||
vars: { PROJECT: .storybook/tsconfig.json }
|
||||
|
||||
typecheck:all:
|
||||
desc: "Typecheck all build variants"
|
||||
cmds:
|
||||
@@ -416,7 +399,6 @@ tasks:
|
||||
- task: typecheck:scripts
|
||||
- task: typecheck:prototypes
|
||||
- task: typecheck:portal
|
||||
- task: typecheck:storybook
|
||||
|
||||
# ============================================================
|
||||
# Quality Gate
|
||||
|
||||
@@ -338,7 +338,6 @@ public class EndpointConfiguration {
|
||||
addEndpointToGroup("PageOps", "split-pages");
|
||||
addEndpointToGroup("PageOps", "rearrange-pages");
|
||||
addEndpointToGroup("PageOps", "rotate-pdf");
|
||||
addEndpointToGroup("PageOps", "auto-rotate-pdf");
|
||||
addEndpointToGroup("PageOps", "multi-page-layout");
|
||||
addEndpointToGroup("PageOps", "booklet-imposition");
|
||||
addEndpointToGroup("PageOps", "scale-pages");
|
||||
@@ -434,6 +433,20 @@ public class EndpointConfiguration {
|
||||
addEndpointToGroup("Automation", "automate"); // Alias for handleData (user-friendly name)
|
||||
addEndpointToGroup("Automation", "pipeline");
|
||||
|
||||
// Adding endpoints to "DocParse" group (parsing, splitting, chunking, extraction,
|
||||
// templating)
|
||||
addEndpointToGroup("DocParse", "parse-document");
|
||||
addEndpointToGroup("DocParse", "extract-fields");
|
||||
addEndpointToGroup("DocParse", "smart-split");
|
||||
addEndpointToGroup("DocParse", "chunk-document");
|
||||
addEndpointToGroup("DocParse", "rag-ingest");
|
||||
addEndpointToGroup("DocParse", "rag-documents");
|
||||
addEndpointToGroup("DocParse", "rag-search");
|
||||
addEndpointToGroup("DocParse", "rag-ask");
|
||||
addEndpointToGroup("DocParse", "extract-tables");
|
||||
addEndpointToGroup("DocParse", "suggest-schema");
|
||||
addEndpointToGroup("DocParse", "fill-template");
|
||||
|
||||
// Adding endpoints to "DeveloperTools" group
|
||||
addEndpointToGroup("DeveloperTools", "show-javascript");
|
||||
|
||||
|
||||
+19
-14
@@ -77,6 +77,7 @@ public class ApplicationProperties {
|
||||
private ProcessExecutor processExecutor = new ProcessExecutor();
|
||||
private PdfEditor pdfEditor = new PdfEditor();
|
||||
private AiEngine aiEngine = new AiEngine();
|
||||
private Docparse docparse = new Docparse();
|
||||
private Mcp mcp = new Mcp();
|
||||
private InternalApi internalApi = new InternalApi();
|
||||
private Cluster cluster = new Cluster();
|
||||
@@ -425,6 +426,24 @@ public class ApplicationProperties {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DocParse settings (top-level {@code docparse.*}): document understanding for ingestion
|
||||
* pipelines. The basic tier (text layer) always works; the advanced tier lives in the engine's
|
||||
* docparse addon.
|
||||
*/
|
||||
@Data
|
||||
public static class Docparse {
|
||||
|
||||
/** Master switch; hides the DocParse endpoints when false. */
|
||||
private boolean enabled = true;
|
||||
|
||||
/** Requested tier: 'auto', 'basic', or 'advanced'. 'auto' resolves per document. */
|
||||
private String mode = "auto";
|
||||
|
||||
/** Mirrors DOCPARSE_AUTO_INSTALL for the engine's boot-time addon install script. */
|
||||
private boolean autoInstall = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model Context Protocol (MCP) server configuration. All keys live under the top-level {@code
|
||||
* mcp.*} prefix. {@link #enabled} defaults to {@code false}: when off, no MCP beans are wired,
|
||||
@@ -1076,20 +1095,6 @@ public class ApplicationProperties {
|
||||
private Quotas quotas = new Quotas();
|
||||
private Sharing sharing = new Sharing();
|
||||
private Signing signing = new Signing();
|
||||
private Encryption encryption = new Encryption();
|
||||
|
||||
/**
|
||||
* Encryption at rest for stored files (Pro/Enterprise). Enabling encrypts new writes;
|
||||
* disabling later only stops encrypting new writes — existing encrypted files keep
|
||||
* decrypting as long as the key material is present. The master key is resolved like the
|
||||
* credential key: {@code stirling.security.fileEncryptionKey} property, {@code
|
||||
* STIRLING_FILE_ENCRYPTION_KEY} env var, or an auto-generated {@code file-encryption.key}
|
||||
* in the config directory.
|
||||
*/
|
||||
@Data
|
||||
public static class Encryption {
|
||||
private boolean enabled = false;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class Local {
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
/**
|
||||
* View of the engine's DocParse capability for modules that cannot see the proprietary
|
||||
* implementation (e.g. ConfigController in core). Implemented by the proprietary
|
||||
* DocparseCapabilityService; absent when the proprietary module is not loaded.
|
||||
*/
|
||||
public interface DocparseCapabilityServiceInterface {
|
||||
|
||||
/**
|
||||
* Whether the engine reports the docparse addon (advanced tier) as installed. Must be cheap and
|
||||
* non-blocking: returns the cached probe result, {@code false} when the engine is disabled,
|
||||
* unreachable, or not yet probed.
|
||||
*/
|
||||
boolean isAdvancedInstalled();
|
||||
}
|
||||
@@ -53,7 +53,7 @@ public class InternalApiClient {
|
||||
// ApiConnectionResolver.
|
||||
private static final Pattern ALLOWED_ENDPOINT_PATH =
|
||||
Pattern.compile(
|
||||
"^/api/v1/(general|misc|security|convert|filter|integration)(/[A-Za-z0-9_-]+)+$"
|
||||
"^/api/v1/(general|misc|security|convert|filter|integration|docparse)(/[A-Za-z0-9_-]+)+$"
|
||||
+ "|^/api/v1/ai/tools(/[A-Za-z0-9_-]+)+$");
|
||||
|
||||
/**
|
||||
|
||||
@@ -50,7 +50,7 @@ dependencies {
|
||||
implementation "org.apache.pdfbox:preflight:$pdfboxVersion"
|
||||
implementation "org.apache.pdfbox:xmpbox:$pdfboxVersion"
|
||||
|
||||
implementation 'org.verapdf:validation-model:1.30.2'
|
||||
implementation 'org.verapdf:validation-model:1.28.2'
|
||||
// CVE-2025-66453: Explicit rhino 1.7.15 to override verapdf's 1.7.13
|
||||
implementation "org.mozilla:rhino:${rhinoVersion}"
|
||||
|
||||
|
||||
-385
@@ -1,385 +0,0 @@
|
||||
package stirling.software.SPDF.controller.api.misc;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult.PageResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotatePdfRequest;
|
||||
import stirling.software.SPDF.model.api.misc.PageRotation;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.OsdResult;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.TextDirection;
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.annotations.api.MiscApi;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.ProcessExecutor;
|
||||
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
|
||||
import stirling.software.common.util.TempDirectory;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
|
||||
@MiscApi
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class AutoRotateController {
|
||||
|
||||
// OSD decides orientation from script shape, not character identity, so it does not need
|
||||
// OCR-grade resolution. 150 DPI is ample for that and a quarter of the pixels of 300.
|
||||
private static final int OSD_RENDER_DPI = 150;
|
||||
|
||||
private static final String METHOD_TEXT = "text";
|
||||
private static final String METHOD_OSD = "osd";
|
||||
private static final String METHOD_INFERRED = "inferred";
|
||||
private static final String METHOD_NONE = "none";
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final EndpointConfiguration endpointConfiguration;
|
||||
private final RuntimePathConfig runtimePathConfig;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/auto-rotate-pdf",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Detect and fix the orientation of every page",
|
||||
description =
|
||||
"Detects each page's orientation (embedded-text direction first, Tesseract OSD"
|
||||
+ " for scanned pages) and sets the page rotation so the content"
|
||||
+ " displays upright. With dryRun=true, returns a JSON per-page report"
|
||||
+ " instead of the PDF. With pageRotations set, applies the given"
|
||||
+ " corrections without running detection."
|
||||
+ " Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<?> autoRotatePdf(@Valid @ModelAttribute AutoRotatePdfRequest request)
|
||||
throws IOException, InterruptedException {
|
||||
String mode =
|
||||
request.getDetectionMode() == null
|
||||
? "auto"
|
||||
: request.getDetectionMode().toLowerCase(Locale.ROOT);
|
||||
if (!"auto".equals(mode) && !"text".equals(mode) && !"osd".equals(mode)) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"detectionMode",
|
||||
"must be one of auto, text, osd");
|
||||
}
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(request)) {
|
||||
if (request.getPageRotations() != null && !request.getPageRotations().isEmpty()) {
|
||||
applyExplicitRotations(document, request.getPageRotations());
|
||||
return pdfResponse(document, request);
|
||||
}
|
||||
|
||||
AutoRotateAnalysisResult analysis = analyse(document, mode, request);
|
||||
if (request.isDryRun()) {
|
||||
return ResponseEntity.ok(analysis);
|
||||
}
|
||||
for (PageResult pageResult : analysis.getPages()) {
|
||||
if (pageResult.isApply()) {
|
||||
PDPage page = document.getPage(pageResult.getPageNumber() - 1);
|
||||
page.setRotation(
|
||||
Math.floorMod(page.getRotation() + pageResult.getCorrection(), 360));
|
||||
}
|
||||
}
|
||||
return pdfResponse(document, request);
|
||||
}
|
||||
}
|
||||
|
||||
private AutoRotateAnalysisResult analyse(
|
||||
PDDocument document, String mode, AutoRotatePdfRequest request)
|
||||
throws IOException, InterruptedException {
|
||||
double threshold =
|
||||
request.getConfidenceThreshold() == null ? 14.0 : request.getConfidenceThreshold();
|
||||
boolean tesseractAvailable = endpointConfiguration.isGroupEnabled("tesseract");
|
||||
boolean useText = !"osd".equals(mode);
|
||||
boolean useOsd = !"text".equals(mode);
|
||||
|
||||
List<PageResult> results = new ArrayList<>();
|
||||
List<Integer> osdCandidates = new ArrayList<>();
|
||||
|
||||
int pageCount = document.getNumberOfPages();
|
||||
// One walk of the document for all pages, rather than one walk per page.
|
||||
List<TextDirection> textDirections =
|
||||
useText ? AutoRotateDetection.detectTextDirections(document) : List.of();
|
||||
|
||||
for (int i = 0; i < pageCount; i++) {
|
||||
int currentRotation = Math.floorMod(document.getPage(i).getRotation(), 360);
|
||||
PageResult result =
|
||||
PageResult.builder()
|
||||
.pageNumber(i + 1)
|
||||
.currentRotation(currentRotation)
|
||||
.method(METHOD_NONE)
|
||||
.build();
|
||||
|
||||
if (useText) {
|
||||
TextDirection direction = textDirections.get(i);
|
||||
if (direction.isConclusive()) {
|
||||
int correction =
|
||||
AutoRotateDetection.correctionFromTextDirection(
|
||||
direction.dominantDirection(), currentRotation);
|
||||
result.setMethod(METHOD_TEXT);
|
||||
result.setCorrection(correction);
|
||||
result.setConfidence(direction.dominance() * 100);
|
||||
result.setApply(correction != 0);
|
||||
} else if (!useOsd) {
|
||||
result.setNote(
|
||||
direction.glyphCount() < AutoRotateDetection.MIN_GLYPHS
|
||||
? "tooFewGlyphs"
|
||||
: "noDominantDirection");
|
||||
}
|
||||
}
|
||||
|
||||
if (useOsd && METHOD_NONE.equals(result.getMethod())) {
|
||||
if (tesseractAvailable) {
|
||||
osdCandidates.add(i);
|
||||
} else {
|
||||
result.setNote("tesseractUnavailable");
|
||||
}
|
||||
}
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
if (!osdCandidates.isEmpty()) {
|
||||
runOsdOnPages(document, osdCandidates, results, threshold);
|
||||
}
|
||||
|
||||
if (request.isInferUndetected()) {
|
||||
inferUndetectedPages(results);
|
||||
}
|
||||
|
||||
return summarise(results, pageCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill in pages that no signal could decide, using the pages that could. When every decided
|
||||
* page sharing an undecided page's current rotation agrees on one correction, that correction
|
||||
* is the document's consensus for that rotation and is applied to the undecided page. This is
|
||||
* the common "whole document rotated uniformly, but a cover or near-blank page has too little
|
||||
* text to detect on its own" case. If decided pages disagree, nothing is inferred.
|
||||
*/
|
||||
private void inferUndetectedPages(List<PageResult> results) {
|
||||
// rotation -> the single agreed correction, or null once a conflict is seen
|
||||
Map<Integer, Integer> consensus = new HashMap<>();
|
||||
Set<Integer> conflicted = new HashSet<>();
|
||||
for (PageResult result : results) {
|
||||
if (METHOD_NONE.equals(result.getMethod())) {
|
||||
continue;
|
||||
}
|
||||
int rotation = result.getCurrentRotation();
|
||||
if (conflicted.contains(rotation)) {
|
||||
continue;
|
||||
}
|
||||
Integer existing = consensus.get(rotation);
|
||||
if (existing == null) {
|
||||
consensus.put(rotation, result.getCorrection());
|
||||
} else if (existing != result.getCorrection()) {
|
||||
conflicted.add(rotation);
|
||||
consensus.remove(rotation);
|
||||
}
|
||||
}
|
||||
|
||||
for (PageResult result : results) {
|
||||
if (!METHOD_NONE.equals(result.getMethod())) {
|
||||
continue;
|
||||
}
|
||||
Integer correction = consensus.get(result.getCurrentRotation());
|
||||
if (correction == null) {
|
||||
continue;
|
||||
}
|
||||
result.setMethod(METHOD_INFERRED);
|
||||
result.setCorrection(correction);
|
||||
result.setConfidence(null);
|
||||
result.setApply(correction != 0);
|
||||
result.setNote("inferredFromDocument");
|
||||
}
|
||||
}
|
||||
|
||||
private void runOsdOnPages(
|
||||
PDDocument document,
|
||||
List<Integer> pageIndexes,
|
||||
List<PageResult> results,
|
||||
double threshold)
|
||||
throws IOException, InterruptedException {
|
||||
String tessDataPath = runtimePathConfig.getTessDataPath();
|
||||
boolean haveOsdData =
|
||||
tessDataPath != null && new File(tessDataPath, "osd.traineddata").exists();
|
||||
|
||||
int dpi = OSD_RENDER_DPI;
|
||||
if (applicationProperties != null && applicationProperties.getSystem() != null) {
|
||||
dpi = Math.min(OSD_RENDER_DPI, applicationProperties.getSystem().getMaxDPI());
|
||||
}
|
||||
final int renderDpi = dpi;
|
||||
|
||||
try (TempDirectory tempDir = new TempDirectory(tempFileManager)) {
|
||||
PDFRenderer renderer = new PDFRenderer(document);
|
||||
renderer.setSubsamplingAllowed(true);
|
||||
// One reused path, deleted after every page: the images are throwaway input for
|
||||
// Tesseract, so a long document must not accumulate one file per page on disk.
|
||||
File imageFile = new File(tempDir.getPath().toFile(), "osd-page.bmp");
|
||||
|
||||
for (int pageIndex : pageIndexes) {
|
||||
PageResult result = results.get(pageIndex);
|
||||
try {
|
||||
// Rendering honours the page's current /Rotate, so OSD sees the page exactly
|
||||
// as a viewer would and its verdict is always an additive correction.
|
||||
BufferedImage image =
|
||||
ExceptionUtils.handleOomRendering(
|
||||
pageIndex + 1,
|
||||
renderDpi,
|
||||
() ->
|
||||
renderer.renderImageWithDPI(
|
||||
pageIndex, renderDpi, ImageType.GRAY));
|
||||
|
||||
if (AutoRotateDetection.isBlankRender(image)) {
|
||||
// Nothing for OSD to read; skip the process spawn entirely.
|
||||
result.setNote("blankPage");
|
||||
continue;
|
||||
}
|
||||
|
||||
// BMP, not PNG: the file is deleted straight after Tesseract reads it, so
|
||||
// paying for compression only to discard the result is wasted work.
|
||||
ImageIO.write(image, "bmp", imageFile);
|
||||
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add("tesseract");
|
||||
command.add(imageFile.getAbsolutePath());
|
||||
command.add("stdout");
|
||||
command.add("--psm");
|
||||
command.add("0");
|
||||
if (haveOsdData) {
|
||||
command.add("--tessdata-dir");
|
||||
command.add(tessDataPath);
|
||||
}
|
||||
|
||||
ProcessExecutorResult processResult =
|
||||
ProcessExecutor.getInstance(ProcessExecutor.Processes.TESSERACT)
|
||||
.runCommandWithOutputHandling(command);
|
||||
|
||||
Optional<OsdResult> osd =
|
||||
AutoRotateDetection.parseOsd(processResult.getMessages());
|
||||
if (osd.isEmpty()) {
|
||||
result.setNote("osdNoVerdict");
|
||||
continue;
|
||||
}
|
||||
result.setConfidence(osd.get().confidence());
|
||||
result.setCorrection(osd.get().rotate());
|
||||
if (osd.get().confidence() >= threshold) {
|
||||
result.setMethod(METHOD_OSD);
|
||||
result.setApply(osd.get().rotate() != 0);
|
||||
} else {
|
||||
result.setNote("belowThreshold");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Textless pages make Tesseract exit non-zero; skip, never guess.
|
||||
log.debug("OSD failed for page {}: {}", pageIndex + 1, e.getMessage());
|
||||
result.setNote("osdFailed");
|
||||
} finally {
|
||||
Files.deleteIfExists(imageFile.toPath());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void applyExplicitRotations(PDDocument document, List<PageRotation> rotations) {
|
||||
int pageCount = document.getNumberOfPages();
|
||||
Set<Integer> seen = new HashSet<>();
|
||||
for (PageRotation entry : rotations) {
|
||||
Integer pageNumber = entry.getPageNumber();
|
||||
Integer angle = entry.getRotation();
|
||||
if (pageNumber == null
|
||||
|| angle == null
|
||||
|| pageNumber < 1
|
||||
|| pageNumber > pageCount
|
||||
|| angle % 90 != 0) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"pageRotations",
|
||||
"page numbers must exist and rotations must be multiples of 90");
|
||||
}
|
||||
// Rotations are additive, so a repeated page would be turned twice; reject rather
|
||||
// than silently pick a winner.
|
||||
if (!seen.add(pageNumber)) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
ExceptionUtils.ErrorCode.INVALID_ARGUMENT,
|
||||
"pageRotations",
|
||||
"page " + pageNumber + " is listed more than once");
|
||||
}
|
||||
PDPage page = document.getPage(pageNumber - 1);
|
||||
page.setRotation(Math.floorMod(page.getRotation() + angle, 360));
|
||||
}
|
||||
}
|
||||
|
||||
private AutoRotateAnalysisResult summarise(List<PageResult> results, int pageCount) {
|
||||
int toRotate = 0;
|
||||
int byText = 0;
|
||||
int byOsd = 0;
|
||||
int byInference = 0;
|
||||
int undetected = 0;
|
||||
for (PageResult result : results) {
|
||||
if (result.isApply()) {
|
||||
toRotate++;
|
||||
}
|
||||
switch (result.getMethod()) {
|
||||
case METHOD_TEXT -> byText++;
|
||||
case METHOD_OSD -> byOsd++;
|
||||
case METHOD_INFERRED -> byInference++;
|
||||
default -> undetected++;
|
||||
}
|
||||
}
|
||||
return AutoRotateAnalysisResult.builder()
|
||||
.pages(results)
|
||||
.totalPages(pageCount)
|
||||
.pagesToRotate(toRotate)
|
||||
.detectedByText(byText)
|
||||
.detectedByOsd(byOsd)
|
||||
.inferred(byInference)
|
||||
.undetected(undetected)
|
||||
.build();
|
||||
}
|
||||
|
||||
private ResponseEntity<?> pdfResponse(PDDocument document, AutoRotatePdfRequest request)
|
||||
throws IOException {
|
||||
String originalName =
|
||||
request.getFileInput() != null
|
||||
? request.getFileInput().getOriginalFilename()
|
||||
: "document.pdf";
|
||||
return WebResponseUtils.pdfDocToWebResponse(
|
||||
document,
|
||||
GeneralUtils.generateFilename(originalName, "_auto_rotated.pdf"),
|
||||
tempFileManager);
|
||||
}
|
||||
}
|
||||
+16
-1
@@ -24,6 +24,7 @@ import stirling.software.common.annotations.api.ConfigApi;
|
||||
import stirling.software.common.configuration.AppConfig;
|
||||
import stirling.software.common.configuration.interfaces.ShowAdminInterface;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.DocparseCapabilityServiceInterface;
|
||||
import stirling.software.common.service.ServerCertificateServiceInterface;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
@@ -41,6 +42,7 @@ public class ConfigController {
|
||||
private final ShowAdminInterface showAdmin;
|
||||
private final stirling.software.common.service.LicenseServiceInterface licenseService;
|
||||
private final stirling.software.SPDF.config.ExternalAppDepConfig externalAppDepConfig;
|
||||
private final DocparseCapabilityServiceInterface docparseCapabilityService;
|
||||
|
||||
public ConfigController(
|
||||
ApplicationProperties applicationProperties,
|
||||
@@ -54,7 +56,9 @@ public class ConfigController {
|
||||
ShowAdminInterface showAdmin,
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
stirling.software.common.service.LicenseServiceInterface licenseService,
|
||||
stirling.software.SPDF.config.ExternalAppDepConfig externalAppDepConfig) {
|
||||
stirling.software.SPDF.config.ExternalAppDepConfig externalAppDepConfig,
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
DocparseCapabilityServiceInterface docparseCapabilityService) {
|
||||
this.applicationProperties = applicationProperties;
|
||||
this.applicationContext = applicationContext;
|
||||
this.endpointConfiguration = endpointConfiguration;
|
||||
@@ -63,6 +67,7 @@ public class ConfigController {
|
||||
this.showAdmin = showAdmin;
|
||||
this.licenseService = licenseService;
|
||||
this.externalAppDepConfig = externalAppDepConfig;
|
||||
this.docparseCapabilityService = docparseCapabilityService;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -350,6 +355,16 @@ public class ConfigController {
|
||||
Map.entry("pdfComment", aiFeatures.isPdfComment()),
|
||||
Map.entry("classify", aiFeatures.isClassify())));
|
||||
|
||||
// DocParse settings; "advanced" reflects the cached engine capability probe and is
|
||||
// false when the engine is disabled, unreachable, or the proprietary module is absent.
|
||||
boolean docparseEnabled = applicationProperties.getDocparse().isEnabled();
|
||||
configData.put("docparseEnabled", docparseEnabled);
|
||||
configData.put(
|
||||
"docparseAdvanced",
|
||||
docparseEnabled
|
||||
&& docparseCapabilityService != null
|
||||
&& docparseCapabilityService.isAdvancedInstalled());
|
||||
|
||||
// Timestamp TSA settings — single source of truth for presets + admin URLs
|
||||
ApplicationProperties.Security.Timestamp tsConfig =
|
||||
applicationProperties.getSecurity().getTimestamp();
|
||||
|
||||
+2
-6
@@ -140,15 +140,11 @@ public class ValidateSignatureController {
|
||||
// covers the TSTInfo, which can never match.
|
||||
boolean isDocTimeStamp = SUBFILTER_RFC3161.equals(sig.getSubFilter());
|
||||
CMSSignedData signedData;
|
||||
// Parse from a stream: /Contents is zero-padded to its reserved length and the
|
||||
// byte[] constructors reject those trailing bytes since BC 1.85.
|
||||
if (isDocTimeStamp) {
|
||||
signedData = new CMSSignedData(new ByteArrayInputStream(signatureBytes));
|
||||
signedData = new CMSSignedData(signatureBytes);
|
||||
} else {
|
||||
CMSProcessable content = new CMSProcessableByteArray(signedContent);
|
||||
signedData =
|
||||
new CMSSignedData(
|
||||
content, new ByteArrayInputStream(signatureBytes));
|
||||
signedData = new CMSSignedData(content, signatureBytes);
|
||||
}
|
||||
|
||||
// What actually binds a timestamp to this document: the TSTInfo's message
|
||||
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** Per-page orientation report returned by auto-rotate-pdf when dryRun is set. */
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AutoRotateAnalysisResult {
|
||||
|
||||
private List<PageResult> pages;
|
||||
|
||||
private int totalPages;
|
||||
|
||||
@Schema(description = "Number of pages a correction would be applied to")
|
||||
private int pagesToRotate;
|
||||
|
||||
private int detectedByText;
|
||||
|
||||
private int detectedByOsd;
|
||||
|
||||
@Schema(description = "Pages whose correction was inherited from the document consensus")
|
||||
private int inferred;
|
||||
|
||||
private int undetected;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class PageResult {
|
||||
|
||||
@Schema(description = "1-based page number")
|
||||
private int pageNumber;
|
||||
|
||||
@Schema(description = "The page's current /Rotate value, normalised to 0-270")
|
||||
private int currentRotation;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detected additional clockwise rotation that would make the page upright."
|
||||
+ " Reported for diagnostics even when it is not used (for example"
|
||||
+ " an OSD verdict below the confidence threshold); 'apply' is the"
|
||||
+ " authority on whether it is actually applied")
|
||||
private int correction;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detection confidence: percentage of glyphs sharing the dominant direction"
|
||||
+ " for method 'text', Tesseract orientation confidence for method"
|
||||
+ " 'osd', absent when nothing was detected")
|
||||
private Double confidence;
|
||||
|
||||
@Schema(
|
||||
description = "How the orientation was determined",
|
||||
allowableValues = {"text", "osd", "inferred", "none"})
|
||||
private String method;
|
||||
|
||||
@Schema(description = "Whether the correction will be (or was) applied")
|
||||
private boolean apply;
|
||||
|
||||
@Schema(description = "Machine-readable reason when no correction is applied")
|
||||
private String note;
|
||||
}
|
||||
}
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Min;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class AutoRotatePdfRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Detection method. 'auto' tries embedded-text direction first and falls back"
|
||||
+ " to Tesseract OSD for pages without usable text; 'text' uses only"
|
||||
+ " embedded-text direction; 'osd' forces Tesseract OSD for every page",
|
||||
allowableValues = {"auto", "text", "osd"},
|
||||
defaultValue = "auto")
|
||||
private String detectionMode = "auto";
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Minimum Tesseract OSD orientation confidence required before a correction is"
|
||||
+ " applied. Matches OCRmyPDF's --rotate-pages-threshold scale",
|
||||
minimum = "0",
|
||||
defaultValue = "14.0")
|
||||
@Min(value = 0, message = "Confidence threshold must be non-negative")
|
||||
private Double confidenceThreshold = 14.0;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"If true, no rotation is applied; returns a JSON report of the per-page"
|
||||
+ " detection results instead of a PDF")
|
||||
private boolean dryRun;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"When a page cannot be decided on its own but the pages that could be decided"
|
||||
+ " agree on a single correction for that same current rotation, apply"
|
||||
+ " that shared correction to the undecided page. Handles documents"
|
||||
+ " rotated uniformly where some pages are too sparse to detect alone",
|
||||
defaultValue = "true")
|
||||
private boolean inferUndetected = true;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Optional pre-computed corrections to apply without running detection. Pages"
|
||||
+ " not listed are left unchanged, and a page may only appear once")
|
||||
@Valid
|
||||
private List<PageRotation> pageRotations;
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package stirling.software.SPDF.model.api.misc;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** A pre-computed rotation for one page, used by auto-rotate-pdf's apply-only path. */
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class PageRotation {
|
||||
|
||||
@Schema(
|
||||
description = "1-based page number to rotate",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example = "1")
|
||||
private Integer pageNumber;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Additional clockwise rotation to add to the page's current rotation, in"
|
||||
+ " degrees. Must be a multiple of 90",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example = "90")
|
||||
private Integer rotation;
|
||||
}
|
||||
@@ -200,8 +200,8 @@ public class VeraPDFService {
|
||||
|
||||
try (PDFAParser detectionParser =
|
||||
Foundries.defaultInstance().createParser(new ByteArrayInputStream(pdfBytes))) {
|
||||
detectedFlavours = detectedFlavours(detectionParser);
|
||||
declaredFlavour = firstFlavour(detectedFlavours);
|
||||
declaredFlavour = detectionParser.getFlavour();
|
||||
detectedFlavours = detectionParser.getFlavours();
|
||||
}
|
||||
|
||||
// For PDF/A flavours, we need to validate first to check if PDF/A identification exists in
|
||||
@@ -299,7 +299,7 @@ public class VeraPDFService {
|
||||
Foundries.defaultInstance()
|
||||
.createParser(new ByteArrayInputStream(pdfBytes), flavour)) {
|
||||
|
||||
PDFAFlavour parserDeclared = firstFlavour(detectedFlavours(parser));
|
||||
PDFAFlavour parserDeclared = parser.getFlavour();
|
||||
PDFAValidator validator =
|
||||
Foundries.defaultInstance().createValidator(flavour, false);
|
||||
ValidationResult result = validator.validate(parser);
|
||||
@@ -322,19 +322,7 @@ public class VeraPDFService {
|
||||
}
|
||||
|
||||
private static boolean isPdfaFlavour(PDFAFlavour flavour) {
|
||||
return flavour != null
|
||||
&& PDFFlavours.isFlavourFamily(flavour, PDFAFlavour.SpecificationFamily.PDF_A);
|
||||
}
|
||||
|
||||
// veraPDF 1.30+ returns an empty flavour list for non-PDF/A files, where getFlavour() throws
|
||||
private static List<PDFAFlavour> detectedFlavours(PDFAParser parser) {
|
||||
List<PDFAFlavour> flavours = parser.getFlavours();
|
||||
return flavours != null ? flavours : List.of();
|
||||
}
|
||||
|
||||
// null means "no PDF/A flavour detected" rather than an error
|
||||
private static PDFAFlavour firstFlavour(List<PDFAFlavour> flavours) {
|
||||
return flavours.isEmpty() ? null : flavours.get(0);
|
||||
return PDFFlavours.isFlavourFamily(flavour, PDFAFlavour.SpecificationFamily.PDF_A);
|
||||
}
|
||||
|
||||
private static String formatStandardDisplay(
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
package stirling.software.SPDF.utils;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* Page-orientation detection primitives for the auto-rotate tool.
|
||||
*
|
||||
* <p>Two independent signals are supported: the dominant direction of a page's embedded text
|
||||
* (cheap, digital PDFs only) and Tesseract's orientation-and-script-detection output (works on
|
||||
* scans, requires the external binary). Both express their result as the additional clockwise
|
||||
* /Rotate correction that makes the page display upright.
|
||||
*/
|
||||
public final class AutoRotateDetection {
|
||||
|
||||
private AutoRotateDetection() {}
|
||||
|
||||
/** Glyphs needed to trust the text signal at the ordinary dominance bar. */
|
||||
public static final int MIN_GLYPHS = 30;
|
||||
|
||||
/** Fraction of glyphs that must share one direction at the ordinary bar. */
|
||||
public static final double MIN_DOMINANCE = 0.95;
|
||||
|
||||
/**
|
||||
* Glyphs needed to trust the text signal when the glyphs are near-unanimous. Lets sparse pages
|
||||
* (a header, a single line, a rotated URL) be decided from their own text instead of falling
|
||||
* through to OSD, as long as effectively every glyph agrees on the direction.
|
||||
*/
|
||||
public static final int MIN_GLYPHS_UNANIMOUS = 8;
|
||||
|
||||
/** Dominance required for the sparse-page path — essentially total agreement. */
|
||||
public static final double UNANIMOUS_DOMINANCE = 0.99;
|
||||
|
||||
/**
|
||||
* Dominant embedded-text direction of one page.
|
||||
*
|
||||
* @param dominantDirection glyph direction in page space, degrees CCW (0/90/180/270)
|
||||
* @param dominance fraction of counted glyphs sharing the dominant direction (0..1)
|
||||
* @param glyphCount number of non-whitespace glyphs counted
|
||||
*/
|
||||
public record TextDirection(int dominantDirection, double dominance, int glyphCount) {
|
||||
|
||||
public boolean isConclusive() {
|
||||
if (glyphCount >= MIN_GLYPHS && dominance >= MIN_DOMINANCE) {
|
||||
return true;
|
||||
}
|
||||
return glyphCount >= MIN_GLYPHS_UNANIMOUS && dominance >= UNANIMOUS_DOMINANCE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed Tesseract OSD verdict.
|
||||
*
|
||||
* @param rotate clockwise degrees to rotate the rendered page so text is upright
|
||||
* @param confidence Tesseract's orientation confidence (same scale OCRmyPDF thresholds on)
|
||||
*/
|
||||
public record OsdResult(int rotate, double confidence) {}
|
||||
|
||||
private static final Pattern OSD_ROTATE =
|
||||
Pattern.compile("^Rotate:\\s*(\\d+)", Pattern.MULTILINE);
|
||||
private static final Pattern OSD_CONFIDENCE =
|
||||
Pattern.compile("^Orientation confidence:\\s*([0-9.]+)", Pattern.MULTILINE);
|
||||
|
||||
/**
|
||||
* Counts non-whitespace glyph directions for every page in one pass. A stripper per page would
|
||||
* re-walk the document once per page, which is quadratic on long documents; this walks it once
|
||||
* and buckets glyphs by the page being processed.
|
||||
*
|
||||
* @return one entry per page, in page order
|
||||
*/
|
||||
public static List<TextDirection> detectTextDirections(PDDocument document) throws IOException {
|
||||
int pageCount = document.getNumberOfPages();
|
||||
DirectionCountingStripper stripper = new DirectionCountingStripper(pageCount);
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(pageCount);
|
||||
stripper.getText(document);
|
||||
|
||||
List<TextDirection> directions = new ArrayList<>(pageCount);
|
||||
for (int page = 0; page < pageCount; page++) {
|
||||
int[] counts = stripper.counts[page];
|
||||
int total = 0;
|
||||
int bestIndex = 0;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
total += counts[i];
|
||||
if (counts[i] > counts[bestIndex]) {
|
||||
bestIndex = i;
|
||||
}
|
||||
}
|
||||
double dominance = total == 0 ? 0 : (double) counts[bestIndex] / total;
|
||||
directions.add(new TextDirection(bestIndex * 90, dominance, total));
|
||||
}
|
||||
return directions;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a rendered page carries no ink worth analysing. Checked after rendering but before
|
||||
* spawning Tesseract, since the process spawn costs far more than the pixel scan and this
|
||||
* catches both empty generated pages and scanned blanks (the back of a duplex sheet).
|
||||
*/
|
||||
public static boolean isBlankRender(BufferedImage image) {
|
||||
final int darkThreshold = 200; // 8-bit grey; anything lighter counts as paper
|
||||
final int step = 4; // subsample: blank pages are uniform, no need for every pixel
|
||||
long sampled = 0;
|
||||
long dark = 0;
|
||||
for (int y = 0; y < image.getHeight(); y += step) {
|
||||
for (int x = 0; x < image.getWidth(); x += step) {
|
||||
sampled++;
|
||||
if ((image.getRGB(x, y) & 0xFF) < darkThreshold) {
|
||||
dark++;
|
||||
// A page needs a meaningful amount of ink before OSD can do anything;
|
||||
// bail out as soon as we know there is enough.
|
||||
if (dark > sampled / 1000 + 20) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clockwise /Rotate correction for a page whose dominant glyph direction (page space, CCW) is
|
||||
* {@code dominantDirection} and whose current /Rotate is {@code pageRotation}. Derivation: the
|
||||
* on-screen text angle is (direction - rotation) CCW, and adding d to /Rotate turns the display
|
||||
* a further d clockwise, so the correction that zeroes the screen angle is their difference.
|
||||
*/
|
||||
public static int correctionFromTextDirection(int dominantDirection, int pageRotation) {
|
||||
return Math.floorMod(dominantDirection - pageRotation, 360);
|
||||
}
|
||||
|
||||
/** Extracts rotation and confidence from `tesseract <img> stdout --psm 0` output. */
|
||||
public static Optional<OsdResult> parseOsd(String tesseractOutput) {
|
||||
if (tesseractOutput == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Matcher rotate = OSD_ROTATE.matcher(tesseractOutput);
|
||||
Matcher confidence = OSD_CONFIDENCE.matcher(tesseractOutput);
|
||||
if (!rotate.find() || !confidence.find()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
try {
|
||||
return Optional.of(
|
||||
new OsdResult(
|
||||
Integer.parseInt(rotate.group(1)),
|
||||
Double.parseDouble(confidence.group(1))));
|
||||
} catch (NumberFormatException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
private static class DirectionCountingStripper extends PDFTextStripper {
|
||||
|
||||
// counts[page][i] holds glyphs on that page whose direction is i * 90 degrees
|
||||
final int[][] counts;
|
||||
|
||||
DirectionCountingStripper(int pageCount) throws IOException {
|
||||
this.counts = new int[pageCount][4];
|
||||
}
|
||||
|
||||
/**
|
||||
* PDFBox snaps glyph direction to a quadrant, so getDir() only ever yields 0/90/180/270 —
|
||||
* obliquely drawn text (30, 45, 135 degrees) is reported as 0 rather than as its true
|
||||
* angle. Skew is therefore invisible to this signal by construction, which is consistent
|
||||
* with skew being out of scope here: only 90-degree orientation is corrected.
|
||||
*/
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
String unicode = text.getUnicode();
|
||||
if (unicode == null || unicode.isBlank()) {
|
||||
return;
|
||||
}
|
||||
int page = getCurrentPageNo() - 1;
|
||||
if (page < 0 || page >= counts.length) {
|
||||
return;
|
||||
}
|
||||
counts[page][Math.floorMod(Math.round(text.getDir()), 360) / 90]++;
|
||||
// super is intentionally not called: we only count, no text assembly needed
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -292,26 +292,6 @@ storage:
|
||||
linkExpirationDays: 3 # Number of days before share links expire
|
||||
signing:
|
||||
enabled: false # set to 'true' to enable group signing workflow (requires storage.enabled) [ALPHA]
|
||||
# ====================================================================================
|
||||
# ENCRYPTION AT REST - PRO / ENTERPRISE LICENSE REQUIRED TO ENABLE
|
||||
# ====================================================================================
|
||||
# Encrypts stored files (AES-256 envelope encryption, per-team keys). The master key is
|
||||
# resolved in this order:
|
||||
# 1. stirling.security.fileEncryptionKey property
|
||||
# 2. STIRLING_FILE_ENCRYPTION_KEY environment variable
|
||||
# 3. an auto-generated configs/file-encryption.key (single-node only; cluster mode
|
||||
# requires an explicitly shared key on every node)
|
||||
# Generate a key with: openssl rand -base64 32
|
||||
#
|
||||
# *** BACK UP THE MASTER KEY. Losing it makes every encrypted stored file ***
|
||||
# *** permanently unrecoverable. Verify backups against the key fingerprint logged ***
|
||||
# *** at startup. ***
|
||||
#
|
||||
# Enabling encrypts new writes only (existing files stay readable as plaintext).
|
||||
# Disabling later only stops encrypting new writes - existing encrypted files remain
|
||||
# readable as long as the key material is present.
|
||||
encryption:
|
||||
enabled: false # set to 'true' to encrypt stored files at rest
|
||||
userListScope: org # Signing user-picker scope: 'org' (default) = whole instance, else caller's team only.
|
||||
autoPipeline:
|
||||
outputFolder: "" # Output folder for processed pipeline files (leave empty for default)
|
||||
@@ -416,6 +396,14 @@ aiEngine:
|
||||
pdfComment: true # AI-authored PDF comments/annotations
|
||||
classify: true # Automatic document classification/labelling
|
||||
|
||||
# DocParse: document understanding for ingestion pipelines (chunking + knowledge-base
|
||||
# indexing). The basic tier (text layer) always works; the advanced tier (layout parsing)
|
||||
# requires the engine's docparse addon. Env overrides: DOCPARSE_ENABLED, DOCPARSE_MODE.
|
||||
docparse:
|
||||
enabled: true # Master switch; hides the DocParse endpoints when false
|
||||
mode: auto # Tier selection: 'auto' (best available), 'basic', or 'advanced'
|
||||
autoInstall: false # Mirrors DOCPARSE_AUTO_INSTALL for the engine's boot-time addon install script
|
||||
|
||||
policies:
|
||||
# Folder automations can read from and write to the directories you allow here, so treat this as a
|
||||
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs,
|
||||
|
||||
-342
@@ -1,342 +0,0 @@
|
||||
package stirling.software.SPDF.controller.api.misc;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import stirling.software.SPDF.config.EndpointConfiguration;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotateAnalysisResult;
|
||||
import stirling.software.SPDF.model.api.misc.AutoRotatePdfRequest;
|
||||
import stirling.software.SPDF.model.api.misc.PageRotation;
|
||||
import stirling.software.common.configuration.RuntimePathConfig;
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AutoRotateControllerTest {
|
||||
|
||||
private static final String SAMPLE_TEXT =
|
||||
"The quick brown fox jumps over the lazy dog again and again";
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private EndpointConfiguration endpointConfiguration;
|
||||
@Mock private RuntimePathConfig runtimePathConfig;
|
||||
@Mock private ApplicationProperties applicationProperties;
|
||||
|
||||
@InjectMocks private AutoRotateController controller;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient()
|
||||
.when(tempFileManager.createManagedTempFile(anyString()))
|
||||
.thenAnswer(
|
||||
inv -> {
|
||||
File f =
|
||||
Files.createTempFile("test", inv.<String>getArgument(0))
|
||||
.toFile();
|
||||
TempFile tf = mock(TempFile.class);
|
||||
lenient().when(tf.getFile()).thenReturn(f);
|
||||
lenient().when(tf.getPath()).thenReturn(f.toPath());
|
||||
return tf;
|
||||
});
|
||||
lenient().when(endpointConfiguration.isGroupEnabled("tesseract")).thenReturn(false);
|
||||
}
|
||||
|
||||
private static PDDocument docWithUprightText(int... pageRotations) throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
for (int rotation : pageRotations) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(72, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(rotation);
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
private AutoRotatePdfRequest request(PDDocument document) throws IOException {
|
||||
AutoRotatePdfRequest request = new AutoRotatePdfRequest();
|
||||
request.setFileInput(
|
||||
new MockMultipartFile(
|
||||
"fileInput",
|
||||
"test.pdf",
|
||||
MediaType.APPLICATION_PDF_VALUE,
|
||||
new byte[] {1, 2, 3}));
|
||||
when(pdfDocumentFactory.load(request)).thenReturn(document);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static PDDocument reload(ResponseEntity<?> response) throws IOException {
|
||||
Resource resource = (Resource) response.getBody();
|
||||
return Loader.loadPDF(resource.getContentAsByteArray());
|
||||
}
|
||||
|
||||
private static PDDocument docWithTextAt(int textAngleDegrees, int pageRotation)
|
||||
throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(textAngleDegrees), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(pageRotation);
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the dominant glyph direction straight from a document, independently of the production
|
||||
* detection code, so the round-trip assertion below validates the result rather than restating
|
||||
* the formula under test.
|
||||
*/
|
||||
private static int dominantGlyphDirection(PDDocument document) throws IOException {
|
||||
int[] counts = new int[4];
|
||||
PDFTextStripper stripper =
|
||||
new PDFTextStripper() {
|
||||
@Override
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
if (!text.getUnicode().isBlank()) {
|
||||
counts[Math.floorMod(Math.round(text.getDir()), 360) / 90]++;
|
||||
}
|
||||
}
|
||||
};
|
||||
stripper.setStartPage(1);
|
||||
stripper.setEndPage(1);
|
||||
stripper.getText(document);
|
||||
int best = 0;
|
||||
for (int i = 1; i < 4; i++) {
|
||||
if (counts[i] > counts[best]) {
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best * 90;
|
||||
}
|
||||
|
||||
/**
|
||||
* End-to-end round trip: build a page whose text is drawn at a known angle under a known
|
||||
* /Rotate, run the real controller, then assert the output actually displays upright. Upright
|
||||
* means the glyph direction and the page rotation cancel — computed here in the test, not via
|
||||
* the production helper.
|
||||
*/
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
"0, 0",
|
||||
"0, 90",
|
||||
"0, 180",
|
||||
"0, 270",
|
||||
"90, 0",
|
||||
"90, 90",
|
||||
"180, 0",
|
||||
"180, 270",
|
||||
"270, 90",
|
||||
})
|
||||
void roundTripLeavesPageUpright(int textAngle, int pageRotation) throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithTextAt(textAngle, pageRotation));
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
int glyphDirection = dominantGlyphDirection(corrected);
|
||||
int finalRotation = Math.floorMod(corrected.getPage(0).getRotation(), 360);
|
||||
assertThat(Math.floorMod(glyphDirection - finalRotation, 360))
|
||||
.as(
|
||||
"text drawn at %d under /Rotate %d should display upright, got glyph"
|
||||
+ " direction %d with /Rotate %d",
|
||||
textAngle, pageRotation, glyphDirection, finalRotation)
|
||||
.isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void dryRunReportsTextDetection() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(90, 0));
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
assertThat(result.getTotalPages()).isEqualTo(2);
|
||||
assertThat(result.getPagesToRotate()).isEqualTo(1);
|
||||
assertThat(result.getDetectedByText()).isEqualTo(2);
|
||||
|
||||
AutoRotateAnalysisResult.PageResult first = result.getPages().get(0);
|
||||
assertThat(first.getMethod()).isEqualTo("text");
|
||||
assertThat(first.getCorrection()).isEqualTo(270);
|
||||
assertThat(first.isApply()).isTrue();
|
||||
assertThat(first.getConfidence()).isEqualTo(100.0);
|
||||
|
||||
AutoRotateAnalysisResult.PageResult second = result.getPages().get(1);
|
||||
assertThat(second.getCorrection()).isZero();
|
||||
assertThat(second.isApply()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesDetectedCorrections() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(90, 0));
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
assertThat(corrected.getPage(0).getRotation()).isZero();
|
||||
assertThat(corrected.getPage(1).getRotation()).isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesExplicitPageRotations() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0, 0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 90)));
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
try (PDDocument corrected = reload(response)) {
|
||||
assertThat(corrected.getPage(0).getRotation()).isEqualTo(90);
|
||||
assertThat(corrected.getPage(1).getRotation()).isZero();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportsTesseractUnavailableForTextlessPages() throws Exception {
|
||||
PDDocument document = new PDDocument();
|
||||
document.addPage(new PDPage(PDRectangle.LETTER));
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page = result.getPages().get(0);
|
||||
assertThat(page.getMethod()).isEqualTo("none");
|
||||
assertThat(page.getNote()).isEqualTo("tesseractUnavailable");
|
||||
assertThat(result.getUndetected()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void infersUndetectedPageFromDocumentConsensus() throws Exception {
|
||||
// Page 1 has body text and is rotated 90 (-> 270 correction); page 2 is blank and shares
|
||||
// the same rotation. With OSD unavailable, page 2 can't be detected on its own, so it
|
||||
// should inherit page 1's 270 correction.
|
||||
PDDocument document = docWithUprightText(90);
|
||||
PDPage blank = new PDPage(PDRectangle.LETTER);
|
||||
blank.setRotation(90);
|
||||
document.addPage(blank);
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page2 = result.getPages().get(1);
|
||||
assertThat(page2.getMethod()).isEqualTo("inferred");
|
||||
assertThat(page2.getCorrection()).isEqualTo(270);
|
||||
assertThat(page2.isApply()).isTrue();
|
||||
assertThat(page2.getNote()).isEqualTo("inferredFromDocument");
|
||||
assertThat(result.getInferred()).isEqualTo(1);
|
||||
assertThat(result.getPagesToRotate()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotInferWhenDisabled() throws Exception {
|
||||
PDDocument document = docWithUprightText(90);
|
||||
PDPage blank = new PDPage(PDRectangle.LETTER);
|
||||
blank.setRotation(90);
|
||||
document.addPage(blank);
|
||||
AutoRotatePdfRequest request = request(document);
|
||||
request.setDryRun(true);
|
||||
request.setDetectionMode("text");
|
||||
request.setInferUndetected(false);
|
||||
|
||||
ResponseEntity<?> response = controller.autoRotatePdf(request);
|
||||
|
||||
AutoRotateAnalysisResult result = (AutoRotateAnalysisResult) response.getBody();
|
||||
AutoRotateAnalysisResult.PageResult page2 = result.getPages().get(1);
|
||||
assertThat(page2.getMethod()).isEqualTo("none");
|
||||
assertThat(page2.isApply()).isFalse();
|
||||
assertThat(result.getInferred()).isZero();
|
||||
assertThat(result.getUndetected()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsInvalidDetectionMode() {
|
||||
AutoRotatePdfRequest request = new AutoRotatePdfRequest();
|
||||
request.setDetectionMode("magic");
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsRotationThatIsNotAMultipleOf90() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 45)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsPageRotationOutsideTheDocument() throws Exception {
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0));
|
||||
request.setPageRotations(List.of(new PageRotation(5, 90)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsDuplicatePageInRotations() throws Exception {
|
||||
// Rotations are additive, so applying the same page twice would over-rotate it.
|
||||
AutoRotatePdfRequest request = request(docWithUprightText(0, 0));
|
||||
request.setPageRotations(List.of(new PageRotation(1, 90), new PageRotation(1, 90)));
|
||||
|
||||
assertThatThrownBy(() -> controller.autoRotatePdf(request))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -74,7 +74,8 @@ class ConfigControllerMoreTest {
|
||||
userService,
|
||||
showAdmin,
|
||||
licenseService,
|
||||
externalAppDepConfig);
|
||||
externalAppDepConfig,
|
||||
null);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+2
-1
@@ -52,7 +52,8 @@ class ConfigControllerTest {
|
||||
userService,
|
||||
showAdmin,
|
||||
licenseService,
|
||||
mock(stirling.software.SPDF.config.ExternalAppDepConfig.class));
|
||||
mock(stirling.software.SPDF.config.ExternalAppDepConfig.class),
|
||||
null);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
-22
@@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
@@ -292,27 +291,6 @@ class ValidateSignatureControllerMoreTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Padded signature contents")
|
||||
class PaddedContentsTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("Zero-padded /Contents still parses as CMS")
|
||||
void zeroPaddedContentsParsesAsCms() throws Exception {
|
||||
byte[] contents;
|
||||
try (PDDocument doc = Loader.loadPDF(signedPdfBytes)) {
|
||||
PDSignature sig = doc.getSignatureDictionaries().get(0);
|
||||
contents = sig.getContents(new ByteArrayInputStream(signedPdfBytes));
|
||||
}
|
||||
|
||||
// /Contents is zero-filled up to its reserved length. Since BC 1.85 the byte[]
|
||||
// constructors reject that tail ("Extra data detected in stream"), streams do not.
|
||||
assertThat(contents[contents.length - 1]).isZero();
|
||||
CMSSignedData parsed = new CMSSignedData(new ByteArrayInputStream(contents));
|
||||
assertThat(parsed.getSignerInfos().size()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Error and edge handling")
|
||||
class ErrorHandlingTests {
|
||||
|
||||
-197
@@ -1,197 +0,0 @@
|
||||
package stirling.software.SPDF.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.verapdf.gf.foundry.VeraGreenfieldFoundryProvider;
|
||||
import org.verapdf.pdfa.Foundries;
|
||||
import org.verapdf.pdfa.PDFAParser;
|
||||
import org.verapdf.pdfa.PDFAValidator;
|
||||
import org.verapdf.pdfa.flavours.PDFAFlavour;
|
||||
import org.verapdf.pdfa.results.ValidationResult;
|
||||
|
||||
import stirling.software.SPDF.model.api.security.PDFVerificationResult;
|
||||
|
||||
/**
|
||||
* Exercises {@link VeraPDFService} against real PDF/A files. Fixtures were produced by Ghostscript
|
||||
* with the same flags as ConvertPDFToPDFA and independently confirmed conformant by veraPDF.
|
||||
*/
|
||||
class VeraPDFServicePdfaFixtureTest {
|
||||
|
||||
private static final String VALID_1B = "valid-pdfa-1b.pdf";
|
||||
private static final String VALID_2B = "valid-pdfa-2b.pdf";
|
||||
private static final String DECLARED_BUT_INVALID_1B = "declared-pdfa-1b-no-outputintent.pdf";
|
||||
|
||||
private VeraPDFService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new VeraPDFService();
|
||||
service.initialize();
|
||||
}
|
||||
|
||||
@Test
|
||||
void fixtures_areGenuinePdfaAccordingToVeraPdfItself() throws Exception {
|
||||
assertVeraPdfVerdict(VALID_1B, PDFAFlavour.PDFA_1_B, true);
|
||||
assertVeraPdfVerdict(VALID_2B, PDFAFlavour.PDFA_2_B, true);
|
||||
assertVeraPdfVerdict(DECLARED_BUT_INVALID_1B, PDFAFlavour.PDFA_1_B, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_realPdfa1b_reportsCompliantPdfa1b() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(VALID_1B));
|
||||
|
||||
assertEquals("1b", result.getStandard());
|
||||
assertEquals("1b", result.getValidationProfile());
|
||||
assertTrue(result.isDeclaredPdfa(), "Genuine PDF/A-1b must be reported as declared PDF/A");
|
||||
assertTrue(result.isCompliant(), "Genuine PDF/A-1b must validate as compliant");
|
||||
assertEquals(
|
||||
0, result.getTotalFailures(), () -> "Unexpected failures: " + messages(result));
|
||||
assertTrue(
|
||||
result.getStandardName().startsWith("PDF/A-"),
|
||||
"Display name should name the PDF/A standard, got: " + result.getStandardName());
|
||||
assertTrue(
|
||||
result.getStandardName().endsWith(" compliant"),
|
||||
"Display name should read as compliant, got: " + result.getStandardName());
|
||||
assertEquals(result.getStandardName(), result.getComplianceSummary());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_realPdfa2b_reportsPdfa2bAndNotPdfa1b() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(VALID_2B));
|
||||
|
||||
// Proves firstFlavour() returns the flavour actually declared, not just a non-null one
|
||||
assertEquals("2b", result.getStandard());
|
||||
assertEquals("2b", result.getValidationProfile());
|
||||
assertNotEquals("1b", result.getStandard());
|
||||
assertTrue(result.isDeclaredPdfa());
|
||||
assertTrue(result.isCompliant(), () -> "Unexpected failures: " + messages(result));
|
||||
assertEquals(0, result.getTotalFailures());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_plainPdf_reportsNotPdfaAndDoesNotThrowIndexOutOfBounds() throws Exception {
|
||||
byte[] pdfBytes = createSimplePdf();
|
||||
|
||||
// veraPDF 1.30 returns an empty flavour list here where 1.28 returned [1b]; get(0) threw
|
||||
List<PDFVerificationResult> results =
|
||||
assertDoesNotThrow(
|
||||
() -> service.validatePDF(new ByteArrayInputStream(pdfBytes)),
|
||||
"Empty veraPDF flavour list must not surface as IndexOutOfBoundsException");
|
||||
|
||||
assertEquals(1, results.size());
|
||||
PDFVerificationResult result = results.get(0);
|
||||
assertEquals("not-pdfa", result.getStandard());
|
||||
assertFalse(result.isDeclaredPdfa());
|
||||
assertFalse(result.isCompliant());
|
||||
assertEquals("Not PDF/A (no PDF/A identification metadata)", result.getStandardName());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePDF_declaresPdfaButNotConformant_reportsFlavourWithFailures() throws Exception {
|
||||
PDFVerificationResult result = onlyResult(fixture(DECLARED_BUT_INVALID_1B));
|
||||
|
||||
// "declares PDF/A but broken" must stay distinct from "not PDF/A at all"
|
||||
assertEquals("1b", result.getStandard());
|
||||
assertNotEquals("not-pdfa", result.getStandard());
|
||||
assertTrue(result.isDeclaredPdfa(), "XMP still declares pdfaid:part=1");
|
||||
assertFalse(result.isCompliant(), "Stripped OutputIntent must fail conformance");
|
||||
assertTrue(result.getTotalFailures() > 0, "Non-conformance must be reported as issues");
|
||||
assertTrue(
|
||||
result.getStandardName().endsWith(" with errors"),
|
||||
"Display name should flag errors, got: " + result.getStandardName());
|
||||
assertTrue(
|
||||
messages(result).contains("OutputIntent"),
|
||||
"Expected the missing OutputIntent to be reported, got: " + messages(result));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firstFlavour_withEmptyList_returnsNullInsteadOfThrowing() throws Exception {
|
||||
Method method = VeraPDFService.class.getDeclaredMethod("firstFlavour", List.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
assertNull(method.invoke(null, List.of()));
|
||||
assertEquals(PDFAFlavour.PDFA_2_B, method.invoke(null, List.of(PDFAFlavour.PDFA_2_B)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectedFlavours_withNullFlavourList_returnsEmptyList() throws Exception {
|
||||
Method method =
|
||||
VeraPDFService.class.getDeclaredMethod("detectedFlavours", PDFAParser.class);
|
||||
method.setAccessible(true);
|
||||
|
||||
PDFAParser parser = mock(PDFAParser.class);
|
||||
when(parser.getFlavours()).thenReturn(null);
|
||||
|
||||
assertEquals(List.of(), method.invoke(null, parser));
|
||||
}
|
||||
|
||||
private static void assertVeraPdfVerdict(
|
||||
String fixtureName, PDFAFlavour expectedFlavour, boolean expectedCompliant)
|
||||
throws Exception {
|
||||
VeraGreenfieldFoundryProvider.initialise();
|
||||
byte[] bytes = fixture(fixtureName);
|
||||
|
||||
List<PDFAFlavour> flavours;
|
||||
try (PDFAParser parser =
|
||||
Foundries.defaultInstance().createParser(new ByteArrayInputStream(bytes))) {
|
||||
flavours = parser.getFlavours();
|
||||
}
|
||||
assertEquals(List.of(expectedFlavour), flavours, fixtureName + " declared flavours");
|
||||
|
||||
try (PDFAParser parser =
|
||||
Foundries.defaultInstance()
|
||||
.createParser(new ByteArrayInputStream(bytes), expectedFlavour)) {
|
||||
PDFAValidator validator =
|
||||
Foundries.defaultInstance().createValidator(expectedFlavour, false);
|
||||
ValidationResult result = validator.validate(parser);
|
||||
assertEquals(
|
||||
expectedCompliant, result.isCompliant(), fixtureName + " veraPDF compliance");
|
||||
}
|
||||
}
|
||||
|
||||
private PDFVerificationResult onlyResult(byte[] pdfBytes) throws Exception {
|
||||
List<PDFVerificationResult> results =
|
||||
service.validatePDF(new ByteArrayInputStream(pdfBytes));
|
||||
|
||||
assertNotNull(results);
|
||||
assertEquals(1, results.size(), () -> "Expected a single result, got: " + results);
|
||||
return results.get(0);
|
||||
}
|
||||
|
||||
private static String messages(PDFVerificationResult result) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (PDFVerificationResult.ValidationIssue issue : result.getFailures()) {
|
||||
builder.append(issue.getMessage()).append(" | ");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private static byte[] fixture(String name) throws IOException {
|
||||
try (InputStream in =
|
||||
VeraPDFServicePdfaFixtureTest.class.getResourceAsStream("/pdfa/" + name)) {
|
||||
assertNotNull(in, "Missing test fixture /pdfa/" + name);
|
||||
return in.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] createSimplePdf() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
document.addPage(new PDPage());
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
document.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package stirling.software.SPDF.utils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.OsdResult;
|
||||
import stirling.software.SPDF.utils.AutoRotateDetection.TextDirection;
|
||||
|
||||
class AutoRotateDetectionTest {
|
||||
|
||||
private static final String SAMPLE_TEXT =
|
||||
"The quick brown fox jumps over the lazy dog again and again";
|
||||
|
||||
private PDDocument docWithText(int textAngleDegrees, int pageRotation) throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(textAngleDegrees), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
page.setRotation(pageRotation);
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ground truth per the PDF spec: /Rotate R displays the page R degrees clockwise, so upright
|
||||
* page-space text under /Rotate R needs a further (360 - R) % 360 to display upright again;
|
||||
* text drawn rotated T degrees CCW in page space needs T clockwise to correct. Combined, the
|
||||
* expected correction is (T - R) mod 360.
|
||||
*/
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
// textAngle, pageRotation, expectedCorrection
|
||||
"0, 0, 0",
|
||||
"0, 90, 270",
|
||||
"0, 180, 180",
|
||||
"0, 270, 90",
|
||||
"90, 0, 90",
|
||||
"180, 0, 180",
|
||||
"270, 0, 270",
|
||||
"90, 90, 0",
|
||||
"180, 90, 90",
|
||||
})
|
||||
void detectsCorrectionForRotatedTextAndPages(
|
||||
int textAngle, int pageRotation, int expectedCorrection) throws IOException {
|
||||
try (PDDocument document = docWithText(textAngle, pageRotation)) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
|
||||
assertThat(direction.isConclusive())
|
||||
.as(
|
||||
"direction should be conclusive, glyphs=%d dominance=%s",
|
||||
direction.glyphCount(), direction.dominance())
|
||||
.isTrue();
|
||||
assertThat(
|
||||
AutoRotateDetection.correctionFromTextDirection(
|
||||
direction.dominantDirection(),
|
||||
Math.floorMod(pageRotation, 360)))
|
||||
.isEqualTo(expectedCorrection);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedDirectionsAreNotConclusive() throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(100, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(90), 300, 200));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyPageIsNotConclusive() throws IOException {
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
document.addPage(new PDPage(PDRectangle.LETTER));
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.glyphCount()).isZero();
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shortTextIsNotConclusive() throws IOException {
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getTranslateInstance(100, 400));
|
||||
content.showText("Short");
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.isConclusive()).isFalse();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unanimousShortTextIsConclusive() throws IOException {
|
||||
// Between MIN_GLYPHS_UNANIMOUS (8) and MIN_GLYPHS (30): trusted only because every
|
||||
// glyph agrees on direction, the sparse-page path (e.g. a lone header or URL line).
|
||||
PDDocument document = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(90), 300, 200));
|
||||
content.showText("york.gov.uk/pay");
|
||||
content.endText();
|
||||
}
|
||||
try (document) {
|
||||
TextDirection direction = AutoRotateDetection.detectTextDirections(document).get(0);
|
||||
assertThat(direction.glyphCount())
|
||||
.isBetween(
|
||||
AutoRotateDetection.MIN_GLYPHS_UNANIMOUS,
|
||||
AutoRotateDetection.MIN_GLYPHS - 1);
|
||||
assertThat(direction.dominance()).isEqualTo(1.0);
|
||||
assertThat(direction.isConclusive()).isTrue();
|
||||
assertThat(direction.dominantDirection()).isEqualTo(90);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void bucketsGlyphsPerPageInOneWalk() throws IOException {
|
||||
// Each page carries text at a different angle; the single-pass walk must attribute
|
||||
// glyphs to the right page rather than pooling them.
|
||||
int[] angles = {0, 90, 180, 270};
|
||||
try (PDDocument document = new PDDocument()) {
|
||||
for (int angle : angles) {
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
document.addPage(page);
|
||||
try (PDPageContentStream content = new PDPageContentStream(document, page)) {
|
||||
content.beginText();
|
||||
content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12);
|
||||
content.setTextMatrix(
|
||||
Matrix.getRotateInstance(Math.toRadians(angle), 300, 400));
|
||||
content.showText(SAMPLE_TEXT);
|
||||
content.endText();
|
||||
}
|
||||
}
|
||||
|
||||
List<TextDirection> directions = AutoRotateDetection.detectTextDirections(document);
|
||||
|
||||
assertThat(directions).hasSize(angles.length);
|
||||
for (int i = 0; i < angles.length; i++) {
|
||||
assertThat(directions.get(i).isConclusive()).as("page %d", i + 1).isTrue();
|
||||
assertThat(directions.get(i).dominantDirection())
|
||||
.as("page %d direction", i + 1)
|
||||
.isEqualTo(angles[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsBlankAndInkedRenders() {
|
||||
BufferedImage blank = new BufferedImage(200, 200, BufferedImage.TYPE_BYTE_GRAY);
|
||||
Graphics2D g = blank.createGraphics();
|
||||
g.setColor(Color.WHITE);
|
||||
g.fillRect(0, 0, 200, 200);
|
||||
g.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(blank)).isTrue();
|
||||
|
||||
BufferedImage speck = copyOf(blank);
|
||||
Graphics2D specked = speck.createGraphics();
|
||||
specked.setColor(Color.BLACK);
|
||||
specked.fillRect(0, 0, 2, 2); // a dust speck must not count as content
|
||||
specked.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(speck)).isTrue();
|
||||
|
||||
BufferedImage inked = copyOf(blank);
|
||||
Graphics2D inkedG = inked.createGraphics();
|
||||
inkedG.setColor(Color.BLACK);
|
||||
inkedG.fillRect(20, 20, 120, 60);
|
||||
inkedG.dispose();
|
||||
assertThat(AutoRotateDetection.isBlankRender(inked)).isFalse();
|
||||
}
|
||||
|
||||
private static BufferedImage copyOf(BufferedImage source) {
|
||||
BufferedImage copy =
|
||||
new BufferedImage(source.getWidth(), source.getHeight(), source.getType());
|
||||
Graphics2D g = copy.createGraphics();
|
||||
g.drawImage(source, 0, 0, null);
|
||||
g.dispose();
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesTypicalOsdOutput() {
|
||||
String output =
|
||||
"""
|
||||
Estimating resolution as 336
|
||||
Page number: 0
|
||||
Orientation in degrees: 180
|
||||
Rotate: 180
|
||||
Orientation confidence: 9.15
|
||||
Script: Latin
|
||||
Script confidence: 4.43
|
||||
""";
|
||||
Optional<OsdResult> result = AutoRotateDetection.parseOsd(output);
|
||||
assertThat(result).isPresent();
|
||||
assertThat(result.get().rotate()).isEqualTo(180);
|
||||
assertThat(result.get().confidence()).isEqualTo(9.15);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseOsdRejectsIncompleteOutput() {
|
||||
assertThat(AutoRotateDetection.parseOsd("Too few characters. Skipping this page"))
|
||||
.isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd("Rotate: 90")).isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd(null)).isEmpty();
|
||||
assertThat(AutoRotateDetection.parseOsd("")).isEmpty();
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -67,9 +67,6 @@ dependencies {
|
||||
implementation "software.amazon.awssdk:s3:${awsSdkVersion}"
|
||||
implementation "software.amazon.awssdk:url-connection-client:${awsSdkVersion}"
|
||||
|
||||
// Streaming AEAD (AES-GCM-HKDF segments) for storage encryption at rest. Apache-2.0.
|
||||
implementation "com.google.crypto.tink:tink:${tinkVersion}"
|
||||
|
||||
// @DataJpaTest slice (Boot 4 ships test slices as separate starters, like webmvc-test at the
|
||||
// root) so policy.source repositories can be exercised against embedded H2.
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test'
|
||||
|
||||
+532
@@ -0,0 +1,532 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ModelAttribute;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.enumeration.ResourceWeight;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.FormUtils;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.proprietary.model.api.docparse.ChunkDocumentApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.ExtractFieldsApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.ExtractTablesApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.ParseDocumentApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.RagAskApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.RagIngestApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.RagSearchApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.SmartSplitApiRequest;
|
||||
import stirling.software.proprietary.model.api.docparse.SuggestSchemaApiRequest;
|
||||
import stirling.software.proprietary.model.docparse.ChunkDocumentResponse;
|
||||
import stirling.software.proprietary.model.docparse.DocChunk;
|
||||
import stirling.software.proprietary.model.docparse.DocTable;
|
||||
import stirling.software.proprietary.model.docparse.DocparseCapabilitiesView;
|
||||
import stirling.software.proprietary.model.docparse.DocparseMode;
|
||||
import stirling.software.proprietary.model.docparse.ExtractFieldsResponse;
|
||||
import stirling.software.proprietary.model.docparse.ExtractTablesResponse;
|
||||
import stirling.software.proprietary.model.docparse.FillDocxResponse;
|
||||
import stirling.software.proprietary.model.docparse.ParseDocumentResponse;
|
||||
import stirling.software.proprietary.model.docparse.RagIngestResponse;
|
||||
import stirling.software.proprietary.model.docparse.RagStatsView;
|
||||
import stirling.software.proprietary.model.docparse.SmartSplitResponse;
|
||||
import stirling.software.proprietary.model.docparse.SplitPart;
|
||||
import stirling.software.proprietary.model.docparse.SuggestSchemaResponse;
|
||||
import stirling.software.proprietary.service.AiToolResponseHeaders;
|
||||
import stirling.software.proprietary.service.DocParseService;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.node.ObjectNode;
|
||||
|
||||
/**
|
||||
* Public DocParse ingestion API. Thin HTTP layer over {@link DocParseService}, which owns the
|
||||
* engine wire contract; this class owns the pipeline step shape (report header, export ZIP).
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/docparse")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(
|
||||
name = "DocParse",
|
||||
description =
|
||||
"Document ingestion: chunk, embed, and index documents into the searchable"
|
||||
+ " knowledge base, or export the parsed content (markdown, chunks JSONL)"
|
||||
+ " for external systems.")
|
||||
public class DocParseController {
|
||||
|
||||
private static final MediaType CSV = MediaType.parseMediaType("text/csv");
|
||||
|
||||
private static final MediaType MARKDOWN = MediaType.parseMediaType("text/markdown");
|
||||
private static final MediaType DOCX =
|
||||
MediaType.parseMediaType(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document");
|
||||
|
||||
private final DocParseService docParseService;
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/rag-ingest",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Chunk, embed, and index a document into the RAG store (pipeline shape)",
|
||||
description =
|
||||
"Ingests the document into the engine's RAG store under a stable documentId"
|
||||
+ " (default: content hash). Returns the ORIGINAL PDF unchanged as the"
|
||||
+ " body, with the ingest summary JSON in the X-Stirling-Tool-Report"
|
||||
+ " header so policy pipelines pick it up as the step report. With"
|
||||
+ " exportMarkdown/exportChunksJsonl the body becomes a ZIP holding the"
|
||||
+ " original plus the corpus files, ready for delivery to external"
|
||||
+ " systems. Input:PDF Output:PDF/ZIP Type:SISO")
|
||||
public ResponseEntity<Resource> ragIngest(@ModelAttribute RagIngestApiRequest request)
|
||||
throws IOException {
|
||||
MultipartFile file = request.getFileInput();
|
||||
boolean export = request.isExportMarkdown() || request.isExportChunksJsonl();
|
||||
RagIngestResponse result =
|
||||
docParseService.ragIngest(
|
||||
file,
|
||||
request.getDocumentId(),
|
||||
request.getChunkSize(),
|
||||
request.getOverlap(),
|
||||
DocparseMode.fromWire(request.getMode()),
|
||||
request.isIndex(),
|
||||
request.isExportMarkdown(),
|
||||
request.isExportChunksJsonl());
|
||||
|
||||
// The report header must stay small: summary fields only, never the echoed content.
|
||||
ObjectNode report = objectMapper.createObjectNode();
|
||||
report.put("mode", result.mode().wire());
|
||||
report.put("documentId", result.documentId());
|
||||
report.put("chunksIndexed", result.chunksIndexed());
|
||||
report.put("pages", result.pages());
|
||||
report.put("indexed", request.isIndex());
|
||||
|
||||
String fileName = DocParseService.fileName(file);
|
||||
byte[] original = file.getBytes();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set(AiToolResponseHeaders.TOOL_REPORT, objectMapper.writeValueAsString(report));
|
||||
|
||||
if (!export) {
|
||||
headers.setContentType(MediaType.APPLICATION_PDF);
|
||||
headers.setContentDispositionFormData("attachment", fileName);
|
||||
headers.setContentLength(original.length);
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(original));
|
||||
}
|
||||
|
||||
byte[] zip = exportZip(fileName, original, result, request);
|
||||
headers.setContentType(MediaType.parseMediaType("application/zip"));
|
||||
headers.setContentDispositionFormData("attachment", baseName(fileName) + "-ingested.zip");
|
||||
headers.setContentLength(zip.length);
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(zip));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/extract-fields",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Extract typed fields from a document (pipeline shape)",
|
||||
description =
|
||||
"Extracts the fields described by the JSON Schema and returns the ORIGINAL PDF"
|
||||
+ " unchanged as the body, with the extraction JSON in the"
|
||||
+ " X-Stirling-Tool-Report header so policy pipelines pick it up as the"
|
||||
+ " step report. Use /extract-fields/json for the raw JSON."
|
||||
+ " Input:PDF Output:PDF Type:SISO")
|
||||
public ResponseEntity<Resource> extractFields(@ModelAttribute ExtractFieldsApiRequest request)
|
||||
throws IOException {
|
||||
MultipartFile file = request.getFileInput();
|
||||
ExtractFieldsResponse result =
|
||||
docParseService.extractFields(
|
||||
file,
|
||||
request.getFieldsSchema(),
|
||||
DocparseMode.fromWire(request.getMode()),
|
||||
request.getInstructions());
|
||||
byte[] original = file.getBytes();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_PDF);
|
||||
headers.setContentDispositionFormData("attachment", DocParseService.fileName(file));
|
||||
headers.setContentLength(original.length);
|
||||
headers.set(AiToolResponseHeaders.TOOL_REPORT, objectMapper.writeValueAsString(result));
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(original));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/extract-fields/json",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Extract typed fields from a document (JSON)",
|
||||
description =
|
||||
"Extracts the fields described by the JSON Schema and returns the extraction"
|
||||
+ " result (fields, confidence, citations) as JSON."
|
||||
+ " Input:PDF Output:JSON Type:SISO")
|
||||
public ResponseEntity<ExtractFieldsResponse> extractFieldsJson(
|
||||
@ModelAttribute ExtractFieldsApiRequest request) throws IOException {
|
||||
return ResponseEntity.ok(
|
||||
docParseService.extractFields(
|
||||
request.getFileInput(),
|
||||
request.getFieldsSchema(),
|
||||
DocparseMode.fromWire(request.getMode()),
|
||||
request.getInstructions()));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/suggest-schema",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Suggest an extraction schema for a document",
|
||||
description =
|
||||
"Reads the document and proposes the fields worth extracting (name, type,"
|
||||
+ " description), ready to feed into /extract-fields as a JSON Schema."
|
||||
+ " Input:PDF Output:JSON Type:SISO")
|
||||
public ResponseEntity<SuggestSchemaResponse> suggestSchema(
|
||||
@ModelAttribute SuggestSchemaApiRequest request) throws IOException {
|
||||
return ResponseEntity.ok(
|
||||
docParseService.suggestSchema(request.getFileInput(), request.getMaxFields()));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/parse-document",
|
||||
resourceWeight = ResourceWeight.XLARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Parse a document into structured blocks, tables, and markdown",
|
||||
description =
|
||||
"Parses the PDF into layout blocks, tables, and a markdown rendering. The"
|
||||
+ " basic tier reads the text layer; the advanced tier (docparse addon)"
|
||||
+ " adds OCR, real table structure, and bounding boxes."
|
||||
+ " Input:PDF Output:JSON Type:SISO")
|
||||
public ResponseEntity<?> parseDocument(@ModelAttribute ParseDocumentApiRequest request)
|
||||
throws IOException {
|
||||
ParseDocumentResponse result =
|
||||
docParseService.parse(
|
||||
request.getFileInput(),
|
||||
DocparseMode.fromWire(request.getMode()),
|
||||
request.isWithOcr());
|
||||
if ("markdown".equalsIgnoreCase(request.getOutputFormat())) {
|
||||
return WebResponseUtils.bytesToWebResponse(
|
||||
result.markdown().getBytes(StandardCharsets.UTF_8),
|
||||
outputName(request.getFileInput(), "_parsed.md"),
|
||||
MARKDOWN);
|
||||
}
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/smart-split",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Split a document at content-derived boundaries",
|
||||
description =
|
||||
"Asks the engine where sub-documents start (per the natural-language rule) and"
|
||||
+ " returns a ZIP with one PDF per part, named from the part labels."
|
||||
+ " Input:PDF Output:ZIP-PDF Type:SIMO")
|
||||
public ResponseEntity<Resource> smartSplit(@ModelAttribute SmartSplitApiRequest request)
|
||||
throws IOException {
|
||||
MultipartFile file = request.getFileInput();
|
||||
SmartSplitResponse split =
|
||||
docParseService.split(file, request.getRule(), request.getMaxParts());
|
||||
if (split.parts().isEmpty()) {
|
||||
throw new ResponseStatusException(
|
||||
HttpStatus.UNPROCESSABLE_ENTITY,
|
||||
"The split rule produced no parts for this document");
|
||||
}
|
||||
TempFile zipTempFile = tempFileManager.createManagedTempFile(".zip");
|
||||
try {
|
||||
try (TempFile sourceTempFile = new TempFile(tempFileManager, ".pdf")) {
|
||||
Files.copy(
|
||||
file.getInputStream(),
|
||||
sourceTempFile.getPath(),
|
||||
StandardCopyOption.REPLACE_EXISTING);
|
||||
try (ZipOutputStream zipOut =
|
||||
new ZipOutputStream(Files.newOutputStream(zipTempFile.getPath()))) {
|
||||
writeParts(sourceTempFile, split.parts(), zipOut);
|
||||
}
|
||||
}
|
||||
return WebResponseUtils.zipFileToWebResponse(
|
||||
zipTempFile,
|
||||
GeneralUtils.generateFilename(file.getOriginalFilename(), "_split.zip"));
|
||||
} catch (Exception e) {
|
||||
zipTempFile.close();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/chunk-document",
|
||||
resourceWeight = ResourceWeight.MEDIUM_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Chunk a document for RAG",
|
||||
description =
|
||||
"Splits the document text into overlapping chunks with page spans and (advanced"
|
||||
+ " tier) heading breadcrumbs. Input:PDF Output:JSON Type:SISO")
|
||||
public ResponseEntity<ChunkDocumentResponse> chunkDocument(
|
||||
@ModelAttribute ChunkDocumentApiRequest request) throws IOException {
|
||||
return ResponseEntity.ok(
|
||||
docParseService.chunk(
|
||||
request.getFileInput(),
|
||||
request.getChunkSize(),
|
||||
request.getOverlap(),
|
||||
DocparseMode.fromWire(request.getMode())));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/fill-template",
|
||||
resourceWeight = ResourceWeight.SMALL_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Fill a DOCX template with JSON data",
|
||||
description =
|
||||
"Replaces the template's placeholders with values from the JSON object and"
|
||||
+ " returns the filled DOCX. Replacement counts and missing keys ride"
|
||||
+ " the X-Stirling-Tool-Report header."
|
||||
+ " Input:DOCX Output:DOCX Type:SISO")
|
||||
public ResponseEntity<Resource> fillTemplate(
|
||||
@RequestParam("templateFile") MultipartFile templateFile,
|
||||
@RequestParam("data") String data)
|
||||
throws IOException {
|
||||
FillDocxResponse result = docParseService.fillDocx(templateFile, data);
|
||||
byte[] filled = Base64.getDecoder().decode(result.docxBase64());
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(DOCX);
|
||||
headers.setContentDispositionFormData(
|
||||
"attachment",
|
||||
GeneralUtils.generateFilename(templateFile.getOriginalFilename(), "_filled.docx"));
|
||||
headers.setContentLength(filled.length);
|
||||
headers.set(
|
||||
AiToolResponseHeaders.TOOL_REPORT,
|
||||
objectMapper.writeValueAsString(
|
||||
new FillDocxResponse("", result.replaced(), result.missing())));
|
||||
return ResponseEntity.ok().headers(headers).body(new ByteArrayResource(filled));
|
||||
}
|
||||
|
||||
@GetMapping("/capabilities")
|
||||
@Operation(
|
||||
summary = "DocParse capability summary",
|
||||
description =
|
||||
"Merged view of the Java settings and the engine's capability probe, so"
|
||||
+ " clients can gate advanced-tier UI.")
|
||||
public ResponseEntity<DocparseCapabilitiesView> capabilities() {
|
||||
return ResponseEntity.ok(docParseService.capabilitiesView());
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
|
||||
value = "/extract-tables",
|
||||
resourceWeight = ResourceWeight.LARGE_WEIGHT)
|
||||
@Operation(
|
||||
summary = "Extract tables from a document",
|
||||
description =
|
||||
"Extracts table structure and returns CSV (all tables concatenated, blank line"
|
||||
+ " between them) or the structured JSON table list."
|
||||
+ " Input:PDF Output:CSV/JSON Type:SISO")
|
||||
public ResponseEntity<?> extractTables(@ModelAttribute ExtractTablesApiRequest request)
|
||||
throws IOException {
|
||||
ExtractTablesResponse result = docParseService.tables(request.getFileInput());
|
||||
if ("json".equalsIgnoreCase(request.getOutputFormat())) {
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
return WebResponseUtils.bytesToWebResponse(
|
||||
tablesToCsv(result.tables()).getBytes(StandardCharsets.UTF_8),
|
||||
outputName(request.getFileInput(), "_tables.csv"),
|
||||
CSV);
|
||||
}
|
||||
|
||||
@GetMapping("/rag-stats")
|
||||
@Operation(
|
||||
summary = "RAG store statistics",
|
||||
description =
|
||||
"The engine's document-store totals (backend, documents, chunks, embedding"
|
||||
+ " model) merged with the DocParse capability fields. Answers with"
|
||||
+ " zeros and engineReachable=false when the engine is down.")
|
||||
public ResponseEntity<RagStatsView> ragStats() {
|
||||
return ResponseEntity.ok(docParseService.ragStats());
|
||||
}
|
||||
|
||||
@GetMapping("/rag-documents")
|
||||
@Operation(
|
||||
summary = "List documents in the RAG store",
|
||||
description =
|
||||
"Engine passthrough of the caller-visible indexed documents (documentId,"
|
||||
+ " source, chunk count).")
|
||||
public ResponseEntity<String> ragDocuments() throws IOException {
|
||||
return jsonPassthrough(docParseService.ragDocuments());
|
||||
}
|
||||
|
||||
@PostMapping(value = "/rag-search", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Semantic search over the RAG store",
|
||||
description =
|
||||
"Searches the caller-visible indexed documents and returns the top passages"
|
||||
+ " with scores, page spans, and heading breadcrumbs.")
|
||||
public ResponseEntity<String> ragSearch(@RequestBody RagSearchApiRequest request)
|
||||
throws IOException {
|
||||
return jsonPassthrough(docParseService.ragSearch(request.getQuery(), request.getTopK()));
|
||||
}
|
||||
|
||||
@PostMapping(value = "/rag-ask", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
@Operation(
|
||||
summary = "Ask a question over the RAG store",
|
||||
description =
|
||||
"Answers the question from the caller-visible indexed documents and returns"
|
||||
+ " the answer with its supporting passages.")
|
||||
public ResponseEntity<String> ragAsk(@RequestBody RagAskApiRequest request) throws IOException {
|
||||
return jsonPassthrough(docParseService.ragAsk(request.getQuestion(), request.getTopK()));
|
||||
}
|
||||
|
||||
private static ResponseEntity<String> jsonPassthrough(String engineJson) {
|
||||
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(engineJson);
|
||||
}
|
||||
|
||||
/** Original + requested corpus files in one ZIP, so destinations receive them together. */
|
||||
private byte[] exportZip(
|
||||
String fileName, byte[] original, RagIngestResponse result, RagIngestApiRequest request)
|
||||
throws IOException {
|
||||
String base = baseName(fileName);
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
try (ZipOutputStream zip = new ZipOutputStream(out)) {
|
||||
zip.putNextEntry(new ZipEntry(fileName));
|
||||
zip.write(original);
|
||||
zip.closeEntry();
|
||||
if (request.isExportMarkdown()) {
|
||||
zip.putNextEntry(new ZipEntry(base + ".md"));
|
||||
zip.write(
|
||||
(result.markdown() == null ? "" : result.markdown())
|
||||
.getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
if (request.isExportChunksJsonl()) {
|
||||
zip.putNextEntry(new ZipEntry(base + ".chunks.jsonl"));
|
||||
zip.write(chunksJsonl(result).getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
}
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
/** One chunk per line, each self-describing (documentId + source travel on every line). */
|
||||
private String chunksJsonl(RagIngestResponse result) {
|
||||
if (result.chunks() == null) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder lines = new StringBuilder();
|
||||
for (DocChunk chunk : result.chunks()) {
|
||||
ObjectNode line = objectMapper.createObjectNode();
|
||||
line.put("documentId", result.documentId());
|
||||
line.put("index", chunk.index());
|
||||
line.put("text", chunk.text());
|
||||
if (chunk.pageStart() != null) {
|
||||
line.put("pageStart", chunk.pageStart());
|
||||
}
|
||||
if (chunk.pageEnd() != null) {
|
||||
line.put("pageEnd", chunk.pageEnd());
|
||||
}
|
||||
var headings = line.putArray("headingPath");
|
||||
chunk.headingPath().forEach(headings::add);
|
||||
lines.append(objectMapper.writeValueAsString(line)).append('\n');
|
||||
}
|
||||
return lines.toString();
|
||||
}
|
||||
|
||||
private static String baseName(String fileName) {
|
||||
int dot = fileName.lastIndexOf('.');
|
||||
return dot > 0 ? fileName.substring(0, dot) : fileName;
|
||||
}
|
||||
|
||||
private void writeParts(TempFile sourceTempFile, List<SplitPart> parts, ZipOutputStream zipOut)
|
||||
throws IOException {
|
||||
for (int i = 0; i < parts.size(); i++) {
|
||||
SplitPart part = parts.get(i);
|
||||
// Load per part and remove pages outside the range: avoids the PDFBox cross-document
|
||||
// addPage pitfalls while keeping shared resources intact.
|
||||
try (PDDocument partDoc = pdfDocumentFactory.load(sourceTempFile.getFile())) {
|
||||
int pageCount = partDoc.getNumberOfPages();
|
||||
int start = Math.clamp(part.startPage(), 1, pageCount);
|
||||
int end = Math.clamp(part.endPage(), start, pageCount);
|
||||
for (int p = pageCount - 1; p >= 0; p--) {
|
||||
int pageNumber = p + 1;
|
||||
if (pageNumber < start || pageNumber > end) {
|
||||
partDoc.removePage(p);
|
||||
}
|
||||
}
|
||||
FormUtils.pruneOrphanedFormFields(partDoc);
|
||||
zipOut.putNextEntry(new ZipEntry(partEntryName(i, part)));
|
||||
partDoc.save(zipOut);
|
||||
zipOut.closeEntry();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String partEntryName(int index, SplitPart part) {
|
||||
String label = part.label() == null ? "" : part.label().trim();
|
||||
String sanitized = label.replaceAll("[^A-Za-z0-9 ._-]", "_").replaceAll("\\s+", "_");
|
||||
if (sanitized.isBlank() || sanitized.chars().allMatch(c -> c == '_' || c == '.')) {
|
||||
sanitized = "part";
|
||||
}
|
||||
// Index prefix keeps entries unique even when labels repeat.
|
||||
return String.format(Locale.ROOT, "%02d_%s.pdf", index + 1, sanitized);
|
||||
}
|
||||
|
||||
private static String tablesToCsv(List<DocTable> tables) throws IOException {
|
||||
CSVFormat format = CSVFormat.EXCEL.builder().setEscape('"').build();
|
||||
StringWriter writer = new StringWriter();
|
||||
try (CSVPrinter printer = format.print(writer)) {
|
||||
boolean first = true;
|
||||
for (DocTable table : tables) {
|
||||
if (!first) {
|
||||
printer.println();
|
||||
}
|
||||
first = false;
|
||||
for (List<String> row : table.cells()) {
|
||||
printer.printRecord(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
return writer.toString();
|
||||
}
|
||||
|
||||
private static String outputName(MultipartFile file, String suffix) {
|
||||
return GeneralUtils.removeExtension(DocParseService.fileName(file)) + suffix;
|
||||
}
|
||||
}
|
||||
-17
@@ -115,7 +115,6 @@ public class ExternalApiCallController {
|
||||
@RequestParam(value = "headers", required = false) String headers,
|
||||
@RequestParam(value = "includeContext", defaultValue = "false") boolean includeContext,
|
||||
@RequestParam(value = "includeFile", defaultValue = "true") boolean includeFile,
|
||||
@RequestParam(value = "maxRequestBytes", defaultValue = "0") long maxRequestBytes,
|
||||
@RequestHeader(value = InternalApiClient.POLICY_NAME_HEADER, required = false)
|
||||
String policyName,
|
||||
@RequestHeader(value = AutomationRunContext.RUN_ID_HEADER, required = false)
|
||||
@@ -139,17 +138,6 @@ public class ExternalApiCallController {
|
||||
: fileInput.getContentType();
|
||||
byte[] content = fileInput.getBytes();
|
||||
|
||||
// Some destinations cap uploads (Discord's varies with Nitro tier), so the operator
|
||||
// sets the limit; we fail clearly here rather than on an opaque vendor rejection.
|
||||
if (maxRequestBytes > 0 && content.length > maxRequestBytes) {
|
||||
throw new IllegalArgumentException(
|
||||
"The document is "
|
||||
+ megabytes(content.length)
|
||||
+ " MB, over the "
|
||||
+ megabytes(maxRequestBytes)
|
||||
+ " MB limit set for this step.");
|
||||
}
|
||||
|
||||
ObjectNode context =
|
||||
DocumentContext.build(fileInput, content, policyName, runId, objectMapper);
|
||||
|
||||
@@ -557,11 +545,6 @@ public class ExternalApiCallController {
|
||||
: oneLine.substring(0, MAX_REPORT_BODY_CHARS) + "…";
|
||||
}
|
||||
|
||||
/** Bytes as MB to one decimal, for a size message an operator reads in the units they set. */
|
||||
private static String megabytes(long bytes) {
|
||||
return String.format(Locale.ROOT, "%.1f", bytes / (1024.0 * 1024.0));
|
||||
}
|
||||
|
||||
private static String safeFileName(String originalFilename) {
|
||||
String name = Filenames.toSimpleFileName(originalFilename);
|
||||
return (name == null || name.isBlank()) ? "document" : name;
|
||||
|
||||
+2
-13
@@ -21,8 +21,6 @@ import tools.jackson.databind.node.StringNode;
|
||||
*/
|
||||
final class Placeholders {
|
||||
|
||||
private static final int MAX_TREE_DEPTH = 64;
|
||||
|
||||
private static final Pattern PLACEHOLDER = Pattern.compile("\\{\\{\\s*([\\w.]+)\\s*}}");
|
||||
|
||||
/** How a resolved value is escaped for the position it lands in. */
|
||||
@@ -70,24 +68,15 @@ final class Placeholders {
|
||||
* documents[0].data} as readily as a flat field - without a connector per vendor.
|
||||
*/
|
||||
static JsonNode resolveTree(JsonNode node, JsonNode context) {
|
||||
return resolveTree(node, context, 0);
|
||||
}
|
||||
|
||||
private static JsonNode resolveTree(JsonNode node, JsonNode context, int depth) {
|
||||
// Deeply nested JSON is left untouched rather than recursed into, so a
|
||||
// pathological template cannot overflow the stack.
|
||||
if (depth > MAX_TREE_DEPTH) {
|
||||
return node;
|
||||
}
|
||||
if (node instanceof ObjectNode object) {
|
||||
for (String name : new java.util.ArrayList<>(object.propertyNames())) {
|
||||
object.set(name, resolveTree(object.get(name), context, depth + 1));
|
||||
object.set(name, resolveTree(object.get(name), context));
|
||||
}
|
||||
return object;
|
||||
}
|
||||
if (node instanceof ArrayNode array) {
|
||||
for (int i = 0; i < array.size(); i++) {
|
||||
array.set(i, resolveTree(array.get(i), context, depth + 1));
|
||||
array.set(i, resolveTree(array.get(i), context));
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ChunkDocumentApiRequest extends PDFFile {
|
||||
|
||||
@Schema(description = "Target chunk size in characters (64-32768)", defaultValue = "512")
|
||||
private int chunkSize = 512;
|
||||
|
||||
@Schema(
|
||||
description = "Overlap between adjacent chunks in characters (0-4096)",
|
||||
defaultValue = "64")
|
||||
private int overlap = 64;
|
||||
|
||||
@Schema(
|
||||
description = "Tier to use: 'auto' picks per document, or force 'basic'/'advanced'",
|
||||
allowableValues = {"auto", "basic", "advanced"},
|
||||
defaultValue = "auto")
|
||||
private String mode = "auto";
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ExtractFieldsApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description = "JSON Schema object describing the fields to extract, as a JSON string",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
example =
|
||||
"{\"type\":\"object\",\"properties\":{\"invoiceNumber\":{\"type\":\"string\"}}}")
|
||||
private String fieldsSchema;
|
||||
|
||||
@Schema(
|
||||
description = "Tier to use: 'auto' picks per document, or force 'basic'/'advanced'",
|
||||
allowableValues = {"auto", "basic", "advanced"},
|
||||
defaultValue = "auto")
|
||||
private String mode = "auto";
|
||||
|
||||
@Schema(description = "Optional natural-language guidance for the extraction")
|
||||
private String instructions;
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ExtractTablesApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description = "Response format: CSV text or the structured JSON table list",
|
||||
allowableValues = {"csv", "json"},
|
||||
defaultValue = "csv")
|
||||
private String outputFormat = "csv";
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class ParseDocumentApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description = "Tier to use: 'auto' picks per document, or force 'basic'/'advanced'",
|
||||
allowableValues = {"auto", "basic", "advanced"},
|
||||
defaultValue = "auto")
|
||||
private String mode = "auto";
|
||||
|
||||
@Schema(
|
||||
description = "Apply OCR when parsing scanned pages (advanced tier only)",
|
||||
defaultValue = "true")
|
||||
private boolean withOcr = true;
|
||||
|
||||
@Schema(
|
||||
description = "Response format: full JSON result or the markdown rendering only",
|
||||
allowableValues = {"json", "markdown"},
|
||||
defaultValue = "json")
|
||||
private String outputFormat = "json";
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class RagAskApiRequest {
|
||||
|
||||
@Schema(
|
||||
description = "Question to answer from the indexed documents",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String question;
|
||||
|
||||
@Schema(description = "Number of passages to ground the answer on (1-20)", defaultValue = "5")
|
||||
private int topK = 5;
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class RagIngestApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Stable identifier for the ingested document; re-ingesting the same id replaces"
|
||||
+ " its chunks. Defaults to a content hash of the uploaded bytes.")
|
||||
private String documentId;
|
||||
|
||||
@Schema(description = "Target chunk size in characters (64-32768)", defaultValue = "512")
|
||||
private int chunkSize = 512;
|
||||
|
||||
@Schema(
|
||||
description = "Overlap between adjacent chunks in characters (0-4096)",
|
||||
defaultValue = "64")
|
||||
private int overlap = 64;
|
||||
|
||||
@Schema(
|
||||
description = "Tier to use: 'auto' picks per document, or force 'basic'/'advanced'",
|
||||
allowableValues = {"auto", "basic", "advanced"},
|
||||
defaultValue = "auto")
|
||||
private String mode = "auto";
|
||||
|
||||
@Schema(
|
||||
description = "Index the document into the built-in knowledge base",
|
||||
defaultValue = "true")
|
||||
private boolean index = true;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Also return the parsed document as a markdown file, for delivery to external"
|
||||
+ " systems (vector DBs, training corpora)",
|
||||
defaultValue = "false")
|
||||
private boolean exportMarkdown = false;
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Also return the chunks as a JSONL file (one chunk per line with page span and"
|
||||
+ " heading breadcrumb), ready for external embedding or indexing",
|
||||
defaultValue = "false")
|
||||
private boolean exportChunksJsonl = false;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class RagSearchApiRequest {
|
||||
|
||||
@Schema(
|
||||
description = "Natural-language search query",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String query;
|
||||
|
||||
@Schema(description = "Number of passages to return (1-50)", defaultValue = "10")
|
||||
private int topK = 10;
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class SmartSplitApiRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description = "Natural-language boundary rule, e.g. 'split where a new invoice starts'",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String rule;
|
||||
|
||||
@Schema(description = "Maximum number of parts to produce (1-500)", defaultValue = "50")
|
||||
private int maxParts = 50;
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.proprietary.model.api.docparse;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class SuggestSchemaApiRequest extends PDFFile {
|
||||
|
||||
@Schema(description = "Maximum number of fields to suggest (1-20)", defaultValue = "10")
|
||||
private int maxFields = 10;
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/docparse/chunk}. */
|
||||
public record ChunkDocumentRequest(
|
||||
String fileName,
|
||||
List<AiPageText> pages,
|
||||
String contentBase64,
|
||||
int chunkSize,
|
||||
int overlap,
|
||||
DocparseMode mode) {}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Engine response for {@code POST /api/v1/docparse/chunk}. */
|
||||
public record ChunkDocumentResponse(DocparseTier mode, List<DocChunk> chunks) {
|
||||
|
||||
public ChunkDocumentResponse {
|
||||
chunks = chunks == null ? List.of() : chunks;
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* One layout block. {@code bbox} is [x0, y0, x1, y1] normalized to 0..1 with a top-left origin;
|
||||
* {@code null} in basic tier (no layout model ran). Mirrors {@code docparse.py DocBlock}.
|
||||
*/
|
||||
public record DocBlock(String type, String text, int page, List<Double> bbox, Double confidence) {}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** One RAG chunk with page span and heading breadcrumb. Mirrors {@code docparse.py DocChunk}. */
|
||||
public record DocChunk(
|
||||
int index, String text, Integer pageStart, Integer pageEnd, List<String> headingPath) {
|
||||
|
||||
public DocChunk {
|
||||
headingPath = headingPath == null ? List.of() : headingPath;
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** One extracted table. Mirrors {@code docparse.py DocTable}. */
|
||||
public record DocTable(
|
||||
int page, List<Double> bbox, List<List<String>> cells, String markdown, Double confidence) {
|
||||
|
||||
public DocTable {
|
||||
cells = cells == null ? List.of() : cells;
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* What the engine can actually do right now; Java caches and republishes this. Mirrors {@code
|
||||
* docparse.py DocparseCapabilities}.
|
||||
*/
|
||||
public record DocparseCapabilities(
|
||||
boolean advancedInstalled,
|
||||
String doclingVersion,
|
||||
String torchVersion,
|
||||
boolean modelsAvailable,
|
||||
String modelsPath,
|
||||
List<String> errors) {
|
||||
|
||||
public DocparseCapabilities {
|
||||
errors = errors == null ? List.of() : errors;
|
||||
}
|
||||
|
||||
/** The addon-absent view used when the engine is disabled, unreachable, or probing failed. */
|
||||
public static DocparseCapabilities absent(String reason) {
|
||||
return new DocparseCapabilities(false, null, null, false, null, List.of(reason));
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Merged capability view served by {@code GET /api/v1/docparse/capabilities} (Java side). */
|
||||
public record DocparseCapabilitiesView(
|
||||
boolean enabled,
|
||||
String mode,
|
||||
boolean advancedInstalled,
|
||||
boolean engineReachable,
|
||||
String doclingVersion) {}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* What the caller asked for; {@code AUTO} resolves per request. Wire values are lowercase to match
|
||||
* {@code engine/src/stirling/contracts/docparse.py DocparseMode}.
|
||||
*/
|
||||
public enum DocparseMode {
|
||||
AUTO("auto"),
|
||||
BASIC("basic"),
|
||||
ADVANCED("advanced");
|
||||
|
||||
private final String wire;
|
||||
|
||||
DocparseMode(String wire) {
|
||||
this.wire = wire;
|
||||
}
|
||||
|
||||
@JsonValue
|
||||
public String wire() {
|
||||
return wire;
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public static DocparseMode fromWire(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return AUTO;
|
||||
}
|
||||
return valueOf(value.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Which implementation actually served a request. Wire values are lowercase to match {@code
|
||||
* engine/src/stirling/contracts/docparse.py DocparseTier}.
|
||||
*/
|
||||
public enum DocparseTier {
|
||||
BASIC("basic"),
|
||||
ADVANCED("advanced");
|
||||
|
||||
private final String wire;
|
||||
|
||||
DocparseTier(String wire) {
|
||||
this.wire = wire;
|
||||
}
|
||||
|
||||
@JsonValue
|
||||
public String wire() {
|
||||
return wire;
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public static DocparseTier fromWire(String value) {
|
||||
return valueOf(value.trim().toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Engine response for {@code GET /api/v1/documents/stats}: the RAG document store totals. */
|
||||
public record DocumentStoreStats(
|
||||
String backend, long documents, long chunks, String embeddingModel) {}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.AiPageText;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* Engine request for {@code POST /api/v1/docparse/extract}. {@code pages} drives the basic tier
|
||||
* (Java-extracted text); {@code contentBase64} lets the advanced tier parse the raw file itself.
|
||||
*/
|
||||
public record ExtractFieldsRequest(
|
||||
String fileName,
|
||||
JsonNode fieldsSchema,
|
||||
List<AiPageText> pages,
|
||||
String contentBase64,
|
||||
DocparseMode mode,
|
||||
String instructions) {}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Engine response for {@code POST /api/v1/docparse/extract}. */
|
||||
public record ExtractFieldsResponse(
|
||||
DocparseTier mode, List<ExtractedField> fields, double overallConfidence) {
|
||||
|
||||
public ExtractFieldsResponse {
|
||||
fields = fields == null ? List.of() : fields;
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/docparse/tables}. */
|
||||
public record ExtractTablesRequest(String fileName, String contentBase64) {}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Engine response for {@code POST /api/v1/docparse/tables}. */
|
||||
public record ExtractTablesResponse(DocparseTier mode, List<DocTable> tables) {
|
||||
|
||||
public ExtractTablesResponse {
|
||||
tables = tables == null ? List.of() : tables;
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* One extracted field with confidence and citations. Mirrors {@code docparse.py ExtractedField}.
|
||||
*/
|
||||
public record ExtractedField(
|
||||
String name, JsonNode value, double confidence, List<FieldCitation> citations) {
|
||||
|
||||
public ExtractedField {
|
||||
citations = citations == null ? List.of() : citations;
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Where a value came from. {@code quote} is always set; {@code bbox} only when a layout parse ran
|
||||
* (advanced tier); offsets index into the cited page's text. Mirrors {@code docparse.py
|
||||
* FieldCitation}.
|
||||
*/
|
||||
public record FieldCitation(
|
||||
Integer page, List<Double> bbox, String quote, Integer startOffset, Integer endOffset) {}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/docparse/fill-docx}. */
|
||||
public record FillDocxRequest(String templateBase64, JsonNode data) {}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Engine response for {@code POST /api/v1/docparse/fill-docx}. */
|
||||
public record FillDocxResponse(String docxBase64, int replaced, List<String> missing) {
|
||||
|
||||
public FillDocxResponse {
|
||||
missing = missing == null ? List.of() : missing;
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/docparse/parse}. */
|
||||
public record ParseDocumentRequest(String fileName, String contentBase64, boolean withOcr) {}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Engine response for {@code POST /api/v1/docparse/parse}; also produced by the Java basic tier.
|
||||
*/
|
||||
public record ParseDocumentResponse(
|
||||
DocparseTier mode,
|
||||
int pages,
|
||||
List<DocBlock> blocks,
|
||||
List<DocTable> tables,
|
||||
String markdown,
|
||||
boolean ocrApplied) {
|
||||
|
||||
public ParseDocumentResponse {
|
||||
blocks = blocks == null ? List.of() : blocks;
|
||||
tables = tables == null ? List.of() : tables;
|
||||
markdown = markdown == null ? "" : markdown;
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package stirling.software.proprietary.model.docparse;
|
||||
|
||||
/** Engine request for {@code POST /api/v1/documents/ask}: grounded Q&A over the RAG store. */
|
||||
public record RagAskRequest(String question, int topK) {}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user